Reorganize some classes in common module, remove duplicate classes (#12321)
This commit is contained in:
parent
17cd644506
commit
db1d319159
|
|
@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.AccessTokenService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.AlertGroupService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -30,9 +30,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
||||
import java.util.Map;
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.ClusterService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.service.DqExecuteResultService;
|
|||
import org.apache.dolphinscheduler.api.service.DqRuleService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.DataSourceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
|
||||
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
|
||||
import org.apache.dolphinscheduler.service.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
|
||||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.EnvironmentService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.K8sNamespaceService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.K8sNamespace;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.slf4j.Logger;
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.ProjectService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.QueueService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.QueueService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.enums.UdfType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,8 @@ import org.apache.dolphinscheduler.common.enums.FailureStrategy;
|
|||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
||||
import java.util.Map;
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.service.TaskInstanceService;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.TenantService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.UsersService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
|
|||
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
|
|||
import org.apache.dolphinscheduler.api.service.WorkerGroupService;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import springfox.documentation.annotations.ApiIgnore;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
package org.apache.dolphinscheduler.api.k8s;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.ClusterConfUtils;
|
||||
import org.apache.dolphinscheduler.api.utils.ClusterConfUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Cluster;
|
||||
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
|
||||
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
|
||||
|
|
@ -128,4 +128,4 @@ public class K8sManager {
|
|||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.api.dto.ClusterDto;
|
|||
import org.apache.dolphinscheduler.api.enums.Status;
|
||||
import org.apache.dolphinscheduler.api.k8s.K8sManager;
|
||||
import org.apache.dolphinscheduler.api.service.ClusterService;
|
||||
import org.apache.dolphinscheduler.api.utils.ClusterConfUtils;
|
||||
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
||||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.ClusterConfUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||
import org.apache.dolphinscheduler.dao.entity.Cluster;
|
||||
|
|
@ -384,4 +384,3 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ import org.apache.dolphinscheduler.common.enums.UserType;
|
|||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||
|
|
@ -109,6 +108,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
|
|
@ -1004,7 +1004,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
|
|||
|
||||
// check process instances is already running
|
||||
List<ProcessInstance> processInstances = processInstanceService
|
||||
.queryByProcessDefineCodeAndStatus(processDefinition.getCode(), Constants.NOT_TERMINATED_STATES);
|
||||
.queryByProcessDefineCodeAndStatus(processDefinition.getCode(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
|
||||
if (CollectionUtils.isNotEmpty(processInstances)) {
|
||||
throw new ServiceException(Status.DELETE_PROCESS_DEFINITION_EXECUTING_FAIL, processInstances.size());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,11 +37,10 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
|
|
@ -68,6 +67,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
|
|
@ -57,6 +56,7 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.beanutils.BeanMap;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.api.utils.RegexUtils;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
|
|
@ -41,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -304,7 +304,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
|
|||
}
|
||||
|
||||
private List<ProcessInstance> getProcessInstancesByTenant(Tenant tenant) {
|
||||
return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES);
|
||||
return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
|
||||
|
|
@ -56,7 +55,7 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
|
||||
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
|
|
|||
|
|
@ -322,7 +322,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
|
|||
return result;
|
||||
}
|
||||
List<ProcessInstance> processInstances = processInstanceMapper
|
||||
.queryByWorkerGroupNameAndStatus(workerGroup.getName(), Constants.NOT_TERMINATED_STATES);
|
||||
.queryByWorkerGroupNameAndStatus(workerGroup.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES);
|
||||
if (CollectionUtils.isNotEmpty(processInstances)) {
|
||||
List<Integer> processInstanceIds =
|
||||
processInstances.stream().map(ProcessInstance::getId).collect(Collectors.toList());
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.api.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
|
@ -134,7 +134,7 @@ public class WorkerGroupControllerTest extends AbstractControllerTest {
|
|||
workerGroup.setId(12);
|
||||
workerGroup.setName("测试");
|
||||
Mockito.when(workerGroupMapper.selectById(12)).thenReturn(workerGroup);
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", Constants.NOT_TERMINATED_STATES))
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES))
|
||||
.thenReturn(null);
|
||||
Mockito.when(workerGroupMapper.deleteById(12)).thenReturn(1);
|
||||
Mockito.when(processInstanceMapper.updateProcessInstanceByWorkerGroupName("测试", "")).thenReturn(1);
|
||||
|
|
|
|||
|
|
@ -22,9 +22,8 @@ import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
|
|||
import org.apache.dolphinscheduler.api.utils.Result;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.User;
|
||||
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.enums.Flag;
|
|||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
|
|
@ -61,6 +60,7 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Resource;
|
||||
|
|
@ -42,6 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ public class TenantServiceTest {
|
|||
Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.TENANT, null, 0,
|
||||
baseServiceLogger)).thenReturn(true);
|
||||
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
||||
Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES))
|
||||
Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES))
|
||||
.thenReturn(getInstanceList());
|
||||
Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList());
|
||||
Mockito.when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList());
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
|
||||
import org.apache.dolphinscheduler.dao.entity.Project;
|
||||
|
|
@ -49,6 +48,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ public class WorkerGroupServiceTest {
|
|||
user.setUserType(UserType.ADMIN_USER);
|
||||
WorkerGroup wg2 = getWorkerGroup(2);
|
||||
Mockito.when(workerGroupMapper.selectById(2)).thenReturn(wg2);
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList());
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList());
|
||||
Map<String, Object> result = workerGroupService.deleteWorkerGroupById(user, 1);
|
||||
Assert.assertEquals(Status.DELETE_WORKER_GROUP_NOT_EXIST.getCode(), ((Status) result.get(Constants.STATUS)).getCode());
|
||||
result = workerGroupService.deleteWorkerGroupById(user, 2);
|
||||
|
|
@ -97,7 +97,7 @@ public class WorkerGroupServiceTest {
|
|||
// correct
|
||||
WorkerGroup wg3 = getWorkerGroup(3);
|
||||
Mockito.when(workerGroupMapper.selectById(3)).thenReturn(wg3);
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>());
|
||||
Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>());
|
||||
result = workerGroupService.deleteWorkerGroupById(user, 3);
|
||||
Assert.assertEquals(Status.SUCCESS.getMsg(), result.get(Constants.MSG));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,13 +42,8 @@
|
|||
<dependencies>
|
||||
<!-- dolphinscheduler -->
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-spi</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -99,208 +94,6 @@
|
|||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>jdk.tools</groupId>
|
||||
<artifactId>jdk.tools</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
</exclusion>
|
||||
|
||||
<exclusion>
|
||||
<groupId>commons-configuration</groupId>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.grpc</groupId>
|
||||
<artifactId>grpc-protobuf</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>xmlenc</groupId>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-net</groupId>
|
||||
<artifactId>commons-net</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet.jsp</groupId>
|
||||
<artifactId>jsp-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-json</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hbase.thirdparty</groupId>
|
||||
<artifactId>hbase-noop-htrace</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-jaxrs</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-xc</artifactId>
|
||||
</exclusion>
|
||||
|
||||
<exclusion>
|
||||
<groupId>org.fusesource.leveldbjni</groupId>
|
||||
<artifactId>leveldbjni-all</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-client</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
<artifactId>jaxb-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>xmlenc</groupId>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.fusesource.leveldbjni</groupId>
|
||||
<artifactId>leveldbjni-all</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
|
|
|
|||
|
|
@ -17,9 +17,6 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
|
|
@ -642,23 +639,6 @@ public final class Constants {
|
|||
*/
|
||||
public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s";
|
||||
|
||||
public static final int[] NOT_TERMINATED_STATES = new int[]{
|
||||
WorkflowExecutionStatus.SUBMITTED_SUCCESS.getCode(),
|
||||
TaskExecutionStatus.DISPATCH.getCode(),
|
||||
WorkflowExecutionStatus.RUNNING_EXECUTION.getCode(),
|
||||
WorkflowExecutionStatus.DELAY_EXECUTION.getCode(),
|
||||
WorkflowExecutionStatus.READY_PAUSE.getCode(),
|
||||
WorkflowExecutionStatus.READY_STOP.getCode(),
|
||||
TaskExecutionStatus.NEED_FAULT_TOLERANCE.getCode(),
|
||||
};
|
||||
|
||||
public static final int[] RUNNING_PROCESS_STATE = new int[]{
|
||||
TaskExecutionStatus.RUNNING_EXECUTION.getCode(),
|
||||
TaskExecutionStatus.SUBMITTED_SUCCESS.getCode(),
|
||||
TaskExecutionStatus.DISPATCH.getCode(),
|
||||
WorkflowExecutionStatus.SERIAL_WAIT.getCode()
|
||||
};
|
||||
|
||||
/**
|
||||
* status
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -17,6 +17,28 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
||||
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
|
@ -33,26 +55,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
|
|||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.fasterxml.jackson.databind.node.TextNode;
|
||||
import com.fasterxml.jackson.databind.type.CollectionType;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
|
||||
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
||||
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import com.google.common.base.Strings;
|
||||
|
||||
/**
|
||||
* json utils
|
||||
|
|
@ -62,7 +65,7 @@ public class JSONUtils {
|
|||
private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class);
|
||||
|
||||
static {
|
||||
logger.info("init timezone: {}",TimeZone.getDefault());
|
||||
logger.info("init timezone: {}", TimeZone.getDefault());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -129,7 +132,7 @@ public class JSONUtils {
|
|||
* classOfT
|
||||
*/
|
||||
public static @Nullable <T> T parseObject(String json, Class<T> clazz) {
|
||||
if (StringUtils.isEmpty(json)) {
|
||||
if (Strings.isNullOrEmpty(json)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -166,7 +169,7 @@ public class JSONUtils {
|
|||
* @return list
|
||||
*/
|
||||
public static <T> List<T> toList(String json, Class<T> clazz) {
|
||||
if (StringUtils.isEmpty(json)) {
|
||||
if (Strings.isNullOrEmpty(json)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
|
@ -188,7 +191,7 @@ public class JSONUtils {
|
|||
*/
|
||||
public static boolean checkJsonValid(String json) {
|
||||
|
||||
if (StringUtils.isEmpty(json)) {
|
||||
if (Strings.isNullOrEmpty(json)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -229,7 +232,8 @@ public class JSONUtils {
|
|||
* @return json to map
|
||||
*/
|
||||
public static Map<String, String> toMap(String json) {
|
||||
return parseObject(json, new TypeReference<Map<String, String>>() {});
|
||||
return parseObject(json, new TypeReference<Map<String, String>>() {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -243,7 +247,7 @@ public class JSONUtils {
|
|||
* @return to map
|
||||
*/
|
||||
public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) {
|
||||
if (StringUtils.isEmpty(json)) {
|
||||
if (Strings.isNullOrEmpty(json)) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
|
|
@ -285,7 +289,7 @@ public class JSONUtils {
|
|||
* @return return parse object
|
||||
*/
|
||||
public static <T> T parseObject(String json, TypeReference<T> type) {
|
||||
if (StringUtils.isEmpty(json)) {
|
||||
if (Strings.isNullOrEmpty(json)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -319,7 +323,7 @@ public class JSONUtils {
|
|||
* @param <T> object type
|
||||
* @return byte array
|
||||
*/
|
||||
public static <T> byte[] toJsonByteArray(T obj) {
|
||||
public static <T> byte[] toJsonByteArray(T obj) {
|
||||
if (obj == null) {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
|
||||
|
||||
import oshi.SystemInfo;
|
||||
|
|
@ -417,12 +418,16 @@ public class OSUtils {
|
|||
* @return result of sudo execute command
|
||||
*/
|
||||
public static String getSudoCmd(String tenantCode, String command) {
|
||||
if (!CommonUtils.isSudoEnable() || StringUtils.isEmpty(tenantCode)) {
|
||||
if (!isSudoEnable() || StringUtils.isEmpty(tenantCode)) {
|
||||
return command;
|
||||
}
|
||||
return String.format("sudo -u %s %s", tenantCode, command);
|
||||
}
|
||||
|
||||
public static boolean isSudoEnable() {
|
||||
return PropertyUtils.getBoolean(Constants.SUDO_ENABLE, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the corresponding command of Linux or Windows
|
||||
*
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.common.utils;
|
|||
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
|
|
|||
|
|
@ -17,13 +17,13 @@
|
|||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
public class PropertyUtilsTest {
|
||||
|
||||
@Test
|
||||
|
|
@ -32,7 +32,7 @@ public class PropertyUtilsTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void getResUploadStartupState(){
|
||||
public void getResUploadStartupState() {
|
||||
Assert.assertFalse(PropertyUtils.getResUploadStartupState());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils.placeholder;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TimePlaceholderUtilsTest {
|
||||
|
||||
private Date date;
|
||||
|
||||
@BeforeClass
|
||||
public static void setup() {
|
||||
TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai"));
|
||||
}
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
date = DateUtils.parse("20170101010101", "yyyyMMddHHmmss", null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void replacePlaceholdersT() {
|
||||
Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils
|
||||
.replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]",
|
||||
date, true));
|
||||
|
||||
Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)],"
|
||||
+ "$[timestamp(month_begin(yyyyMMddHHmmss, 1))],"
|
||||
+ "$[timestamp(month_end(yyyyMMddHHmmss, -1))],"
|
||||
+ "$[timestamp(week_begin(yyyyMMddHHmmss, 1))],"
|
||||
+ "$[timestamp(week_end(yyyyMMdd000000, -1))],"
|
||||
+ "$[timestamp(yyyyMMddHHmmss)]",
|
||||
date, true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calcMinutesT() {
|
||||
Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString());
|
||||
Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString());
|
||||
Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString());
|
||||
Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString());
|
||||
Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString());
|
||||
Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString());
|
||||
Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calcMonthsT() {
|
||||
Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString());
|
||||
Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetPlaceHolderTime() {
|
||||
|
||||
Assert.assertEquals("20170101", TimePlaceholderUtils.getPlaceHolderTime("yyyyMMdd", date));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -15,7 +15,6 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
|
@ -27,6 +26,18 @@
|
|||
<artifactId>dolphinscheduler-dao</artifactId>
|
||||
<name>${project.artifactId}</name>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-bom</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- dolphinscheduler -->
|
||||
<dependency>
|
||||
|
|
@ -34,6 +45,11 @@
|
|||
<artifactId>dolphinscheduler-common</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
|
|
@ -43,8 +59,8 @@
|
|||
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j-to-slf4j</artifactId>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-to-slf4j</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
|
@ -98,18 +114,6 @@
|
|||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-bom</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
|
|
|
|||
|
|
@ -1,115 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.dao.entity;
|
||||
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* definition json data structure
|
||||
*/
|
||||
public class ProcessData {
|
||||
/**
|
||||
* task list
|
||||
*/
|
||||
private List<TaskNode> tasks;
|
||||
|
||||
/**
|
||||
* global parameters
|
||||
*/
|
||||
private List<Property> globalParams;
|
||||
|
||||
|
||||
private int timeout;
|
||||
|
||||
private int tenantId;
|
||||
|
||||
|
||||
public ProcessData() {
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param tasks tasks
|
||||
* @param globalParams globalParams
|
||||
*/
|
||||
public ProcessData(List<TaskNode> tasks, List<Property> globalParams) {
|
||||
this.tasks = tasks;
|
||||
this.globalParams = globalParams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ProcessData that = (ProcessData) o;
|
||||
|
||||
return CollectionUtils.equalLists(tasks, that.tasks) &&
|
||||
CollectionUtils.equalLists(globalParams, that.globalParams);
|
||||
}
|
||||
|
||||
public List<TaskNode> getTasks() {
|
||||
return tasks;
|
||||
}
|
||||
|
||||
public void setTasks(List<TaskNode> tasks) {
|
||||
this.tasks = tasks;
|
||||
}
|
||||
|
||||
public List<Property> getGlobalParams() {
|
||||
return globalParams;
|
||||
}
|
||||
|
||||
public void setGlobalParams(List<Property> globalParams) {
|
||||
this.globalParams = globalParams;
|
||||
}
|
||||
|
||||
public int getTimeout() {
|
||||
return timeout;
|
||||
}
|
||||
|
||||
public void setTimeout(int timeout) {
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public int getTenantId() {
|
||||
return tenantId;
|
||||
}
|
||||
|
||||
public void setTenantId(int tenantId) {
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ProcessData{" +
|
||||
"tasks=" + tasks +
|
||||
", globalParams=" + globalParams +
|
||||
", timeout=" + timeout +
|
||||
", tenantId=" + tenantId +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils;
|
|||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*;
|
||||
import static org.apache.dolphinscheduler.spi.utils.Constants.RESOURCE_STORAGE_TYPE;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.utils;
|
|||
|
||||
import static org.apache.dolphinscheduler.spi.utils.Constants.JAVA_SECURITY_KRB5_CONF;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.spi.utils.Constants;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
|
|
@ -30,10 +30,10 @@ import org.apache.hadoop.security.UserGroupInformation;
|
|||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
|
||||
public class CommonUtil {
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
private CommonUtil() {
|
||||
}
|
||||
@UtilityClass
|
||||
public class CommonUtil {
|
||||
|
||||
public static boolean getKerberosStartupState() {
|
||||
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import io.netty.channel.Channel;
|
|||
import org.apache.dolphinscheduler.common.enums.StateEventType;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
import org.apache.dolphinscheduler.remote.command.WorkflowStateEventChangeCommand;
|
||||
|
|
@ -31,6 +30,7 @@ import org.apache.dolphinscheduler.server.master.event.StateEvent;
|
|||
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -21,13 +21,13 @@ import com.google.common.base.Preconditions;
|
|||
import io.netty.channel.Channel;
|
||||
import org.apache.dolphinscheduler.common.enums.StateEventType;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand;
|
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
|
||||
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -18,14 +18,13 @@
|
|||
package org.apache.dolphinscheduler.server.master.processor;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
|
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -18,14 +18,13 @@
|
|||
package org.apache.dolphinscheduler.server.master.processor;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
import org.apache.dolphinscheduler.remote.command.TaskRejectCommand;
|
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
|
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ package org.apache.dolphinscheduler.server.master.processor.queue;
|
|||
import io.netty.channel.Channel;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand;
|
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
|
||||
import org.apache.dolphinscheduler.server.master.event.StateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
|
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -18,11 +18,10 @@
|
|||
package org.apache.dolphinscheduler.server.master.processor.queue;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.TaskEventType;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.server.master.event.TaskEventHandleError;
|
||||
import org.apache.dolphinscheduler.server.master.event.TaskEventHandleException;
|
||||
import org.apache.dolphinscheduler.server.master.event.TaskEventHandler;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,10 +20,9 @@ package org.apache.dolphinscheduler.server.master.runner;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
|
||||
import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.SlotCheckState;
|
|||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.OSUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Command;
|
||||
|
|
@ -41,7 +40,7 @@ import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
|
|||
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
|||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
|
|
@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|||
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Environment;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
||||
|
|
@ -58,6 +57,7 @@ import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
|
|||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
|
|
|||
|
|
@ -17,11 +17,10 @@
|
|||
|
||||
package org.apache.dolphinscheduler.server.master.runner;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
|
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
|
|
|||
|
|
@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEvent;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleError;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleException;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandler;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue;
|
||||
import org.apache.dolphinscheduler.server.master.event.WorkflowEventType;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -40,13 +40,10 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType;
|
|||
import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Command;
|
||||
import org.apache.dolphinscheduler.dao.entity.Environment;
|
||||
|
|
@ -59,7 +56,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
|
|||
import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
|
||||
import org.apache.dolphinscheduler.dao.utils.DagHelper;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
|
|
@ -83,9 +79,12 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
|
|||
import org.apache.dolphinscheduler.service.cron.CronUtils;
|
||||
import org.apache.dolphinscheduler.service.exceptions.CronParseException;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
|
||||
|
||||
import org.apache.dolphinscheduler.service.utils.DagHelper;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import com.google.common.base.Strings;
|
|||
import lombok.NonNull;
|
||||
import org.apache.dolphinscheduler.common.enums.Flag;
|
||||
import org.apache.dolphinscheduler.common.enums.StateEventType;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
|
|
@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|||
import org.apache.dolphinscheduler.server.master.event.StateEvent;
|
||||
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
|
|||
|
|
@ -36,9 +36,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst
|
|||
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TARGET_DATASOURCE_ID;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.DataSource;
|
||||
import org.apache.dolphinscheduler.dao.entity.DqComparisonType;
|
||||
|
|
@ -77,7 +75,9 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.DbType;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
import org.apache.dolphinscheduler.spi.plugin.SPIIdentify;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.service;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.NodeType;
|
||||
import org.apache.dolphinscheduler.common.model.Server;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
|
|
@ -39,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactor
|
|||
import org.apache.dolphinscheduler.service.log.LogClient;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.registry.RegistryClient;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.service.utils.ProcessUtils;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.Flag;
|
|||
import org.apache.dolphinscheduler.common.enums.NodeType;
|
||||
import org.apache.dolphinscheduler.common.enums.StateEventType;
|
||||
import org.apache.dolphinscheduler.common.model.Server;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
||||
|
|
@ -38,6 +37,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactor
|
|||
import org.apache.dolphinscheduler.service.log.LogClient;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.apache.dolphinscheduler.service.registry.RegistryClient;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.service.utils.ProcessUtils;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYP
|
|||
|
||||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
|
@ -37,6 +36,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|||
import org.apache.dolphinscheduler.server.master.runner.task.BlockingTaskProcessor;
|
||||
import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
|
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.time.Duration;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
|
@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParamete
|
|||
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
|
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.time.Duration;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
|
@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
|
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
package org.apache.dolphinscheduler.server.master;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
|
||||
|
||||
import java.util.Calendar;
|
||||
|
|
@ -62,4 +62,4 @@ public class ParamsTest {
|
|||
logger.info("complement data : {}",command);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master;
|
|||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
|
||||
|
|
@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|||
import org.apache.dolphinscheduler.server.master.runner.task.SubTaskProcessor;
|
||||
import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
|
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessService;
|
||||
|
||||
import java.time.Duration;
|
||||
|
|
|
|||
|
|
@ -17,23 +17,39 @@
|
|||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<artifactId>dolphinscheduler</artifactId>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>dolphinscheduler-remote</artifactId>
|
||||
|
||||
<name>dolphinscheduler-remote</name>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-bom</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- dolphinscheduler -->
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
|
|
@ -73,16 +89,4 @@
|
|||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-bom</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -17,17 +17,18 @@
|
|||
|
||||
package org.apache.dolphinscheduler.remote.utils;
|
||||
|
||||
import lombok.NonNull;
|
||||
import static org.apache.dolphinscheduler.common.Constants.COLON;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
/**
|
||||
* server address
|
||||
*/
|
||||
public class Host implements Serializable {
|
||||
|
||||
private static final String COLON = ":";
|
||||
|
||||
public static final Host EMPTY = new Host();
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<scope>test</scope>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
|
|
@ -178,6 +178,10 @@
|
|||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
@ -187,7 +191,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<scope>test</scope>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
|
|
@ -236,5 +240,55 @@
|
|||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>xmlenc</groupId>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.fusesource.leveldbjni</groupId>
|
||||
<artifactId>leveldbjni-all</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
|
|
@ -32,6 +31,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import java.util.Date;
|
||||
|
|
@ -42,11 +42,11 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
@Component
|
||||
public class CuringGlobalParams implements CuringParamsService {
|
||||
|
||||
|
|
@ -79,7 +79,9 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
* @return
|
||||
*/
|
||||
@Override
|
||||
public String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap, List<Property> globalParamList, CommandType commandType, Date scheduleTime, String timezone) {
|
||||
public String curingGlobalParams(Integer processInstanceId, Map<String, String> globalParamMap,
|
||||
List<Property> globalParamList, CommandType commandType, Date scheduleTime,
|
||||
String timezone) {
|
||||
if (globalParamList == null || globalParamList.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -88,9 +90,8 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
globalMap.putAll(globalParamMap);
|
||||
}
|
||||
Map<String, String> allParamMap = new HashMap<>();
|
||||
//If it is a complement, a complement time needs to be passed in, according to the task type
|
||||
Map<String, String> timeParams = BusinessTimeUtils.
|
||||
getBusinessTime(commandType, scheduleTime, timezone);
|
||||
// If it is a complement, a complement time needs to be passed in, according to the task type
|
||||
Map<String, String> timeParams = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timezone);
|
||||
|
||||
if (timeParams != null) {
|
||||
allParamMap.putAll(timeParams);
|
||||
|
|
@ -130,9 +131,11 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
* @return
|
||||
*/
|
||||
@Override
|
||||
public Map<String, Property> paramParsingPreparation(@NonNull TaskInstance taskInstance, @NonNull AbstractParameters parameters, @NonNull ProcessInstance processInstance) {
|
||||
public Map<String, Property> paramParsingPreparation(@NonNull TaskInstance taskInstance,
|
||||
@NonNull AbstractParameters parameters,
|
||||
@NonNull ProcessInstance processInstance) {
|
||||
// assign value to definedParams here
|
||||
Map<String,String> globalParamsMap = setGlobalParamsMap(processInstance);
|
||||
Map<String, String> globalParamsMap = setGlobalParamsMap(processInstance);
|
||||
Map<String, Property> globalParams = ParamUtils.getUserDefParamsMap(globalParamsMap);
|
||||
CommandType commandType = processInstance.getCmdTypeIfComplement();
|
||||
Date scheduleTime = processInstance.getScheduleTime();
|
||||
|
|
@ -140,7 +143,7 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
// combining local and global parameters
|
||||
Map<String, Property> localParams = parameters.getInputLocalParametersMap();
|
||||
|
||||
//stream pass params
|
||||
// stream pass params
|
||||
parameters.setVarPool(taskInstance.getVarPool());
|
||||
Map<String, Property> varParams = parameters.getVarPoolMap();
|
||||
|
||||
|
|
@ -152,7 +155,7 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
// of the process instance complement
|
||||
Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());
|
||||
String timeZone = cmdParam.get(Constants.SCHEDULE_TIMEZONE);
|
||||
Map<String,String> params = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timeZone);
|
||||
Map<String, String> params = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timeZone);
|
||||
|
||||
if (globalParamsMap != null) {
|
||||
params.putAll(globalParamsMap);
|
||||
|
|
@ -175,7 +178,8 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
Map.Entry<String, Property> en = iter.next();
|
||||
Property property = en.getValue();
|
||||
|
||||
if (StringUtils.isNotEmpty(property.getValue()) && property.getValue().startsWith(Constants.FUNCTION_START_WITH)) {
|
||||
if (StringUtils.isNotEmpty(property.getValue())
|
||||
&& property.getValue().startsWith(Constants.FUNCTION_START_WITH)) {
|
||||
/**
|
||||
* local parameter refers to global parameter with the same name
|
||||
* note: the global parameters of the process instance here are solidified parameters,
|
||||
|
|
@ -186,7 +190,7 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
if (timeFunctionNeedExpand(val)) {
|
||||
val = timeFunctionExtension(taskInstance.getProcessInstanceId(), timeZone, val);
|
||||
} else {
|
||||
val = convertParameterPlaceholders(val, params);
|
||||
val = convertParameterPlaceholders(val, params);
|
||||
}
|
||||
property.setValue(val);
|
||||
}
|
||||
|
|
@ -209,7 +213,8 @@ public class CuringGlobalParams implements CuringParamsService {
|
|||
String globalParamsStr = processInstance.getGlobalParams();
|
||||
if (globalParamsStr != null) {
|
||||
List<Property> globalParamsList = JSONUtils.toList(globalParamsStr, Property.class);
|
||||
globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)));
|
||||
globalParamsMap
|
||||
.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)));
|
||||
}
|
||||
return globalParamsMap;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.dolphinscheduler.service.log;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
|
||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
|
||||
|
|
@ -36,6 +35,7 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
|
|||
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
|
||||
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
|
||||
import org.apache.dolphinscheduler.remote.utils.Host;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.dolphinscheduler.service.log;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
|
|
@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
|
|||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
|
||||
import org.apache.dolphinscheduler.remote.utils.Constants;
|
||||
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
|
|
|||
|
|
@ -15,20 +15,20 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.model;
|
||||
package org.apache.dolphinscheduler.service.model;
|
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.Priority;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
|
||||
import org.apache.dolphinscheduler.common.model.PreviousTaskNode;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
|
@ -37,6 +37,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
|
|
@ -139,7 +140,6 @@ public class TaskNode {
|
|||
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
|
||||
private String dependence;
|
||||
|
||||
|
||||
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class)
|
||||
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
|
||||
private String conditionResult;
|
||||
|
|
@ -402,7 +402,8 @@ public class TaskNode {
|
|||
*/
|
||||
public TaskTimeoutParameter getTaskTimeoutParameter() {
|
||||
if (!StringUtils.isEmpty(this.getTimeout())) {
|
||||
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
|
||||
String formatStr =
|
||||
String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
|
||||
String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name());
|
||||
return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class);
|
||||
}
|
||||
|
|
@ -430,7 +431,8 @@ public class TaskNode {
|
|||
}
|
||||
|
||||
public String getTaskParams() {
|
||||
Map<String, Object> taskParams = JSONUtils.parseObject(this.params, new TypeReference<Map<String, Object>>() {});
|
||||
Map<String, Object> taskParams = JSONUtils.parseObject(this.params, new TypeReference<Map<String, Object>>() {
|
||||
});
|
||||
|
||||
if (taskParams == null) {
|
||||
taskParams = new HashMap<>();
|
||||
|
|
@ -443,7 +445,8 @@ public class TaskNode {
|
|||
}
|
||||
|
||||
public Map<String, Object> taskParamsToJsonObj(String taskParams) {
|
||||
Map<String, Object> taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference<Map<String, Object>>() {});
|
||||
Map<String, Object> taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference<Map<String, Object>>() {
|
||||
});
|
||||
if (taskParamsMap == null) {
|
||||
taskParamsMap = new HashMap<>();
|
||||
}
|
||||
|
|
@ -15,10 +15,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.process;
|
||||
package org.apache.dolphinscheduler.service.process;
|
||||
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.service.process;
|
|||
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.Command;
|
||||
|
|
@ -53,6 +52,7 @@ import org.apache.dolphinscheduler.dao.entity.User;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval;
|
||||
import org.apache.dolphinscheduler.service.exceptions.CronParseException;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import java.util.Date;
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
|
|||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
|
|
@ -113,7 +111,6 @@ import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
|
|||
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
||||
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
|
||||
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
|
||||
import org.apache.dolphinscheduler.dao.utils.DagHelper;
|
||||
import org.apache.dolphinscheduler.dao.utils.DqRuleUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
|
|
@ -134,7 +131,9 @@ import org.apache.dolphinscheduler.service.exceptions.CronParseException;
|
|||
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.log.LogClient;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
import org.apache.dolphinscheduler.service.utils.DagHelper;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
|
|
@ -333,7 +332,8 @@ public class ProcessServiceImpl implements ProcessService {
|
|||
List<ProcessInstance> runningProcessInstances =
|
||||
this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(
|
||||
processInstance.getProcessDefinitionCode(),
|
||||
processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getProcessDefinitionVersion(),
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getId());
|
||||
if (CollectionUtils.isEmpty(runningProcessInstances)) {
|
||||
processInstance.setStateWithDesc(WorkflowExecutionStatus.SUBMITTED_SUCCESS,
|
||||
|
|
@ -344,7 +344,8 @@ public class ProcessServiceImpl implements ProcessService {
|
|||
List<ProcessInstance> runningProcessInstances =
|
||||
this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(
|
||||
processInstance.getProcessDefinitionCode(),
|
||||
processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getProcessDefinitionVersion(),
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getId());
|
||||
if (CollectionUtils.isNotEmpty(runningProcessInstances)) {
|
||||
processInstance.setStateWithDesc(WorkflowExecutionStatus.STOP, "stop by serial_discard strategy");
|
||||
|
|
@ -358,7 +359,8 @@ public class ProcessServiceImpl implements ProcessService {
|
|||
List<ProcessInstance> runningProcessInstances =
|
||||
this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(
|
||||
processInstance.getProcessDefinitionCode(),
|
||||
processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getProcessDefinitionVersion(),
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE,
|
||||
processInstance.getId());
|
||||
for (ProcessInstance info : runningProcessInstances) {
|
||||
info.setCommandType(CommandType.STOP);
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.storage;
|
||||
package org.apache.dolphinscheduler.service.storage;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
|
|
@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
|||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public interface StorageOperate {
|
||||
|
||||
String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
|
||||
|
|
@ -127,7 +126,8 @@ public interface StorageOperate {
|
|||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
|
||||
boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource,
|
||||
boolean overwrite) throws IOException;
|
||||
|
||||
/**
|
||||
* download the srcPath to local
|
||||
|
|
@ -138,7 +138,8 @@ public interface StorageOperate {
|
|||
* @param overwrite
|
||||
* @throws IOException
|
||||
*/
|
||||
void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite)throws IOException;
|
||||
void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource,
|
||||
boolean overwrite) throws IOException;
|
||||
|
||||
/**
|
||||
* vim the context of filePath
|
||||
|
|
@ -15,18 +15,17 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.config;
|
||||
package org.apache.dolphinscheduler.service.storage;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_STORAGE_TYPE;
|
||||
import static org.apache.dolphinscheduler.common.Constants.STORAGE_HDFS;
|
||||
import static org.apache.dolphinscheduler.common.Constants.STORAGE_OSS;
|
||||
import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3;
|
||||
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.OssOperator;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.S3Utils;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.OssOperator;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.S3Utils;
|
||||
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.storage.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
|
||||
import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S;
|
||||
|
|
@ -25,8 +25,13 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_TYPE_UDF;
|
|||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.exception.BaseException;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.HttpUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.KerberosHttpClient;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.service.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
|
@ -350,15 +355,15 @@ public class HadoopUtils implements Closeable, StorageOperate {
|
|||
* copy hdfs file to local
|
||||
*
|
||||
* @param srcHdfsFilePath source hdfs file path
|
||||
*
|
||||
*
|
||||
* @param dstFile destination file
|
||||
*
|
||||
*
|
||||
* @param deleteSource delete source
|
||||
*
|
||||
*
|
||||
* @param overwrite overwrite
|
||||
*
|
||||
*
|
||||
* @return result of copy hdfs file to local
|
||||
*
|
||||
*
|
||||
* @throws IOException errors
|
||||
*/
|
||||
public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource,
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.storage.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
|
||||
import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S;
|
||||
|
|
@ -26,8 +26,9 @@ import org.apache.dolphinscheduler.common.Constants;
|
|||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.factory.OssClientFactory;
|
||||
import org.apache.dolphinscheduler.common.model.OssConnection;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.storage.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.AWS_END_POINT;
|
||||
import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
|
||||
|
|
@ -27,8 +27,9 @@ import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3;
|
|||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
||||
import org.apache.dolphinscheduler.service.storage.StorageOperate;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
|
@ -80,7 +81,6 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
|
||||
public static final String BUCKET_NAME = PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME);
|
||||
|
||||
|
||||
private AmazonS3 s3Client = null;
|
||||
|
||||
private S3Utils() {
|
||||
|
|
@ -88,17 +88,20 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
|
||||
if (!StringUtils.isEmpty(PropertyUtils.getString(AWS_END_POINT))) {
|
||||
s3Client = AmazonS3ClientBuilder
|
||||
.standard()
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(PropertyUtils.getString(AWS_END_POINT), Regions.fromName(REGION).getName()))
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
|
||||
.build();
|
||||
.standard()
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
|
||||
PropertyUtils.getString(AWS_END_POINT), Regions.fromName(REGION).getName()))
|
||||
.withCredentials(
|
||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
|
||||
.build();
|
||||
} else {
|
||||
s3Client = AmazonS3ClientBuilder
|
||||
.standard()
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
|
||||
.withRegion(Regions.fromName(REGION))
|
||||
.build();
|
||||
.standard()
|
||||
.withCredentials(
|
||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
|
||||
.withRegion(Regions.fromName(REGION))
|
||||
.build();
|
||||
}
|
||||
checkBucketNameExists(BUCKET_NAME);
|
||||
}
|
||||
|
|
@ -108,6 +111,7 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
* S3Utils single
|
||||
*/
|
||||
private enum S3Singleton {
|
||||
|
||||
INSTANCE;
|
||||
|
||||
private final S3Utils instance;
|
||||
|
|
@ -176,7 +180,8 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource, boolean overwrite) throws IOException {
|
||||
public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource,
|
||||
boolean overwrite) throws IOException {
|
||||
File dstFile = new File(dstFilePath);
|
||||
if (dstFile.isDirectory()) {
|
||||
Files.delete(dstFile.toPath());
|
||||
|
|
@ -184,8 +189,9 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
Files.createDirectories(dstFile.getParentFile().toPath());
|
||||
}
|
||||
S3Object o = s3Client.getObject(BUCKET_NAME, srcFilePath);
|
||||
try (S3ObjectInputStream s3is = o.getObjectContent();
|
||||
FileOutputStream fos = new FileOutputStream(dstFilePath)) {
|
||||
try (
|
||||
S3ObjectInputStream s3is = o.getObjectContent();
|
||||
FileOutputStream fos = new FileOutputStream(dstFilePath)) {
|
||||
byte[] readBuf = new byte[1024];
|
||||
int readLen;
|
||||
while ((readLen = s3is.read(readBuf)) > 0) {
|
||||
|
|
@ -236,7 +242,8 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException {
|
||||
public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource,
|
||||
boolean overwrite) throws IOException {
|
||||
try {
|
||||
s3Client.putObject(BUCKET_NAME, dstPath, new File(srcFile));
|
||||
return true;
|
||||
|
|
@ -322,7 +329,6 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
s3Client.putObject(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(strPath));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* xxx untest
|
||||
* download S3 Directory to local
|
||||
|
|
@ -334,10 +340,12 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
private void downloadDirectory(String tenantCode, String keyPrefix, String srcPath) {
|
||||
TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3Client).build();
|
||||
try {
|
||||
MultipleFileDownload download = tm.downloadDirectory(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(srcPath));
|
||||
MultipleFileDownload download =
|
||||
tm.downloadDirectory(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(srcPath));
|
||||
download.waitForCompletion();
|
||||
} catch (AmazonS3Exception | InterruptedException e) {
|
||||
logger.error("download the directory failed with the bucketName is {} and the keyPrefix is {}", BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix);
|
||||
logger.error("download the directory failed with the bucketName is {} and the keyPrefix is {}", BUCKET_NAME,
|
||||
tenantCode + FOLDER_SEPARATOR + keyPrefix);
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
tm.shutdownNow();
|
||||
|
|
@ -350,16 +358,17 @@ public class S3Utils implements Closeable, StorageOperate {
|
|||
}
|
||||
|
||||
Bucket existsBucket = s3Client.listBuckets()
|
||||
.stream()
|
||||
.filter(
|
||||
bucket -> bucket.getName().equals(bucketName)
|
||||
)
|
||||
.findFirst()
|
||||
.orElseThrow(() -> {
|
||||
return new IllegalArgumentException("bucketName: " + bucketName + " is not exists, you need to create them by yourself");
|
||||
});
|
||||
.stream()
|
||||
.filter(
|
||||
bucket -> bucket.getName().equals(bucketName))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> {
|
||||
return new IllegalArgumentException(
|
||||
"bucketName: " + bucketName + " is not exists, you need to create them by yourself");
|
||||
});
|
||||
|
||||
logger.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(), s3Client.getRegionName());
|
||||
logger.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(),
|
||||
s3Client.getRegionName());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -15,10 +15,11 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
|
@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
|
|||
* common utils
|
||||
*/
|
||||
public class CommonUtils {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
|
||||
|
||||
private static final Base64 BASE64 = new Base64();
|
||||
|
|
@ -70,13 +72,6 @@ public class CommonUtils {
|
|||
return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return sudo enable
|
||||
*/
|
||||
public static boolean isSudoEnable() {
|
||||
return PropertyUtils.getBoolean(Constants.SUDO_ENABLE, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* if upload resource is HDFS and kerberos startup is true , else false
|
||||
*
|
||||
|
|
@ -85,7 +80,8 @@ public class CommonUtils {
|
|||
public static boolean getKerberosStartupState() {
|
||||
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
|
||||
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
|
||||
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
|
||||
Boolean kerberosStartupState =
|
||||
PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
|
||||
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +106,8 @@ public class CommonUtils {
|
|||
* @param loginUserKeytabPath loginUserKeytabPath
|
||||
* @throws IOException errors
|
||||
*/
|
||||
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException {
|
||||
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername,
|
||||
String loginUserKeytabPath) throws IOException {
|
||||
loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration());
|
||||
}
|
||||
|
||||
|
|
@ -124,13 +121,18 @@ public class CommonUtils {
|
|||
* @return load kerberos config return true
|
||||
* @throws IOException errors
|
||||
*/
|
||||
public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException {
|
||||
public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername,
|
||||
String loginUserKeytabPath, Configuration configuration) throws IOException {
|
||||
if (CommonUtils.getKerberosStartupState()) {
|
||||
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
|
||||
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf,
|
||||
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
|
||||
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
|
||||
UserGroupInformation.setConfiguration(configuration);
|
||||
UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)),
|
||||
StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)));
|
||||
UserGroupInformation.loginUserFromKeytab(
|
||||
StringUtils.defaultIfBlank(loginUserKeytabUsername,
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)),
|
||||
StringUtils.defaultIfBlank(loginUserKeytabPath,
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
|
@ -143,14 +145,15 @@ public class CommonUtils {
|
|||
if (StringUtils.isEmpty(password)) {
|
||||
return StringUtils.EMPTY;
|
||||
}
|
||||
//if encryption is not turned on, return directly
|
||||
// if encryption is not turned on, return directly
|
||||
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
|
||||
if (!encryptionEnable) {
|
||||
return password;
|
||||
}
|
||||
|
||||
// Using Base64 + salt to process password
|
||||
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
|
||||
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,
|
||||
Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
|
||||
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)));
|
||||
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
|
@ -163,14 +166,15 @@ public class CommonUtils {
|
|||
return StringUtils.EMPTY;
|
||||
}
|
||||
|
||||
//if encryption is not turned on, return directly
|
||||
// if encryption is not turned on, return directly
|
||||
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
|
||||
if (!encryptionEnable) {
|
||||
return password;
|
||||
}
|
||||
|
||||
// Using Base64 + salt to process password
|
||||
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
|
||||
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,
|
||||
Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
|
||||
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8);
|
||||
if (!passwordWithSalt.startsWith(salt)) {
|
||||
logger.warn("There is a password and salt mismatch: {} ", password);
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public final class Constants {
|
||||
|
||||
public static final int[] NOT_TERMINATED_STATES = new int[]{
|
||||
WorkflowExecutionStatus.SUBMITTED_SUCCESS.getCode(),
|
||||
TaskExecutionStatus.DISPATCH.getCode(),
|
||||
WorkflowExecutionStatus.RUNNING_EXECUTION.getCode(),
|
||||
WorkflowExecutionStatus.DELAY_EXECUTION.getCode(),
|
||||
WorkflowExecutionStatus.READY_PAUSE.getCode(),
|
||||
WorkflowExecutionStatus.READY_STOP.getCode(),
|
||||
TaskExecutionStatus.NEED_FAULT_TOLERANCE.getCode(),
|
||||
};
|
||||
|
||||
public static final int[] RUNNING_PROCESS_STATE = new int[]{
|
||||
TaskExecutionStatus.RUNNING_EXECUTION.getCode(),
|
||||
TaskExecutionStatus.SUBMITTED_SUCCESS.getCode(),
|
||||
TaskExecutionStatus.DISPATCH.getCode(),
|
||||
WorkflowExecutionStatus.SERIAL_WAIT.getCode()
|
||||
};
|
||||
}
|
||||
|
|
@ -15,13 +15,11 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.dao.utils;
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.TaskDependType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
|
|
@ -29,6 +27,9 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
|||
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
|
|
@ -38,10 +39,9 @@ import java.util.HashMap;
|
|||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -15,31 +15,23 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
/**
|
||||
* logger utils
|
||||
|
|
@ -62,7 +54,8 @@ public class LoggerUtils {
|
|||
// like TaskAppId=TASK-20211107-798_1-4084-15210
|
||||
String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, Constants.YYYYMMDD, null);
|
||||
return String.format("%s=%s-%s-%s_%s-%s-%s",
|
||||
TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr, processDefineCode, processDefineVersion, processInstId, taskId);
|
||||
TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr,
|
||||
processDefineCode, processDefineVersion, processInstId, taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -14,12 +14,31 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
package org.apache.dolphinscheduler.spi.enums;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* data base types
|
||||
*/
|
||||
public enum ResUploadType {
|
||||
HDFS, S3, OSS, NONE
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class ProcessData {
|
||||
@EqualsAndHashCode.Include
|
||||
private List<TaskNode> tasks;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private List<Property> globalParams;
|
||||
|
||||
private int timeout;
|
||||
|
||||
private int tenantId;
|
||||
|
||||
public ProcessData(List<TaskNode> tasks, List<Property> globalParams) {
|
||||
this.tasks = tasks;
|
||||
this.globalParams = globalParams;
|
||||
}
|
||||
}
|
||||
|
|
@ -18,15 +18,14 @@
|
|||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.OSUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.remote.utils.Host;
|
||||
import org.apache.dolphinscheduler.service.log.LogClient;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
|
@ -181,7 +180,7 @@ public class ProcessUtils {
|
|||
}
|
||||
}
|
||||
|
||||
if (CommonUtils.isSudoEnable() && !pidList.isEmpty()) {
|
||||
if (OSUtils.isSudoEnable() && !pidList.isEmpty()) {
|
||||
pidList = pidList.subList(1, pidList.size());
|
||||
}
|
||||
return String.join(" ", pidList).trim();
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.dolphinscheduler.service.log;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.NetUtils;
|
||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
|
|
@ -28,6 +27,7 @@ import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand
|
|||
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
|
||||
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
|
||||
import org.apache.dolphinscheduler.remote.utils.Host;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@
|
|||
package org.apache.dolphinscheduler.service.log;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
|
||||
import org.apache.dolphinscheduler.remote.command.Command;
|
||||
import org.apache.dolphinscheduler.remote.command.CommandType;
|
||||
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
|
||||
import org.apache.dolphinscheduler.service.utils.LoggerUtils;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
|
|||
import org.apache.dolphinscheduler.common.enums.UserType;
|
||||
import org.apache.dolphinscheduler.common.enums.WarningType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
|
|
@ -83,6 +82,7 @@ import org.apache.dolphinscheduler.service.cron.CronUtilsTest;
|
|||
import org.apache.dolphinscheduler.service.exceptions.CronParseException;
|
||||
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
|
||||
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.task.TaskPluginManager;
|
||||
import org.apache.dolphinscheduler.spi.params.base.FormType;
|
||||
|
||||
|
|
@ -432,7 +432,7 @@ public class ProcessServiceTest {
|
|||
command6.setCommandType(CommandType.RECOVER_SERIAL_WAIT);
|
||||
command6.setProcessDefinitionVersion(1);
|
||||
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1,
|
||||
Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists);
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists);
|
||||
Mockito.when(processInstanceMapper.updateNextProcessIdById(223, 222)).thenReturn(true);
|
||||
Mockito.when(commandMapper.deleteById(6)).thenReturn(1);
|
||||
ProcessInstance processInstance6 = processService.handleCommand(host, command6);
|
||||
|
|
@ -454,7 +454,7 @@ public class ProcessServiceTest {
|
|||
command7.setProcessDefinitionVersion(1);
|
||||
Mockito.when(commandMapper.deleteById(7)).thenReturn(1);
|
||||
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1,
|
||||
Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null);
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null);
|
||||
ProcessInstance processInstance8 = processService.handleCommand(host, command7);
|
||||
Assert.assertTrue(processInstance8 != null);
|
||||
|
||||
|
|
@ -477,7 +477,7 @@ public class ProcessServiceTest {
|
|||
command9.setProcessDefinitionVersion(1);
|
||||
Mockito.when(processInstanceMapper.queryDetailById(225)).thenReturn(processInstance9);
|
||||
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(12L, 1,
|
||||
Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists);
|
||||
org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists);
|
||||
Mockito.when(processInstanceMapper.updateById(processInstance)).thenReturn(1);
|
||||
Mockito.when(commandMapper.deleteById(9)).thenReturn(1);
|
||||
ProcessInstance processInstance10 = processService.handleCommand(host, command9);
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.storage.impl;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.HttpUtils;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import org.junit.Assert;
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.storage.impl;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
|
||||
import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S;
|
||||
|
|
@ -29,6 +29,7 @@ import static org.mockito.Mockito.spy;
|
|||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import org.apache.dolphinscheduler.service.storage.impl.OssOperator;
|
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
@ -15,7 +15,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
|
|
@ -15,23 +15,22 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.dao.utils;
|
||||
package org.apache.dolphinscheduler.service.utils;
|
||||
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.TaskDependType;
|
||||
import org.apache.dolphinscheduler.common.graph.DAG;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
|
||||
import org.apache.dolphinscheduler.common.process.ProcessDag;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessData;
|
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
|
||||
import org.apache.dolphinscheduler.service.model.TaskNode;
|
||||
import org.apache.dolphinscheduler.service.process.ProcessDag;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -20,10 +20,10 @@ package org.apache.dolphinscheduler.service.utils;
|
|||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.OSUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
|
||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue