diff --git a/NOTICE b/NOTICE index b07c3dd4ca..458d64af0e 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache DolphinScheduler -Copyright 2019-2021 The Apache Software Foundation +Copyright 2019-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 040459d6b6..bc111cc1d5 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Its main objectives are as follows: ## What's in DolphinScheduler Stability | Accessibility | Features | Scalability | - -- | -- | -- | -- + --------- | ------------- | -------- | ------------| Decentralized multi-master and multi-worker | Visualization of workflow key information, such as task status, task type, retry times, task operation machine information, visual variables, and so on at a glance.  |  Support pause, recover operation | Support customized task types support HA | Visualization of all workflow operations, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, provide API mode operations. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler supports distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic adjustment. Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | | @@ -59,11 +59,11 @@ Overload processing: By using the task queue mechanism, the number of schedulabl ## QuickStart in Docker -Please referer the official website document: [QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html) +Please refer the official website document: [QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/docker.html) ## QuickStart in Kubernetes -Please referer the official website document: [QuickStart in Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html) +Please refer to the official website document: [QuickStart in Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/kubernetes.html) ## How to Build @@ -95,8 +95,8 @@ You are very welcome to communicate with the developers and users of Dolphin Sch 2. Follow the [Twitter account of DolphinScheduler](https://twitter.com/dolphinschedule) and get the latest news on time. ### Contributor over time - -[![Contributor over time](https://contributor-graph-api.apiseven.com/contributors-svg?chart=contributorOverTime&repo=apache/dolphinscheduler)](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/dolphinscheduler) + +[![Contributor over time](https://contributor-graph-api.apiseven.com/contributors-svg?chart=contributorOverTime&repo=apache/dolphinscheduler)](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/dolphinscheduler) ## How to Contribute diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index fb9c5038a3..904af241dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -951,6 +951,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } processDefinition.setLocations(newArrayNode.toString()); } + processDefinition.setCreateTime(new Date()); + processDefinition.setUpdateTime(new Date()); Map createDagResult = createDagDefine(loginUser, taskRelationLogList, processDefinition, Lists.newArrayList()); if (Status.SUCCESS.equals(createDagResult.get(Constants.STATUS))) { putMsg(createDagResult, Status.SUCCESS); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java index fa711ffe2c..7475bf738e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java @@ -28,219 +28,208 @@ import com.fasterxml.jackson.annotation.JsonFormat; @TableName("t_ds_resources") public class Resource { - /** - * id - */ - @TableId(value="id", type=IdType.AUTO) - private int id; + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private int id; - /** - * parent id - */ - private int pid; + /** + * parent id + */ + private int pid; - /** - * resource alias - */ - private String alias; + /** + * resource alias + */ + private String alias; - /** - * full name - */ - private String fullName; + /** + * full name + */ + private String fullName; - /** - * is directory - */ - private boolean isDirectory=false; + /** + * is directory + */ + private boolean isDirectory = false; - /** - * description - */ - private String description; + /** + * description + */ + private String description; - /** - * file alias - */ - private String fileName; + /** + * file alias + */ + private String fileName; - /** - * user id - */ - private int userId; + /** + * user id + */ + private int userId; - /** - * resource type - */ - private ResourceType type; + /** + * resource type + */ + private ResourceType type; - /** - * resource size - */ - private long size; + /** + * resource size + */ + private long size; - /** - * create time - */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") - private Date createTime; + /** + * create time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTime; - /** - * update time - */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") - private Date updateTime; + /** + * update time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date updateTime; - public Resource() { - } + public Resource() { + } - public Resource(int id, String alias, String fileName, String description, int userId, - ResourceType type, long size, - Date createTime, Date updateTime) { - this.id = id; - this.alias = alias; - this.fileName = fileName; - this.description = description; - this.userId = userId; - this.type = type; - this.size = size; - this.createTime = createTime; - this.updateTime = updateTime; - } + public Resource(int id, String alias, String fileName, String description, int userId, + ResourceType type, long size, + Date createTime, Date updateTime) { + this.id = id; + this.alias = alias; + this.fileName = fileName; + this.description = description; + this.userId = userId; + this.type = type; + this.size = size; + this.createTime = createTime; + this.updateTime = updateTime; + } - public Resource(int id, int pid, String alias, String fullName, boolean isDirectory) { - this.id = id; - this.pid = pid; - this.alias = alias; - this.fullName = fullName; - this.isDirectory = isDirectory; - } + public Resource(int id, int pid, String alias, String fullName, boolean isDirectory) { + this.id = id; + this.pid = pid; + this.alias = alias; + this.fullName = fullName; + this.isDirectory = isDirectory; + } - /*public Resource(String alias, String fileName, String description, int userId, ResourceType type, long size, Date createTime, Date updateTime) { - this.alias = alias; - this.fileName = fileName; - this.description = description; - this.userId = userId; - this.type = type; - this.size = size; - this.createTime = createTime; - this.updateTime = updateTime; - }*/ + public Resource(int pid, String alias, String fullName, boolean isDirectory, String description, String fileName, int userId, ResourceType type, long size, Date createTime, Date updateTime) { + this.pid = pid; + this.alias = alias; + this.fullName = fullName; + this.isDirectory = isDirectory; + this.description = description; + this.fileName = fileName; + this.userId = userId; + this.type = type; + this.size = size; + this.createTime = createTime; + this.updateTime = updateTime; + } - public Resource(int pid, String alias, String fullName, boolean isDirectory, String description, String fileName, int userId, ResourceType type, long size, Date createTime, Date updateTime) { - this.pid = pid; - this.alias = alias; - this.fullName = fullName; - this.isDirectory = isDirectory; - this.description = description; - this.fileName = fileName; - this.userId = userId; - this.type = type; - this.size = size; - this.createTime = createTime; - this.updateTime = updateTime; - } + public int getId() { + return id; + } - public int getId() { - return id; - } + public void setId(int id) { + this.id = id; + } - public void setId(int id) { - this.id = id; - } + public String getAlias() { + return alias; + } - public String getAlias() { - return alias; - } + public void setAlias(String alias) { + this.alias = alias; + } - public void setAlias(String alias) { - this.alias = alias; - } + public int getPid() { + return pid; + } - public int getPid() { - return pid; - } + public void setPid(int pid) { + this.pid = pid; + } - public void setPid(int pid) { - this.pid = pid; - } + public String getFullName() { + return fullName; + } - public String getFullName() { - return fullName; - } + public void setFullName(String fullName) { + this.fullName = fullName; + } - public void setFullName(String fullName) { - this.fullName = fullName; - } + public boolean isDirectory() { + return isDirectory; + } - public boolean isDirectory() { - return isDirectory; - } + public void setDirectory(boolean directory) { + isDirectory = directory; + } - public void setDirectory(boolean directory) { - isDirectory = directory; - } + public String getFileName() { + return fileName; + } - public String getFileName() { - return fileName; - } + public void setFileName(String fileName) { + this.fileName = fileName; + } - public void setFileName(String fileName) { - this.fileName = fileName; - } + public String getDescription() { + return description; + } - public String getDescription() { - return description; - } + public void setDescription(String description) { + this.description = description; + } - public void setDescription(String description) { - this.description = description; - } + public int getUserId() { + return userId; + } - public int getUserId() { - return userId; - } - - public void setUserId(int userId) { - this.userId = userId; - } + public void setUserId(int userId) { + this.userId = userId; + } - public ResourceType getType() { - return type; - } + public ResourceType getType() { + return type; + } - public void setType(ResourceType type) { - this.type = type; - } + public void setType(ResourceType type) { + this.type = type; + } - public long getSize() { - return size; - } + public long getSize() { + return size; + } - public void setSize(long size) { - this.size = size; - } + public void setSize(long size) { + this.size = size; + } - public Date getCreateTime() { - return createTime; - } + public Date getCreateTime() { + return createTime; + } - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } - public Date getUpdateTime() { - return updateTime; - } + public Date getUpdateTime() { + return updateTime; + } - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } - @Override - public String toString() { - return "Resource{" + + @Override + public String toString() { + return "Resource{" + "id=" + id + ", pid=" + pid + ", alias='" + alias + '\'' + @@ -254,30 +243,30 @@ public class Resource { ", createTime=" + createTime + ", updateTime=" + updateTime + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; } - Resource resource = (Resource) o; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Resource resource = (Resource) o; + + if (id != resource.id) { + return false; + } + return alias.equals(resource.alias); - if (id != resource.id) { - return false; } - return alias.equals(resource.alias); - } - - @Override - public int hashCode() { - int result = id; - result = 31 * result + alias.hashCode(); - return result; - } + @Override + public int hashCode() { + int result = id; + result = 31 * result + alias.hashCode(); + return result; + } } diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index d9d49dd935..bd84c8c724 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -640,7 +640,7 @@ CREATE TABLE t_ds_resources ( update_time timestamp DEFAULT NULL , pid int, full_name varchar(64), - is_directory int, + is_directory boolean DEFAULT FALSE, PRIMARY KEY (id), CONSTRAINT t_ds_resources_un UNIQUE (full_name, type) ) ; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java index 20a4da6344..a3b72b32d2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessor.java @@ -65,14 +65,16 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; String address; + String jdbcUrl; if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { address = String.format("%s%s:%s", Constants.JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); + jdbcUrl = address + ":" + oracleParam.getDatabase(); } else { address = String.format("%s%s:%s", Constants.JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); + jdbcUrl = address + "/" + oracleParam.getDatabase(); } - String jdbcUrl = address + "/" + oracleParam.getDatabase(); OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); oracleConnectionParam.setUser(oracleParam.getUserName()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java index de88708810..36b5e35dfb 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDatasourceProcessorTest.java @@ -61,13 +61,13 @@ public class OracleDatasourceProcessorTest { .createConnectionParams(oracleDatasourceParamDTO); Assert.assertNotNull(connectionParams); Assert.assertEquals("jdbc:oracle:thin:@localhost:3308", connectionParams.getAddress()); - Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default", connectionParams.getJdbcUrl()); + Assert.assertEquals("jdbc:oracle:thin:@localhost:3308:default", connectionParams.getJdbcUrl()); } @Test public void testCreateConnectionParams2() { String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:oracle:thin:@localhost:3308\"" - + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:oracle:thin:@localhost:3308/default\",\"connectType\":\"ORACLE_SID\"}"; + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:oracle:thin:@localhost:3308:default\",\"connectType\":\"ORACLE_SID\"}"; OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor .createConnectionParams(connectionJson); Assert.assertNotNull(connectionParams); @@ -82,9 +82,9 @@ public class OracleDatasourceProcessorTest { @Test public void testGetJdbcUrl() { OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); - oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308/default"); + oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308:default"); oracleConnectionParam.setOther("other=other"); - Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default?other=other", + Assert.assertEquals("jdbc:oracle:thin:@localhost:3308:default?other=other", oracleDatasourceProcessor.getJdbcUrl(oracleConnectionParam)); } @@ -97,4 +97,4 @@ public class OracleDatasourceProcessorTest { public void testGetValidationQuery() { Assert.assertEquals(Constants.ORACLE_VALIDATION_QUERY, oracleDatasourceProcessor.getValidationQuery()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index 6981f24972..3406a3e37d 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -1,5 +1,5 @@ Apache DolphinScheduler -Copyright 2019-2021 The Apache Software Foundation +Copyright 2019-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java index a1fe51274a..f08cb8208d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java @@ -920,33 +920,50 @@ public class WorkflowExecuteThread implements Runnable { return DependResult.SUCCESS; } TaskNode taskNode = dag.getNode(taskCode); - List depCodeList = taskNode.getDepList(); - for (String depsNode : depCodeList) { - if (!dag.containsNode(depsNode) - || forbiddenTaskList.containsKey(depsNode) - || skipTaskNodeList.containsKey(depsNode)) { - continue; - } - // dependencies must be fully completed - if (!completeTaskList.containsKey(depsNode)) { - return DependResult.WAITING; - } - ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); - if (depTaskState.typeIsPause() || depTaskState.typeIsCancel()) { - return DependResult.NON_EXEC; - } - // ignore task state if current task is condition - if (taskNode.isConditionsTask()) { - continue; - } - if (!dependTaskSuccess(depsNode, taskCode)) { - return DependResult.FAILED; + List indirectDepCodeList = new ArrayList<>(); + setIndirectDepList(taskCode, indirectDepCodeList); + for (String depsNode : indirectDepCodeList) { + if (dag.containsNode(depsNode) && !skipTaskNodeList.containsKey(depsNode)) { + // dependencies must be fully completed + if (!completeTaskList.containsKey(depsNode)) { + return DependResult.WAITING; + } + ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); + if (depTaskState.typeIsPause() || depTaskState.typeIsCancel()) { + return DependResult.NON_EXEC; + } + // ignore task state if current task is condition + if (taskNode.isConditionsTask()) { + continue; + } + if (!dependTaskSuccess(depsNode, taskCode)) { + return DependResult.FAILED; + } } } logger.info("taskCode: {} completeDependTaskList: {}", taskCode, Arrays.toString(completeTaskList.keySet().toArray())); return DependResult.SUCCESS; } + /** + * This function is specially used to handle the dependency situation where the parent node is a prohibited node. + * When the parent node is a forbidden node, the dependency relationship should continue to be traced + * + * @param taskCode taskCode + * @param indirectDepCodeList All indirectly dependent nodes + */ + private void setIndirectDepList(String taskCode, List indirectDepCodeList) { + TaskNode taskNode = dag.getNode(taskCode); + List depCodeList = taskNode.getDepList(); + for (String depsNode : depCodeList) { + if (forbiddenTaskList.containsKey(depsNode)) { + setIndirectDepList(depsNode, indirectDepCodeList); + } else { + indirectDepCodeList.add(depsNode); + } + } + } + /** * depend node is completed, but here need check the condition task branch is the next node */