add master and worker properties and remove data quality config (#2211)

* update logback

* update log

* refactor worker registry (#2107)

* Refactor worker (#2115)

* refactor worker registry

* refactor master server

* Modify workgroupid parameter name (#2105)

* Delete worker group management page

* Modify workgroupid parameter name

* Refactor worker (#2121)

* refactor worker registry

* refactor master server

* refactor MasterSchedulerService

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath (#2126)

* 1, master persistent task
2. extract  master and worker communication model

* 1, master persistent task
2. extract  master and worker communication model

* 1, master persistent task
2. extract  master and worker communication model

* add license

* modify javadoc error

* TaskExecutionContext create modify

* buildAckCommand taskInstanceId not set modify

* java doc error modify

* add comment

* ExecutorManager interface add generic type

* add TaskInstanceCacheManager receive Worker report result

* TaskInstance setExecutePath

* add TaskInstanceCacheManager to receive Worker Task result report

* TaskInstanceCacheManager add remove method

* add license

* add dispatcht task method

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* taskInstanceCache is null ,need load from db

* taskInstanceCache is null ,need load from db

* taskInstanceCache is null ,need load from db

* 1，worker TaskPros use TaskExecutionContext replase
2，Master kill Task , KillTaskProcessor modify

* worker remove db

* ShellTask modify

* master persistence processId and appIds

* master persistence processId and appIds

* master add kill task logic

* master add kill task logic

* master add kill task logic

* javadoc error modify

* remove chinese log

* executeDirectly method add Override

* remote module modify

* TaskKillResponseProcessor command type modify

* create buildKillCommand

* host add host:port format

* host add host:port format

* TaskAckProcessor modify

* TaskAckProcessor modify

* task prioriry refator

* remove ITaskQueue

* task prioriry refator

* remove ITaskQueue

* TaskPriority refactor

* remove logs

* WorkerServer refactor

* MasterSchedulerService modify

* WorkerConfig listen port modify

* modify master and worker listen port

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath

Co-authored-by: qiaozhanwei <qiaozhanwei@analysys.com.cn>

* not exist in openjdk,just delete

* add master and worker properties

* add master and worker properties

* add master and worker properties

Co-authored-by: Tboy <guo.jiwei@immomo.com>
Co-authored-by: break60 <790061044@qq.com>
Co-authored-by: qiaozhanwei <qiaozhanwei@outlook.com>
Co-authored-by: qiaozhanwei <qiaozhanwei@analysys.com.cn>

.gitignore

@@ -148,3 +148,4 @@ dolphinscheduler-ui/dist/lib/external/
 dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
 /dolphinscheduler-dao/src/main/resources/dao/data_source.properties
 
+!/zookeeper_data/

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java

@@ -61,7 +61,7 @@ public class TaskRecordDao {
      * @return whether startup taskrecord
      */
     public static boolean getTaskRecordFlag(){
-       return conf.getBoolean(Constants.TASK_RECORD_FLAG);
+       return conf.getBoolean(Constants.TASK_RECORD_FLAG,false);
     }
     /**
      * create connection

dolphinscheduler-dao/src/main/resources/application.properties

@@ -71,12 +71,3 @@ spring.datasource.spring.datasource.filters=stat,wall,log4j
 spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
 
 
-# data quality analysis is not currently in use. please ignore the following configuration
-# task record
-task.record.flag=false
-task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
-task.record.datasource.username=xx
-task.record.datasource.password=xx
-
-# Logger Config
-#logging.level.org.apache.dolphinscheduler.dao=debug

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java

@@ -16,8 +16,6 @@
  */
 package org.apache.dolphinscheduler.remote.command;
 
-import com.sun.org.apache.regexp.internal.RE;
-
 import java.io.Serializable;
 import java.util.concurrent.atomic.AtomicLong;
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java

@@ -16,10 +16,13 @@
  */
 package org.apache.dolphinscheduler.server.master.config;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
 import org.springframework.stereotype.Component;
 
 @Component
+@PropertySource(value = "master.properties")
 public class MasterConfig {
 
     @Value("${master.exec.threads:100}")
@@ -37,10 +40,10 @@ public class MasterConfig {
     @Value("${master.task.commit.interval:1000}")
     private int masterTaskCommitInterval;
 
-    @Value("${master.max.cpuload.avg:100}")
+    @Value("${master.max.cpuload.avg:-1}")
     private double masterMaxCpuloadAvg;
 
-    @Value("${master.reserved.memory:0.1}")
+    @Value("${master.reserved.memory:0.3}")
     private double masterReservedMemory;
 
     @Value("${master.host.selector:lowerWeight}")
@@ -106,6 +109,9 @@ public class MasterConfig {
     }
 
     public double getMasterMaxCpuloadAvg() {
+        if (masterMaxCpuloadAvg == -1){
+            return Constants.DEFAULT_MASTER_CPU_LOAD;
+        }
         return masterMaxCpuloadAvg;
     }
 
@@ -120,4 +126,4 @@ public class MasterConfig {
     public void setMasterReservedMemory(double masterReservedMemory) {
         this.masterReservedMemory = masterReservedMemory;
     }
-}
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java

@@ -1,3 +1,4 @@
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -16,25 +17,28 @@
  */
 package org.apache.dolphinscheduler.server.worker.config;
 
+import org.apache.dolphinscheduler.common.Constants;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
 import org.springframework.stereotype.Component;
 
 @Component
+@PropertySource(value = "worker.properties")
 public class WorkerConfig {
 
-    @Value("${worker.exec.threads: 100}")
+    @Value("${worker.exec.threads:100}")
     private int workerExecThreads;
 
-    @Value("${worker.heartbeat.interval: 10}")
+    @Value("${worker.heartbeat.interval:10}")
     private int workerHeartbeatInterval;
 
-    @Value("${worker.fetch.task.num: 3}")
+    @Value("${worker.fetch.task.num:3}")
     private int workerFetchTaskNum;
 
-    @Value("${worker.max.cpuload.avg:100}")
+    @Value("${worker.max.cpuload.avg:-1}")
     private int workerMaxCpuloadAvg;
 
-    @Value("${worker.reserved.memory:0.1}")
+    @Value("${worker.reserved.memory:0.5}")
     private double workerReservedMemory;
 
     @Value("${worker.group: default}")
@@ -92,10 +96,13 @@ public class WorkerConfig {
     }
 
     public int getWorkerMaxCpuloadAvg() {
+        if (workerMaxCpuloadAvg == -1){
+            return Constants.DEFAULT_WORKER_CPU_LOAD;
+        }
         return workerMaxCpuloadAvg;
     }
 
     public void setWorkerMaxCpuloadAvg(int workerMaxCpuloadAvg) {
         this.workerMaxCpuloadAvg = workerMaxCpuloadAvg;
     }
-}
+}

dolphinscheduler-server/src/main/resources/master.properties

@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# master execute thread num
+#master.exec.threads=100
+
+# master execute task number in parallel
+#master.exec.task.num=20
+
+# master heartbeat interval
+#master.heartbeat.interval=10
+
+# master commit task retry times
+#master.task.commit.retryTimes=5
+
+# master commit task interval
+#master.task.commit.interval=1000
+
+
+# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
+#master.max.cpuload.avg=100
+
+# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
+#master.reserved.memory=0.3
+
+# master listen port
+#master.listen.port=5678

dolphinscheduler-server/src/main/resources/worker.properties

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# worker execute thread num
+#worker.exec.threads=100
+
+# worker heartbeat interval
+#worker.heartbeat.interval=10
+
+# submit the number of tasks at a time
+#worker.fetch.task.num = 3
+
+# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
+#worker.max.cpuload.avg=100
+
+# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
+#worker.reserved.memory=0.3
+
+# worker listener port
+#worker.listen.port: 1234