natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add multiply settings for application.yml (#973) 

* update english documents

* refactor zk client

* update documents

* update zkclient

* update zkclient

* update documents

* add architecture-design

* change i18n

* update i18n

* update english documents

* add architecture-design

* update english documents

* update en-US documents

* add architecture-design

* update demo site

* add mybatis plus model

* modify mybatisplus

* modify mybatisplus

* change interface by mybatisplus

* add unit test

* refactor dao interface.

* add unit test for dao...

* add unit test for dao...

* add unit test for dao...

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update some dao bugs

* update for some bugs

* update some bugs

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update

* update

* add multiply settings for application.yml

* add multiply settings for application.yml

* revert
This commit is contained in:
bao liang 2019-10-09 18:47:42 +08:00 committed by qiaozhanwei
parent 25695ff21b
commit 6a48b98513
16 changed files with 106 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java
View File

@ -74,7 +74,7 @@ public class TenantService extends BaseService{
return result;
}
if (!checkTenant(tenantCode)){
if (checkTenantExists(tenantCode)){
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode);
return result;
}
@ -168,7 +168,7 @@ public class TenantService extends BaseService{
* if the tenant code is modified, the original resource needs to be copied to the new tenant.
*/
if (!tenant.getTenantCode().equals(tenantCode)){
if (checkTenant(tenantCode)){
if (checkTenantExists(tenantCode)){
// if hdfs startup
if (PropertyUtils.getResUploadStartupState()){
String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources";
@ -278,8 +278,7 @@ public class TenantService extends BaseService{
*/
public Result verifyTenantCode(String tenantCode) {
Result result=new Result();
Tenant tenant = tenantMapper.queryByTenantCode(tenantCode);
if (tenant != null) {
if (checkTenantExists(tenantCode)) {
logger.error("tenant {} has exist, can't create again.", tenantCode);
putMsg(result, Status.TENANT_NAME_EXIST);
}else{
@ -295,7 +294,8 @@ public class TenantService extends BaseService{
* @param tenantCode
* @return
*/
private boolean checkTenant(String tenantCode) {
return tenantMapper.queryByTenantCode(tenantCode) == null ? true : false;
private boolean checkTenantExists(String tenantCode) {
List<Tenant> tenants = tenantMapper.queryByTenantCode(tenantCode);
return (tenants != null && tenants.size() > 0);
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
View File

@ -132,6 +132,9 @@ public class DateUtils {
* @return
*/
public static long differSec(Date d1, Date d2) {
if(d1 == null || d2 == null){
return 0;
}
return (long) Math.ceil(differMs(d1, d2) / 1000.0);
}

6
dolphinscheduler-common/src/main/resources/quartz.properties
View File

@ -32,8 +32,8 @@ org.quartz.jobStore.dataSource = myDs
#============================================================================
org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider
org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.user = xx
org.quartz.dataSource.myDs.password = xx
org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.220.188:3306/dolphinscheduler_bl?characterEncoding=utf8
org.quartz.dataSource.myDs.user = root
org.quartz.dataSource.myDs.password = root@123
org.quartz.dataSource.myDs.maxConnections = 10
org.quartz.dataSource.myDs.validationQuery = select 1

26
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.config.YmlConfig;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
@ -41,25 +42,26 @@ public class MonitorDBDao {
/**
* 加载配置文件
*/
private static Configuration conf;
// private static Configuration conf;
// static {
// try {
// conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
// }catch (ConfigurationException e){
// logger.error("load configuration excetpion",e);
// System.exit(1);
// }
// }
static {
try {
conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
}catch (ConfigurationException e){
logger.error("load configuration excetpion",e);
System.exit(1);
}
}
/**
* create connection
* @return
*/
private static Connection getConn() {
String url = conf.getString(Constants.SPRING_DATASOURCE_URL);
String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
String url = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_URL);
String username = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_USERNAME);
String password = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_PASSWORD);
Connection conn = null;
try {
//classloader,load driver

20
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
View File

@ -146,17 +146,17 @@ public class ProcessDao extends AbstractBaseDao {
delCommandByid(command.getId());
saveErrorCommand(command, "process instance is null");
return null;
}else if(!checkThreadNum(command, validThreadNum)){
logger.info("there is not enough thread for this command: {}",command.toString() );
return setWaitingThreadProcess(command, processInstance);
}else{
processInstance.setCommandType(command.getCommandType());
processInstance.addHistoryCmd(command.getCommandType());
saveProcessInstance(processInstance);
this.setSubProcessParam(processInstance);
delCommandByid(command.getId());
return processInstance;
}
if(!checkThreadNum(command, validThreadNum)){
logger.info("there is not enough thread for this command: {}",command.toString() );
return setWaitingThreadProcess(command, processInstance);
}
processInstance.setCommandType(command.getCommandType());
processInstance.addHistoryCmd(command.getCommandType());
saveProcessInstance(processInstance);
this.setSubProcessParam(processInstance);
delCommandByid(command.getId());
return processInstance;
}catch (Exception e){
logger.error("scan command error ", e);
saveErrorCommand(command, e.toString());

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java
View File

@ -27,7 +27,7 @@ public interface TenantMapper extends BaseMapper<Tenant> {
Tenant queryById(@Param("tenantId") int tenantId);
Tenant queryByTenantCode(@Param("tenantCode") String tenantCode);
List<Tenant> queryByTenantCode(@Param("tenantCode") String tenantCode);
IPage<Tenant> queryTenantPaging(IPage<Tenant> page,
@Param("searchVal") String searchVal);

65
dolphinscheduler-dao/src/main/resources/application.yml
View File

@ -1,14 +1,38 @@
# mysql
spring:
datasource:
driver-class-name: org.postgresql.Driver
# driver-class-name: com.mysql.jdbc.Driver
url: jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
# url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
username: root
password: root@123
# platform:
continue-on-error: true
datasource:
url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
username: root
password: root@123
#
type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: com.mysql.jdbc.Driver
initialSize: 5
minIdle: 5
maxActive: 20
maxWait: 60000
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis: 300000
validationQuery: SELECT 1 FROM DUAL
testWhileIdle: true
testOnBorrow: false
testOnReturn: false
poolPreparedStatements: true
maxPoolPreparedStatementPerConnectionSize: 20
spring.datasource.filters: stat,wall,log4j
connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
# jpa:
# hibernate:
# ddl-auto : update
# datasource:
# platform:
# continue-on-error: true
# druid:
# driver-class-name: org.postgresql.Driver
# url: jdbc:postgresql://192.168.220.154:5432/escheduler
#mybatis
@ -42,3 +66,26 @@ logging:
level:
org.apache.dolphinscheduler.dao: debug
---
# master settings
spring:
profiles: master
server:
port: 5566
---
# worker settings
spring:
profiles: worker
server:
port: 7788
---
# alert settings
spring:
profiles: alert
server:
port: 7789

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml
View File

@ -5,7 +5,7 @@
select command.* from t_ds_command command
join t_ds_process_definition definition on command.process_definition_id = definition.id
where definition.release_state = 1 AND definition.flag = 1
order by update_time asc
order by command.update_time asc
limit 1
</select>
<select id="countCommandState" resultType="org.apache.dolphinscheduler.dao.entity.CommandCount">

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java
View File

@ -58,8 +58,8 @@ public class AccessTokenMapperTest {
//update
accessToken.setToken("hello, token");
int update = accessTokenMapper.updateById(accessToken);
Assert.assertEquals(update, 1);
accessTokenMapper.deleteById(accessToken.getId());
Assert.assertEquals(update, 1);
}
@Test

3
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
View File

@ -109,9 +109,10 @@ public class ProcessInstanceMapperTest {
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.SUCCESS.ordinal()};
processInstanceMapper.queryByHostAndStatus(processInstance.getHost(), stateArray);
List<ProcessInstance> processInstances = processInstanceMapper.queryByHostAndStatus(null, stateArray);
processInstanceMapper.deleteById(processInstance.getId());
Assert.assertNotEquals(processInstances.size(), 0);
}
@Test

1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
View File

@ -39,6 +39,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.utils.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
import java.io.IOException;

4
dolphinscheduler-server/src/main/resources/master.properties
View File

@ -15,7 +15,7 @@ master.task.commit.interval=100
# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
master.max.cpuload.avg=10
master.max.cpuload.avg=100
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
master.reserved.memory=1
master.reserved.memory=0.1

2
dolphinscheduler-ui/.env
View File

@ -1,5 +1,5 @@
# 后端接口地址
API_BASE = http://192.168.xx.xx:12345
API_BASE = http://127.0.0.1:12345
# 本地开发如需ip访问项目把"#"号去掉
#DEV_HOST = 192.168.xx.xx

2
dolphinscheduler-ui/build/webpack.config.dev.js
View File

@ -88,7 +88,7 @@ const config = merge.smart(baseConfig, {
historyApiFallback: true,
disableHostCheck: true,
proxy: {
'/escheduler': {
'/dolphinscheduler': {
target: getEnv('API_BASE', 'http://local.dev:8080/backend'),
changeOrigin: true
}

2
dolphinscheduler-ui/src/js/module/io/index.js
View File

@ -17,7 +17,7 @@
import io from '~/@fedor/io/dist/io'
const apiPrefix = '/escheduler'
const apiPrefix = '/dolphinscheduler'
const reSlashPrefix = /^\/+/
const resolveURL = (url) => {

6
script/dolphinscheduler-daemon.sh
View File

@ -44,13 +44,13 @@ if [ "$command" = "api-server" ]; then
LOG_FILE="-Dlogging.config=conf/apiserver_logback.xml"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dspring.config.location=conf/application_master.properties -Ddruid.mysql.usePingMethod=false"
LOG_FILE="-Dspring.profiles.active=master -Dlogging.config=conf/master_logback.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dspring.config.location=conf/application_worker.properties -Ddruid.mysql.usePingMethod=false"
LOG_FILE="-Dspring.profiles.active=worker -Dlogging.config=conf/worker_logback.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dspring.config.location=conf/application_alert.properties -Ddruid.mysql.usePingMethod=false"
LOG_FILE="-Dspring.profiles.active=alert -Dlogging.config=conf/alert_logback.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer