change escheduler to dolphinscheduler (#963)
* rename from DatasourceUserMapper to DataSourceUserMapper * add unit test in UserMapper and WorkerGroupMapper * change cn.escheduler to org.apache.dolphinscheduler * add unit test in UdfFuncMapperTest * add unit test in UdfFuncMapperTest * remove DatabaseConfiguration * add ConnectionFactoryTest * cal duration in processInstancesList * change desc to description * change table name in mysql ddl * change table name in mysql ddl * change escheduler to dolphinscheduler * change escheduler to dolphinscheduler * change escheduler to dolphinscheduler
This commit is contained in:
lgcareer
qiaozhanwei
parent
ed04835c95
commit
7e815b44ce
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> easyscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; }</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>
|
||||
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; }</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>
|
||||
|
|
@ -25,7 +25,7 @@ import com.alibaba.druid.support.http.StatViewServlet;
|
|||
// @WebInitParam(name="allow",value="127.0.0.1"),
|
||||
// @WebInitParam(name="deny",value="192.168.16.111"),
|
||||
@WebInitParam(name="loginUsername",value="admin"),
|
||||
@WebInitParam(name="loginPassword",value="escheduler123"),
|
||||
@WebInitParam(name="loginPassword",value="dolphinscheduler123"),
|
||||
@WebInitParam(name="resetEnable",value="true")
|
||||
}) */
|
||||
public class DruidStatViewServlet extends StatViewServlet {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
|
||||
# to the conf directory，support s3，for example : s3a://escheduler
|
||||
# to the conf directory，support s3，for example : s3a://dolphinscheduler
|
||||
fs.defaultFS=hdfs://mycluster:8020
|
||||
|
||||
# s3 need，s3 endpoint
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ public class HadoopUtilsTest {
|
|||
@Test
|
||||
public void readFileTest(){
|
||||
try {
|
||||
byte[] bytes = HadoopUtils.getInstance().catFile("/escheduler/hdfs/resources/35435.sh");
|
||||
byte[] bytes = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/35435.sh");
|
||||
logger.info("------------------start");
|
||||
logger.info(new String(bytes));
|
||||
logger.info("---------------------end");
|
||||
|
|
@ -84,7 +84,7 @@ public class HadoopUtilsTest {
|
|||
|
||||
@Test
|
||||
public void catFileTest()throws Exception{
|
||||
List<String> stringList = HadoopUtils.getInstance().catFile("/escheduler/hdfs/resources/WCSparkPython.py", 0, 1000);
|
||||
List<String> stringList = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000);
|
||||
logger.info(String.join(",",stringList));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
# base spring data source configuration
|
||||
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
|
||||
spring.datasource.username=root
|
||||
spring.datasource.password=root@123
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public interface CommandMapper extends BaseMapper<Command> {
|
|||
|
||||
|
||||
|
||||
@Select("select * from t_escheduler_command ${ew.customSqlSegment}")
|
||||
@Select("select * from t_ds_command ${ew.customSqlSegment}")
|
||||
List<Command> getAll(@Param(Constants.WRAPPER) Wrapper wrapper);
|
||||
|
||||
Command getOneToRun();
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ public class DolphinSchedulerManager {
|
|||
}
|
||||
|
||||
public void initDolphinScheduler() {
|
||||
// Determines whether the escheduler table structure has been init
|
||||
// Determines whether the dolphinscheduler table structure has been init
|
||||
if(upgradeDao.isExistsTable("t_escheduler_version") || upgradeDao.isExistsTable("t_escheduler_queue")) {
|
||||
logger.info("The database has been initialized. Skip the initialization step");
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -95,10 +95,10 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
|
||||
public void initSchema(String initSqlPath) {
|
||||
|
||||
// Execute the escheduler DDL, it cannot be rolled back
|
||||
// Execute the dolphinscheduler DDL, it cannot be rolled back
|
||||
runInitDDL(initSqlPath);
|
||||
|
||||
// Execute the escheduler DML, it can be rolled back
|
||||
// Execute the dolphinscheduler DML, it can be rolled back
|
||||
runInitDML(initSqlPath);
|
||||
|
||||
}
|
||||
|
|
@ -108,14 +108,12 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
if (StringUtils.isEmpty(rootDir)) {
|
||||
throw new RuntimeException("Environment variable user.dir not found");
|
||||
}
|
||||
//String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql";
|
||||
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql";
|
||||
try {
|
||||
conn = ConnectionFactory.getDataSource().getConnection();
|
||||
conn.setAutoCommit(false);
|
||||
// 执行escheduler_dml.sql脚本，导入escheduler相关的数据
|
||||
// Execute the ark_manager_dml.sql script to import the data related to escheduler
|
||||
|
||||
// Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
|
||||
ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
|
||||
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
|
||||
initScriptRunner.runScript(initSqlReader);
|
||||
|
|
@ -153,7 +151,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql";
|
||||
try {
|
||||
conn = ConnectionFactory.getDataSource().getConnection();
|
||||
// Execute the escheduler_ddl.sql script to create the table structure of escheduler
|
||||
// Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
|
||||
ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
|
||||
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
|
||||
initScriptRunner.runScript(initSqlReader);
|
||||
|
|
@ -228,16 +226,16 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
if (StringUtils.isEmpty(rootDir)) {
|
||||
throw new RuntimeException("Environment variable user.dir not found");
|
||||
}
|
||||
String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
|
||||
logger.info("mysqlSQLFilePath"+mysqlSQLFilePath);
|
||||
String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
|
||||
logger.info("sqlSQLFilePath"+sqlFilePath);
|
||||
Connection conn = null;
|
||||
PreparedStatement pstmt = null;
|
||||
try {
|
||||
conn = ConnectionFactory.getDataSource().getConnection();
|
||||
conn.setAutoCommit(false);
|
||||
// Execute the upgraded escheduler dml
|
||||
// Execute the upgraded dolphinscheduler dml
|
||||
ScriptRunner scriptRunner = new ScriptRunner(conn, false, true);
|
||||
Reader sqlReader = new FileReader(new File(mysqlSQLFilePath));
|
||||
Reader sqlReader = new FileReader(new File(sqlFilePath));
|
||||
scriptRunner.runScript(sqlReader);
|
||||
if (isExistsTable(T_VERSION_NAME)) {
|
||||
// Change version in the version table to the new version
|
||||
|
|
@ -295,7 +293,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
if (StringUtils.isEmpty(rootDir)) {
|
||||
throw new RuntimeException("Environment variable user.dir not found");
|
||||
}
|
||||
String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
|
||||
String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
|
||||
Connection conn = null;
|
||||
PreparedStatement pstmt = null;
|
||||
try {
|
||||
|
|
@ -303,9 +301,9 @@ public abstract class UpgradeDao extends AbstractBaseDao {
|
|||
String dbName = conn.getCatalog();
|
||||
logger.info(dbName);
|
||||
conn.setAutoCommit(true);
|
||||
// Execute the escheduler ddl.sql for the upgrade
|
||||
// Execute the dolphinscheduler ddl.sql for the upgrade
|
||||
ScriptRunner scriptRunner = new ScriptRunner(conn, true, true);
|
||||
Reader sqlReader = new FileReader(new File(mysqlSQLFilePath));
|
||||
Reader sqlReader = new FileReader(new File(sqlFilePath));
|
||||
scriptRunner.runScript(sqlReader);
|
||||
|
||||
} catch (FileNotFoundException e) {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ spring:
|
|||
datasource:
|
||||
driver-class-name: org.postgresql.Driver
|
||||
# driver-class-name: com.mysql.jdbc.Driver
|
||||
url: jdbc:postgresql://192.168.220.154:5432/escheduler
|
||||
# url: jdbc:mysql://192.168.220.188:3306/escheduler_new?useUnicode=true&characterEncoding=UTF-8
|
||||
url: jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
|
||||
# url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
|
||||
username: root
|
||||
password: root@123
|
||||
# platform:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# base spring data source configuration
|
||||
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/escheduler_new?characterEncoding=UTF-8
|
||||
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?characterEncoding=UTF-8
|
||||
spring.datasource.username=root
|
||||
spring.datasource.password=root@123
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# base spring data source configuration
|
||||
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
|
||||
spring.datasource.username=root
|
||||
spring.datasource.password=root@123
|
||||
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class MasterServer extends AbstractServer {
|
|||
private ScheduledExecutorService heartbeatMasterService;
|
||||
|
||||
/**
|
||||
* escheduler database interface
|
||||
* dolphinscheduler database interface
|
||||
*/
|
||||
@Autowired
|
||||
protected ProcessDao processDao;
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ public class MasterSchedulerThread implements Runnable {
|
|||
private final ExecutorService masterExecService;
|
||||
|
||||
/**
|
||||
* escheduler database interface
|
||||
* dolphinscheduler database interface
|
||||
*/
|
||||
private final ProcessDao processDao;
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ public class MasterSchedulerThread implements Runnable {
|
|||
if(OSUtils.checkResource(conf, true)){
|
||||
if (zkMasterClient.getZkClient().getState() == CuratorFrameworkState.STARTED) {
|
||||
|
||||
// create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master
|
||||
// create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master
|
||||
String znodeLock = zkMasterClient.getMasterLockPath();
|
||||
|
||||
mutex = new InterProcessMutex(zkMasterClient.getZkClient(), znodeLock);
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
|
|||
* for example :
|
||||
* your PYTHON_HOM is /opt/python3.7/
|
||||
* you must set PYTHON_HOME is /opt/python3.7/python under nder common.properties
|
||||
* escheduler.env.path file.
|
||||
* dolphinscheduler.env.path file.
|
||||
*
|
||||
* @param envPath
|
||||
* @return
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ public class ZKMasterClient extends AbstractZKClient {
|
|||
|
||||
InterProcessMutex mutex = null;
|
||||
try {
|
||||
// create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master
|
||||
// create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master
|
||||
String znodeLock = getMasterStartUpLockPath();
|
||||
mutex = new InterProcessMutex(zkClient, znodeLock);
|
||||
mutex.acquire();
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ public class ShellCommandExecutorTest {
|
|||
|
||||
TaskProps taskProps = new TaskProps();
|
||||
// processDefineId_processInstanceId_taskInstanceId
|
||||
taskProps.setTaskDir("/opt/soft/program/tmp/escheduler/exec/flow/5/36/2864/7657");
|
||||
taskProps.setTaskDir("/opt/soft/program/tmp/dolphinscheduler/exec/flow/5/36/2864/7657");
|
||||
taskProps.setTaskAppId("36_2864_7657");
|
||||
// set tenant -> task execute linux user
|
||||
taskProps.setTenantCode("hdfs");
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# base spring data source configuration
|
||||
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
|
||||
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
|
||||
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
|
||||
spring.datasource.username=root
|
||||
spring.datasource.password=root@123
|
||||
|
||||
|
|
|
|||
|
|
@ -14,13 +14,13 @@ or
|
|||
conda install -c conda-forge kazoo
|
||||
|
||||
run script and parameter description：
|
||||
nohup python -u monitor_server.py /data1_1T/escheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /escheduler/masters /escheduler/workers> monitor_server.log 2>&1 &
|
||||
nohup python -u monitor_server.py /data1_1T/dolphinscheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /dolphinscheduler/masters /dolphinscheduler/workers> monitor_server.log 2>&1 &
|
||||
the parameters are as follows:
|
||||
/data1_1T/escheduler : the value comes from the installPath in install.sh
|
||||
/data1_1T/dolphinscheduler : the value comes from the installPath in install.sh
|
||||
192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 : the value comes from zkQuorum in install.sh
|
||||
the value comes from zkWorkers in install.sh
|
||||
/escheduler/masters : the value comes from zkMasters in install.sh
|
||||
/escheduler/workers : the value comes from zkWorkers in install.sh
|
||||
/dolphinscheduler/masters : the value comes from zkMasters in install.sh
|
||||
/dolphinscheduler/workers : the value comes from zkWorkers in install.sh
|
||||
'''
|
||||
import sys
|
||||
import socket
|
||||
|
|
@ -73,7 +73,7 @@ class ZkClient:
|
|||
if (len(restart_master_list) != 0):
|
||||
for master in restart_master_list:
|
||||
print("master " + self.get_ip_by_hostname(master) + " server has down")
|
||||
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start master-server')
|
||||
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
|
||||
|
||||
if (self.zk.exists(workers_zk_path)):
|
||||
zk_worker_list = []
|
||||
|
|
@ -84,7 +84,7 @@ class ZkClient:
|
|||
if (len(restart_worker_list) != 0):
|
||||
for worker in restart_worker_list:
|
||||
print("worker " + self.get_ip_by_hostname(worker) + " server has down")
|
||||
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start worker-server')
|
||||
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
|
||||
|
||||
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
schedule.enter(inc, 0, self.restart_server, (inc,))
|
||||
|
|
|
|||
Loading…
Reference in New Issue