natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add config to override default ssh port for install scripts. #780 (#1388)

This commit is contained in:
elonlo 2019-12-04 19:03:05 +08:00 committed by bao liang
parent bf1fec74de
commit e6395d0eef
7 changed files with 42 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
install.sh
View File

@ -82,6 +82,10 @@ zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ark0,ark1,ark2,ark3,ark4"
# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort=22
# conf/config/run_config.conf config
# run master machine
# Note: list of hosts hostname for deploying master
@ -426,12 +430,14 @@ sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseW
sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf
sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf
sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf
sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf
sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf
sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf
sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/run_config.conf
# 2,create directory
@ -447,23 +453,23 @@ for host in ${hostsArr[@]}
do
# create if programPath does not exist
if ! ssh $host test -e $programPath; then
ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath"
if ! ssh -p $sshPort $host test -e $programPath; then
ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath"
fi
# create if downloadPath does not exist
if ! ssh $host test -e $downloadPath; then
ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath"
if ! ssh -p $sshPort $host test -e $downloadPath; then
ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath"
fi
# create if execPath does not exist
if ! ssh $host test -e $execPath; then
ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath"
if ! ssh -p $sshPort $host test -e $execPath; then
ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath"
fi
# create if xlsFilePath does not exist
if ! ssh $host test -e $xlsFilePath; then
ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath"
if ! ssh -p $sshPort $host test -e $xlsFilePath; then
ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath"
fi
done

1
script/config/install_config.conf
View File

@ -18,3 +18,4 @@
installPath=/data1_1T/dolphinscheduler
deployUser=dolphinscheduler
ips=ark0,ark1,ark2,ark3,ark4
sshPort=22

3
script/config/run_config.conf
View File

@ -18,4 +18,5 @@
masters=ark0,ark1
workers=ark2,ark3,ark4
alertServer=ark3
apiServers=ark1
apiServers=ark1
sshPort=22

9
script/monitor-server.py
View File

@ -77,9 +77,12 @@ class ZkClient:
master_list = list(map(lambda item : self.get_ip_by_hostname(item),master_list))
worker_list = config_dict.get('workers').split(',')
print worker_list
print worker_list
worker_list = list(map(lambda item: self.get_ip_by_hostname(item), worker_list))
ssh_port = config_dict.get("sshPort")
print ssh_port
if (self.zk.exists(masters_zk_path)):
zk_master_list = []
zk_master_nodes = self.zk.get_children(masters_zk_path)
@ -89,7 +92,7 @@ class ZkClient:
if (len(restart_master_list) != 0):
for master in restart_master_list:
print("master " + self.get_ip_by_hostname(master) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
if (self.zk.exists(workers_zk_path)):
zk_worker_list = []
@ -100,7 +103,7 @@ class ZkClient:
if (len(restart_worker_list) != 0):
for worker in restart_worker_list:
print("worker " + self.get_ip_by_hostname(worker) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
schedule.enter(inc, 0, self.restart_server, (inc,))

18
script/scp-hosts.sh
View File

@ -25,15 +25,15 @@ hostsArr=(${ips//,/ })
for host in ${hostsArr[@]}
do
if ! ssh $host test -e $installPath; then
ssh $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath"
if ! ssh -p $sshPort $host test -e $installPath; then
ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath"
fi
ssh $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/"
scp -r $workDir/../bin $host:$installPath
scp -r $workDir/../conf $host:$installPath
scp -r $workDir/../lib $host:$installPath
scp -r $workDir/../script $host:$installPath
scp -r $workDir/../sql $host:$installPath
scp $workDir/../install.sh $host:$installPath
ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/"
scp -P $sshPort -r $workDir/../bin $host:$installPath
scp -P $sshPort -r $workDir/../conf $host:$installPath
scp -P $sshPort -r $workDir/../lib $host:$installPath
scp -P $sshPort -r $workDir/../script $host:$installPath
scp -P $sshPort -r $workDir/../sql $host:$installPath
scp -P $sshPort $workDir/../install.sh $host:$installPath
done

10
script/start-all.sh
View File

@ -25,7 +25,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do
echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;"
ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;"
done
@ -34,17 +34,17 @@ for worker in ${workersHost[@]}
do
echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;"
done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;"
ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;"
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done

10
script/stop-all.sh
View File

@ -26,7 +26,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do
echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;"
ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;"
done
@ -35,17 +35,17 @@ for worker in ${workersHost[@]}
do
echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;"
done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;"
ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;"
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done