* Fix install.sh for DS 2.0 * Use #* make it work in macos * Add comment for properties change * Add JAVA HOME for only one file change to deploy * Reduce config change by using default value of: plugin registry, alert, task as * Change javaHome default value * Delete config param zkQuorum * Reorder param * Add comment for install_config.conf
This commit is contained in:
Jiajie Zhong
GitHub
parent
0659aa3f46
commit
5f19d5be41
|
|
@ -15,55 +15,110 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips="ds1,ds2,ds3,ds4,ds5"
|
||||
|
||||
# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
|
||||
# postgresql or mysql
|
||||
dbtype="mysql"
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort="22"
|
||||
|
||||
# db config
|
||||
# db address and port
|
||||
dbhost="192.168.xx.xx:3306"
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2"
|
||||
masters="ds1,ds2"
|
||||
|
||||
# db username
|
||||
username="xx"
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
|
||||
|
||||
# db password
|
||||
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
|
||||
password="xx"
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3"
|
||||
alertServer="ds3"
|
||||
|
||||
# database name
|
||||
dbname="dolphinscheduler"
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1"
|
||||
apiServers="ds1"
|
||||
|
||||
|
||||
# zk cluster
|
||||
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
|
||||
|
||||
# zk root directory
|
||||
zkRoot="/dolphinscheduler"
|
||||
|
||||
# registry config
|
||||
# registry plugin dir
|
||||
# Note: find and load the Registry Plugin Jar from this dir.
|
||||
registryPluginDir="/data1_1T/dolphinscheduler/lib/plugin/registry"
|
||||
registryPluginName="zookeeper"
|
||||
registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
|
||||
|
||||
|
||||
# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists.
|
||||
# **DO NOT** set this configuration same as the current path (pwd)
|
||||
installPath="/data1_1T/dolphinscheduler"
|
||||
|
||||
# deployment user
|
||||
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser="dolphinscheduler"
|
||||
|
||||
# alert config
|
||||
# alert plugin dir
|
||||
# Note: find and load the Alert Plugin Jar from this dir.
|
||||
alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
|
||||
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
|
||||
dataBasedirPath="/tmp/dolphinscheduler"
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# DolphinScheduler ENV
|
||||
# ---------------------------------------------------------
|
||||
# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
|
||||
# and this configuration only support one parameter so far.
|
||||
javaHome="/your/java/home/here"
|
||||
|
||||
# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345
|
||||
apiServerPort="12345"
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Database
|
||||
# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
|
||||
# ---------------------------------------------------------
|
||||
# The type for the metadata database
|
||||
# Supported values: ``postgresql``, ``mysql``.
|
||||
dbtype="mysql"
|
||||
|
||||
# The <HOST>:<PORT> connection pair DolphinScheduler connect to the metadata database
|
||||
dbhost="192.168.xx.xx:3306"
|
||||
|
||||
# The username DolphinScheduler connect to the metadata database
|
||||
username="xx"
|
||||
|
||||
# The password DolphinScheduler connect to the metadata database
|
||||
password="xx"
|
||||
|
||||
# The database DolphinScheduler connect to the metadata database
|
||||
dbname="dolphinscheduler"
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Registry Server
|
||||
# ---------------------------------------------------------
|
||||
# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
|
||||
# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
|
||||
# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html
|
||||
registryPluginDir="lib/plugin/registry/zookeeper"
|
||||
|
||||
# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
|
||||
registryPluginName="zookeeper"
|
||||
|
||||
# Registry Server address.
|
||||
registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
|
||||
|
||||
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
|
||||
zkRoot="/dolphinscheduler"
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Alert Server
|
||||
# ---------------------------------------------------------
|
||||
# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir.
|
||||
alertPluginDir="lib/plugin/alert"
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Worker Task Server
|
||||
# ---------------------------------------------------------
|
||||
# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
|
||||
taskPluginDir="lib/plugin/task"
|
||||
|
||||
# resource storage type: HDFS, S3, NONE
|
||||
resourceStorageType="NONE"
|
||||
|
||||
|
|
@ -110,34 +165,3 @@ sudoEnable="true"
|
|||
|
||||
# worker tenant auto create
|
||||
workerTenantAutoCreate="false"
|
||||
|
||||
# worker task plugin dir
|
||||
taskPluginDir="/data1_1T/dolphinscheduler/lib/plugin/task"
|
||||
|
||||
# api server port
|
||||
apiServerPort="12345"
|
||||
|
||||
|
||||
# install hosts
|
||||
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
|
||||
ips="ds1,ds2,ds3,ds4,ds5"
|
||||
|
||||
# ssh port, default 22
|
||||
# Note: if ssh port is not default, modify here
|
||||
sshPort="22"
|
||||
|
||||
# run master machine
|
||||
# Note: list of hosts hostname for deploying master
|
||||
masters="ds1,ds2"
|
||||
|
||||
# run worker machine
|
||||
# note: need to write the worker group name of each worker, the default value is "default"
|
||||
workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
|
||||
|
||||
# run alert machine
|
||||
# note: list of machine hostnames for deploying alert server
|
||||
alertServer="ds3"
|
||||
|
||||
# run api machine
|
||||
# note: list of machine hostnames for deploying api server
|
||||
apiServers="ds1"
|
||||
29
install.sh
29
install.sh
|
|
@ -34,11 +34,17 @@ datasourceDriverClassname="com.mysql.jdbc.Driver"
|
|||
if [ $dbtype == "postgresql" ];then
|
||||
datasourceDriverClassname="org.postgresql.Driver"
|
||||
fi
|
||||
|
||||
# Change configuration in conf/config/dolphinscheduler_env.sh
|
||||
sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh
|
||||
|
||||
# Change configuration in conf/datasource.properties
|
||||
sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties
|
||||
sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties
|
||||
sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties
|
||||
sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties
|
||||
|
||||
# Change configuration in conf/common.properties
|
||||
sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties
|
||||
sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties
|
||||
sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties
|
||||
|
|
@ -58,17 +64,22 @@ sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.add
|
|||
sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties
|
||||
sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties
|
||||
|
||||
# the following configurations may be commented, so ddd #\? to ensure successful sed
|
||||
sed -i ${txt} "s@^#\?worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
|
||||
sed -i ${txt} "s@^#\?alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
|
||||
sed -i ${txt} "s@^#\?task.plugin.dir=.*@task.plugin.dir=${taskPluginDir}@g" conf/worker.properties
|
||||
# The following configurations may be commented, so ddd #* to ensure sed work correct
|
||||
# Change configuration in conf/worker.properties
|
||||
sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
|
||||
sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
|
||||
sed -i ${txt} "s@^#*task.plugin.dir=.*@task.plugin.dir=${installPath}/${taskPluginDir}@g" conf/worker.properties
|
||||
|
||||
sed -i ${txt} "s@^#\?alert.plugin.dir=.*@alert.plugin.dir=${alertPluginDir}@g" conf/alert.properties
|
||||
sed -i ${txt} "s@^#\?server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
|
||||
# Change configuration in conf/alert.properties
|
||||
sed -i ${txt} "s@^#*alert.plugin.dir=.*@alert.plugin.dir=${installPath}/${alertPluginDir}@g" conf/alert.properties
|
||||
|
||||
sed -i ${txt} "s@^#\?registry.plugin.dir=.*@registry.plugin.dir=${registryPluginDir}@g" conf/registry.properties
|
||||
sed -i ${txt} "s@^#\?registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
|
||||
sed -i ${txt} "s@^#\?registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
|
||||
# Change configuration in conf/application-api.properties
|
||||
sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
|
||||
|
||||
# Change configuration in conf/registry.properties
|
||||
sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties
|
||||
sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
|
||||
sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
|
||||
|
||||
# 2.create directory
|
||||
echo "2.create directory"
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
|
|||
DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
|
||||
|
||||
source /etc/profile
|
||||
source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
|
||||
|
||||
export JAVA_HOME=$JAVA_HOME
|
||||
#export JAVA_HOME=/opt/soft/jdk
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
|
|||
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
|
||||
|
||||
source ${BIN_DIR}/../conf/config/install_config.conf
|
||||
source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh
|
||||
|
||||
export JAVA_HOME=$JAVA_HOME
|
||||
|
||||
|
|
@ -44,7 +45,7 @@ export STOP_TIMEOUT=5
|
|||
|
||||
CLASS=org.apache.zookeeper.ZooKeeperMain
|
||||
|
||||
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode"
|
||||
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $registryServers rmr $rootNode"
|
||||
|
||||
cd $DOLPHINSCHEDULER_HOME
|
||||
$JAVA_HOME/bin/java $exec_command
|
||||
|
|
|
|||
|
|
@ -57,7 +57,8 @@ do
|
|||
fi
|
||||
|
||||
echo "start to scp $dsDir to $host/$installPath"
|
||||
scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
|
||||
# Use quiet mode to reduce command line output
|
||||
scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath
|
||||
done
|
||||
|
||||
echo "scp dirs to $host/$installPath complete"
|
||||
|
|
|
|||
Loading…
Reference in New Issue