[DSIP-42] Add dolphinscheduler-aws-authentication module (#16043)
This commit is contained in:
Wenjun Ruan
GitHub
parent
baa252a0ed
commit
f754611509
|
|
@ -120,6 +120,12 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
||||||
| conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
|
| conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
|
||||||
| conf.common."alert.rpc.port" | int | `50052` | rpc port |
|
| conf.common."alert.rpc.port" | int | `50052` | rpc port |
|
||||||
| conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
|
| conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
|
||||||
|
| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | |
|
||||||
|
| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
|
||||||
|
| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
|
||||||
|
| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
|
||||||
|
| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
|
||||||
|
| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required |
|
||||||
| conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
|
| conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
|
||||||
| conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
|
| conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
|
||||||
| conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
|
| conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
|
||||||
|
|
@ -138,11 +144,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
||||||
| conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
|
| conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
|
||||||
| conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
|
| conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
|
||||||
| conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
|
| conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
|
||||||
| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
|
||||||
| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
|
||||||
| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
|
|
||||||
| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
|
|
||||||
| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
|
||||||
| conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
|
| conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
|
||||||
| conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
|
| conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
|
||||||
| conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
|
| conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ data:
|
||||||
{{- range $key, $value := index .Values.conf "common" }}
|
{{- range $key, $value := index .Values.conf "common" }}
|
||||||
{{- if and $.Values.minio.enabled }}
|
{{- if and $.Values.minio.enabled }}
|
||||||
{{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
|
{{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
|
||||||
{{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
|
{{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{ $key }}={{ $value }}
|
{{ $key }}={{ $value }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
|
||||||
|
|
@ -250,20 +250,25 @@ conf:
|
||||||
# -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
# -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||||
resource.storage.upload.base.path: /dolphinscheduler
|
resource.storage.upload.base.path: /dolphinscheduler
|
||||||
|
|
||||||
# -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
resource.aws.access.key.id: minioadmin
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
aws.credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
|
||||||
# -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
|
||||||
resource.aws.secret.access.key: minioadmin
|
aws.s3.access.key.id: minioadmin
|
||||||
|
|
||||||
# -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
|
||||||
resource.aws.region: ca-central-1
|
aws.s3.access.key.secret: minioadmin
|
||||||
|
|
||||||
|
# -- The AWS Region to use. if resource.storage.type=S3, This configuration is required
|
||||||
|
aws.s3.region: ca-central-1
|
||||||
|
|
||||||
# -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
# -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||||
resource.aws.s3.bucket.name: dolphinscheduler
|
aws.s3.bucket.name: dolphinscheduler
|
||||||
|
|
||||||
# -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
# -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||||
resource.aws.s3.endpoint: http://minio:9000
|
aws.s3.endpoint: http://minio:9000
|
||||||
|
|
||||||
# -- alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# -- alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id: <your-access-key-id>
|
resource.alibaba.cloud.access.key.id: <your-access-key-id>
|
||||||
|
|
|
||||||
|
|
@ -191,19 +191,21 @@ The default configuration is as follows:
|
||||||
|
|
||||||
Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
|
Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
|
||||||
|
|
||||||
For ETCD Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
|
For ETCD Registry, please see more details
|
||||||
For JDBC Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
|
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
|
||||||
|
For JDBC Registry, please see more details
|
||||||
|
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
|
||||||
|
|
||||||
### common.properties [hadoop、s3、yarn config properties]
|
### common.properties [hadoop、s3、yarn config properties]
|
||||||
|
|
||||||
Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
|
Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
|
||||||
|
|
||||||
| Service | Configuration file |
|
| Service | Configuration file |
|
||||||
|---------------|----------------------------------------|
|
|---------------|-----------------------------------------------------------------------|
|
||||||
| Master Server | `master-server/conf/common.properties` |
|
| Master Server | `master-server/conf/common.properties` |
|
||||||
| Api Server | `api-server/conf/common.properties` |
|
| Api Server | `api-server/conf/common.properties`, `api-server/conf/aws.yaml` |
|
||||||
| Worker Server | `worker-server/conf/common.properties` |
|
| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` |
|
||||||
| Alert Server | `alert-server/conf/common.properties` |
|
| Alert Server | `alert-server/conf/common.properties` |
|
||||||
|
|
||||||
The default configuration is as follows:
|
The default configuration is as follows:
|
||||||
|
|
||||||
|
|
@ -212,10 +214,6 @@ The default configuration is as follows:
|
||||||
| data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files |
|
| data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files |
|
||||||
| resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE |
|
| resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE |
|
||||||
| resource.upload.path | /dolphinscheduler | storage path of resource files |
|
| resource.upload.path | /dolphinscheduler | storage path of resource files |
|
||||||
| aws.access.key.id | minioadmin | access key id of S3 |
|
|
||||||
| aws.secret.access.key | minioadmin | secret access key of S3 |
|
|
||||||
| aws.region | us-east-1 | region of S3 |
|
|
||||||
| aws.s3.endpoint | http://minio:9000 | endpoint of S3 |
|
|
||||||
| hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS |
|
| hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS |
|
||||||
| fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory |
|
| fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory |
|
||||||
| hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission |
|
| hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission |
|
||||||
|
|
|
||||||
|
|
@ -28,74 +28,37 @@ The configuration you may need to change:
|
||||||
|
|
||||||
## connect AWS S3
|
## connect AWS S3
|
||||||
|
|
||||||
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following:
|
||||||
|
|
||||||
config the following fields
|
config the following fields
|
||||||
|
|
||||||
```properties
|
```properties
|
||||||
......
|
|
||||||
|
|
||||||
resource.storage.type=S3
|
resource.storage.type=S3
|
||||||
|
|
||||||
......
|
|
||||||
|
|
||||||
resource.aws.access.key.id=aws_access_key_id
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=aws_secret_access_key
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=us-west-2
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=
|
|
||||||
|
|
||||||
......
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Use HDFS or Remote Object Storage
|
```yaml
|
||||||
|
aws:
|
||||||
|
s3:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
bucket.name: <bucket.name>
|
||||||
|
endpoint: <endpoint>
|
||||||
|
|
||||||
After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`.
|
```
|
||||||
|
|
||||||
|
## connect OSS S3
|
||||||
|
|
||||||
|
if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
||||||
|
|
||||||
|
config the following fields
|
||||||
|
|
||||||
```properties
|
```properties
|
||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
|
||||||
data.basedir.path=/tmp/dolphinscheduler
|
|
||||||
|
|
||||||
# resource view suffixs
|
|
||||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
|
||||||
|
|
||||||
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
|
|
||||||
resource.storage.type=LOCAL
|
|
||||||
# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
|
||||||
resource.storage.upload.base.path=/tmp/dolphinscheduler
|
|
||||||
|
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.access.key.id=minioadmin
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=minioadmin
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=cn-north-1
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=http://localhost:9000
|
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||||
|
|
@ -107,89 +70,24 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
|
||||||
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OBS
|
```
|
||||||
|
|
||||||
|
## connect OBS S3
|
||||||
|
|
||||||
|
if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
||||||
|
|
||||||
|
config the following fields
|
||||||
|
|
||||||
|
```properties
|
||||||
|
# access key id, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
|
# access key secret, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
||||||
# oss bucket name, required if you set resource.storage.type=OBS
|
# oss bucket name, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
||||||
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
||||||
|
|
||||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
|
||||||
resource.hdfs.root.user=hdfs
|
|
||||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
|
||||||
resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
|
|
||||||
|
|
||||||
# whether to startup kerberos
|
|
||||||
hadoop.security.authentication.startup.state=false
|
|
||||||
|
|
||||||
# java.security.krb5.conf path
|
|
||||||
java.security.krb5.conf.path=/opt/krb5.conf
|
|
||||||
|
|
||||||
# login user from keytab username
|
|
||||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
|
||||||
|
|
||||||
# login user from keytab path
|
|
||||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
|
||||||
|
|
||||||
# kerberos expire time, the unit is hour
|
|
||||||
kerberos.expire.time=2
|
|
||||||
|
|
||||||
|
|
||||||
# resourcemanager port, the default value is 8088 if not specified
|
|
||||||
resource.manager.httpaddress.port=8088
|
|
||||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
|
||||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
|
||||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
|
||||||
yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
|
|
||||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
|
||||||
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
|
|
||||||
|
|
||||||
# datasource encryption enable
|
|
||||||
datasource.encryption.enable=false
|
|
||||||
|
|
||||||
# datasource encryption salt
|
|
||||||
datasource.encryption.salt=!@#$%^&*
|
|
||||||
|
|
||||||
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
|
|
||||||
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
|
|
||||||
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
|
|
||||||
data-quality.jar.dir=
|
|
||||||
|
|
||||||
#data-quality.error.output.path=/tmp/data-quality-error-data
|
|
||||||
|
|
||||||
# Network IP gets priority, default inner outer
|
|
||||||
|
|
||||||
# Whether hive SQL is executed in the same session
|
|
||||||
support.hive.oneSession=false
|
|
||||||
|
|
||||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
|
|
||||||
sudo.enable=true
|
|
||||||
|
|
||||||
# network interface preferred like eth0, default: empty
|
|
||||||
#dolphin.scheduler.network.interface.preferred=
|
|
||||||
|
|
||||||
# network IP gets priority, default: inner outer
|
|
||||||
#dolphin.scheduler.network.priority.strategy=default
|
|
||||||
|
|
||||||
# system env path
|
|
||||||
#dolphinscheduler.env.path=dolphinscheduler_env.sh
|
|
||||||
|
|
||||||
# development state
|
|
||||||
development.state=false
|
|
||||||
|
|
||||||
# rpc port
|
|
||||||
alert.rpc.port=50052
|
|
||||||
|
|
||||||
# set path of conda.sh
|
|
||||||
conda.path=/opt/anaconda3/etc/profile.d/conda.sh
|
|
||||||
|
|
||||||
# Task resource limit state
|
|
||||||
task.resource.limit.state=false
|
|
||||||
|
|
||||||
# way to collect applicationId: log(original regex match), aop
|
|
||||||
appId.collect: log
|
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Note:**
|
> **Note:**
|
||||||
|
|
|
||||||
|
|
@ -73,14 +73,17 @@ Parameters of restarting the task by interface
|
||||||
|
|
||||||
## Environment to prepare
|
## Environment to prepare
|
||||||
|
|
||||||
Some AWS configuration is required, modify a field in file `common.properties`
|
Some AWS configuration is required, modify a field in file `aws.yaml`
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
dms:
|
||||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
resource.aws.region=<AWS REGION>
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
endpoint: <endpoint>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -35,14 +35,17 @@ The task plugin are shown as follows:
|
||||||
|
|
||||||
## Environment to prepare
|
## Environment to prepare
|
||||||
|
|
||||||
Some AWS configuration is required, modify a field in file `common.properties`
|
Some AWS configuration is required, modify a field in file `aws.yaml`
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
sagemaker:
|
||||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
resource.aws.region=<AWS REGION>
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
endpoint: <endpoint>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -26,77 +26,35 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
|
||||||
|
|
||||||
## 对接AWS S3
|
## 对接AWS S3
|
||||||
|
|
||||||
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下：
|
||||||
|
|
||||||
配置以下字段
|
配置以下字段
|
||||||
|
|
||||||
```properties
|
```properties
|
||||||
......
|
|
||||||
|
|
||||||
resource.storage.type=S3
|
resource.storage.type=S3
|
||||||
|
|
||||||
......
|
|
||||||
|
|
||||||
resource.aws.access.key.id=aws_access_key_id
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=aws_secret_access_key
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=us-west-2
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=
|
|
||||||
|
|
||||||
......
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## 对接分布式或远端对象存储
|
```yaml
|
||||||
|
aws:
|
||||||
|
s3:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
bucket.name: <bucket.name>
|
||||||
|
endpoint: <endpoint>
|
||||||
|
|
||||||
当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置：
|
```
|
||||||
|
|
||||||
### 配置 common.properties 文件
|
## 对接阿里云 OSS
|
||||||
|
|
||||||
在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||||
|
|
||||||
```properties
|
```properties
|
||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
|
||||||
data.basedir.path=/tmp/dolphinscheduler
|
|
||||||
|
|
||||||
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
|
|
||||||
resource.storage.type=LOCAL
|
|
||||||
|
|
||||||
# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
|
|
||||||
# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
|
||||||
resource.storage.upload.base.path=/tmp/dolphinscheduler
|
|
||||||
|
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.access.key.id=minioadmin
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=minioadmin
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=cn-north-1
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=http://localhost:9000
|
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||||
|
|
@ -108,87 +66,22 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
|
||||||
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OBS
|
```
|
||||||
|
|
||||||
|
## 对接华为云 OBS
|
||||||
|
|
||||||
|
如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||||
|
|
||||||
|
```properties
|
||||||
|
# access key id, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
|
# access key secret, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
||||||
# oss bucket name, required if you set resource.storage.type=OBS
|
# oss bucket name, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
||||||
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
||||||
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
||||||
|
|
||||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
|
||||||
resource.hdfs.root.user=root
|
|
||||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler;
|
|
||||||
# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
|
||||||
resource.hdfs.fs.defaultFS=hdfs://localhost:8020
|
|
||||||
|
|
||||||
# whether to startup kerberos
|
|
||||||
hadoop.security.authentication.startup.state=false
|
|
||||||
|
|
||||||
# java.security.krb5.conf path
|
|
||||||
java.security.krb5.conf.path=/opt/krb5.conf
|
|
||||||
|
|
||||||
# login user from keytab username
|
|
||||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
|
||||||
|
|
||||||
# login user from keytab path
|
|
||||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
|
||||||
|
|
||||||
# kerberos expire time, the unit is hour
|
|
||||||
kerberos.expire.time=2
|
|
||||||
# resource view suffixs
|
|
||||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
|
||||||
|
|
||||||
# resourcemanager port, the default value is 8088 if not specified
|
|
||||||
resource.manager.httpaddress.port=8088
|
|
||||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
|
||||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
|
||||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value;
|
|
||||||
# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
|
||||||
yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s
|
|
||||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
|
||||||
yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s
|
|
||||||
|
|
||||||
# datasource encryption enable
|
|
||||||
datasource.encryption.enable=false
|
|
||||||
|
|
||||||
# datasource encryption salt
|
|
||||||
datasource.encryption.salt=!@#$%^&*
|
|
||||||
|
|
||||||
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
|
|
||||||
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
|
|
||||||
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
|
|
||||||
data-quality.jar.dir=
|
|
||||||
|
|
||||||
#data-quality.error.output.path=/tmp/data-quality-error-data
|
|
||||||
|
|
||||||
# Network IP gets priority, default inner outer
|
|
||||||
|
|
||||||
# Whether hive SQL is executed in the same session
|
|
||||||
support.hive.oneSession=false
|
|
||||||
|
|
||||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions;
|
|
||||||
# if set false, executing user is the deploy user and doesn't need sudo permissions
|
|
||||||
sudo.enable=true
|
|
||||||
|
|
||||||
# network interface preferred like eth0, default: empty
|
|
||||||
#dolphin.scheduler.network.interface.preferred=
|
|
||||||
|
|
||||||
# network IP gets priority, default: inner outer
|
|
||||||
#dolphin.scheduler.network.priority.strategy=default
|
|
||||||
|
|
||||||
# system env path
|
|
||||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
|
||||||
|
|
||||||
# development state
|
|
||||||
development.state=false
|
|
||||||
|
|
||||||
# rpc port
|
|
||||||
alert.rpc.port=50052
|
|
||||||
|
|
||||||
# way to collect applicationId: log(original regex match), aop
|
|
||||||
appId.collect: log
|
|
||||||
```
|
```
|
||||||
|
|
||||||
> **注意**：
|
> **注意**：
|
||||||
|
|
|
||||||
|
|
@ -73,14 +73,17 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS
|
||||||
|
|
||||||
## 环境配置
|
## 环境配置
|
||||||
|
|
||||||
需要进行AWS的一些配置，修改`common.properties`中的以下配置信息
|
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
dms:
|
||||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
resource.aws.region=<AWS REGION>
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
endpoint: <endpoint>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,14 +33,17 @@ DolphinScheduler SageMaker 组件的功能:
|
||||||
|
|
||||||
## 环境配置
|
## 环境配置
|
||||||
|
|
||||||
需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息
|
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
sagemaker:
|
||||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
resource.aws.region=<AWS REGION>
|
access.key.id: <access.key.id>
|
||||||
|
access.key.secret: <access.key.secret>
|
||||||
|
region: <region>
|
||||||
|
endpoint: <endpoint>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,13 @@
|
||||||
</includes>
|
</includes>
|
||||||
<outputDirectory>conf</outputDirectory>
|
<outputDirectory>conf</outputDirectory>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*.yaml</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>conf</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
</fileSets>
|
</fileSets>
|
||||||
<dependencySets>
|
<dependencySets>
|
||||||
<dependencySet>
|
<dependencySet>
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,13 @@
|
||||||
</includes>
|
</includes>
|
||||||
<outputDirectory>conf</outputDirectory>
|
<outputDirectory>conf</outputDirectory>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*.yaml</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>conf</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
<fileSet>
|
<fileSet>
|
||||||
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
|
<directory>${basedir}/../dolphinscheduler-ui/dist</directory>
|
||||||
<outputDirectory>./ui</outputDirectory>
|
<outputDirectory>./ui</outputDirectory>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
~ contributor license agreements. See the NOTICE file distributed with
|
||||||
|
~ this work for additional information regarding copyright ownership.
|
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
~ (the "License"); you may not use this file except in compliance with
|
||||||
|
~ the License. You may obtain a copy of the License at
|
||||||
|
~
|
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
~
|
||||||
|
~ Unless required by applicable law or agreed to in writing, software
|
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
~ See the License for the specific language governing permissions and
|
||||||
|
~ limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler-authentication</artifactId>
|
||||||
|
<version>dev-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.amazonaws</groupId>
|
||||||
|
<artifactId>aws-java-sdk-emr</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.amazonaws</groupId>
|
||||||
|
<artifactId>aws-java-sdk-s3</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.amazonaws</groupId>
|
||||||
|
<artifactId>aws-java-sdk-sagemaker</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.amazonaws</groupId>
|
||||||
|
<artifactId>aws-java-sdk-dms</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>software.amazon.awssdk</groupId>
|
||||||
|
<artifactId>datasync</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.authentication.aws.AwsConfigurationKeys.AWS_AUTHENTICATION_TYPE;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||||
|
import com.amazonaws.auth.BasicAWSCredentials;
|
||||||
|
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@UtilityClass
|
||||||
|
public class AWSCredentialsProviderFactor {
|
||||||
|
|
||||||
|
public static AWSCredentialsProvider credentialsProvider(Map<String, String> awsProperties) {
|
||||||
|
String awsAuthenticationType = awsProperties.getOrDefault(
|
||||||
|
AWS_AUTHENTICATION_TYPE, AWSCredentialsProviderType.STATIC_CREDENTIALS_PROVIDER.getName());
|
||||||
|
AWSCredentialsProviderType awsCredentialsProviderType =
|
||||||
|
AWSCredentialsProviderType.of(awsAuthenticationType).orElse(null);
|
||||||
|
if (awsCredentialsProviderType == null) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated");
|
||||||
|
}
|
||||||
|
switch (awsCredentialsProviderType) {
|
||||||
|
case STATIC_CREDENTIALS_PROVIDER:
|
||||||
|
return createAWSStaticCredentialsProvider(awsProperties);
|
||||||
|
case INSTANCE_PROFILE_CREDENTIALS_PROVIDER:
|
||||||
|
return createInstanceProfileCredentialsProvider();
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AWSCredentialsProvider createAWSStaticCredentialsProvider(Map<String, String> awsProperties) {
|
||||||
|
String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID);
|
||||||
|
String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET);
|
||||||
|
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey);
|
||||||
|
AWSStaticCredentialsProvider awsStaticCredentialsProvider =
|
||||||
|
new AWSStaticCredentialsProvider(basicAWSCredentials);
|
||||||
|
log.info("AWSStaticCredentialsProvider created successfully");
|
||||||
|
return awsStaticCredentialsProvider;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AWSCredentialsProvider createInstanceProfileCredentialsProvider() {
|
||||||
|
InstanceProfileCredentialsProvider instanceProfileCredentialsProvider =
|
||||||
|
InstanceProfileCredentialsProvider.getInstance();
|
||||||
|
log.info("InstanceProfileCredentialsProvider created successfully");
|
||||||
|
return instanceProfileCredentialsProvider;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
public enum AWSCredentialsProviderType {
|
||||||
|
|
||||||
|
STATIC_CREDENTIALS_PROVIDER("AWSStaticCredentialsProvider"),
|
||||||
|
INSTANCE_PROFILE_CREDENTIALS_PROVIDER("InstanceProfileCredentialsProvider"),
|
||||||
|
;
|
||||||
|
|
||||||
|
private final String name;
|
||||||
|
|
||||||
|
AWSCredentialsProviderType(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Optional<AWSCredentialsProviderType> of(String name) {
|
||||||
|
if (name == null) {
|
||||||
|
return Optional.empty();
|
||||||
|
}
|
||||||
|
for (AWSCredentialsProviderType type : values()) {
|
||||||
|
if (type.getName().equalsIgnoreCase(name)) {
|
||||||
|
return Optional.of(type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Optional.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||||
|
import com.amazonaws.regions.Regions;
|
||||||
|
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService;
|
||||||
|
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class AWSDatabaseMigrationServiceClientFactory {
|
||||||
|
|
||||||
|
public AWSDatabaseMigrationService createAWSDatabaseMigrationServiceClient(Map<String, String> awsProperties) {
|
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
|
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
|
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
|
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) {
|
||||||
|
return AWSDatabaseMigrationServiceClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
} else {
|
||||||
|
return AWSDatabaseMigrationServiceClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.withRegion(regions)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||||
|
import com.amazonaws.regions.Regions;
|
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
|
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class AmazonElasticMapReduceClientFactory {
|
||||||
|
|
||||||
|
public AmazonElasticMapReduce createAmazonElasticMapReduceClient(Map<String, String> awsProperties) {
|
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
|
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
|
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
|
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) {
|
||||||
|
return AmazonElasticMapReduceClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
} else {
|
||||||
|
return AmazonElasticMapReduceClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.withRegion(regions)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||||
|
import com.amazonaws.regions.Regions;
|
||||||
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
|
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class AmazonS3ClientFactory {
|
||||||
|
|
||||||
|
public AmazonS3 createAmazonS3Client(Map<String, String> awsProperties) {
|
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
|
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
|
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
|
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) {
|
||||||
|
return AmazonS3ClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withPathStyleAccessEnabled(true)
|
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
} else {
|
||||||
|
return AmazonS3ClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.withRegion(regions)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||||
|
import com.amazonaws.regions.Regions;
|
||||||
|
import com.amazonaws.services.sagemaker.AmazonSageMaker;
|
||||||
|
import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class AmazonSageMakerClientFactory {
|
||||||
|
|
||||||
|
public AmazonSageMaker createAmazonSageMakerClient(Map<String, String> awsProperties) {
|
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties);
|
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION));
|
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT);
|
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) {
|
||||||
|
return AmazonSageMakerClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
} else {
|
||||||
|
return AmazonSageMakerClientBuilder
|
||||||
|
.standard()
|
||||||
|
.withCredentials(awsCredentialsProvider)
|
||||||
|
.withRegion(regions)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
public class AwsConfigurationKeys {
|
||||||
|
|
||||||
|
public static final String AWS_AUTHENTICATION_TYPE = "credentials.provider.type";
|
||||||
|
public static final String AWS_REGION = "region";
|
||||||
|
public static final String AWS_ENDPOINT = "endpoint";
|
||||||
|
|
||||||
|
public static final String AWS_ACCESS_KEY_ID = "access.key.id";
|
||||||
|
public static final String AWS_SECRET = "access.key.secret";
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws;
|
||||||
|
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.regions.Region;
|
||||||
|
import software.amazon.awssdk.services.datasync.DataSyncClient;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class DataSyncClientFactory {
|
||||||
|
|
||||||
|
public DataSyncClient createDataSyncClient(Map<String, String> awsProperties) {
|
||||||
|
// todo: upgrade the version of aws sdk
|
||||||
|
String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID);
|
||||||
|
String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET);
|
||||||
|
final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey);
|
||||||
|
final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials);
|
||||||
|
|
||||||
|
// create a datasync client
|
||||||
|
return DataSyncClient.builder()
|
||||||
|
.region(Region.of(awsProperties.get(AwsConfigurationKeys.AWS_REGION)))
|
||||||
|
.credentialsProvider(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
aws:
|
||||||
|
s3:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: accessKey123
|
||||||
|
access.key.secret: secretKey123
|
||||||
|
region: us-east-1
|
||||||
|
bucket.name: dolphinscheduler
|
||||||
|
endpoint: http://s3:9000
|
||||||
|
emr:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
sagemaker:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
dms:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
datasync:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
~ contributor license agreements. See the NOTICE file distributed with
|
||||||
|
~ this work for additional information regarding copyright ownership.
|
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
~ (the "License"); you may not use this file except in compliance with
|
||||||
|
~ the License. You may obtain a copy of the License at
|
||||||
|
~
|
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
~
|
||||||
|
~ Unless required by applicable law or agreed to in writing, software
|
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
~ See the License for the specific language governing permissions and
|
||||||
|
~ limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler</artifactId>
|
||||||
|
<version>dev-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-authentication</artifactId>
|
||||||
|
<packaging>pom</packaging>
|
||||||
|
|
||||||
|
<modules>
|
||||||
|
<module>dolphinscheduler-aws-authentication</module>
|
||||||
|
</modules>
|
||||||
|
|
||||||
|
<dependencyManagement>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler-bom</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<type>pom</type>
|
||||||
|
<scope>import</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</dependencyManagement>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -41,6 +41,10 @@
|
||||||
</dependencyManagement>
|
</dependencyManagement>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<!-- dolphinscheduler -->
|
<!-- dolphinscheduler -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>commons-io</groupId>
|
<groupId>commons-io</groupId>
|
||||||
<artifactId>commons-io</artifactId>
|
<artifactId>commons-io</artifactId>
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ public final class Constants {
|
||||||
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
|
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
|
public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
|
||||||
|
public static final String AWS_YAML_PATH = "/aws.yaml";
|
||||||
|
|
||||||
public static final String FORMAT_SS = "%s%s";
|
public static final String FORMAT_SS = "%s%s";
|
||||||
public static final String FORMAT_S_S = "%s/%s";
|
public static final String FORMAT_S_S = "%s/%s";
|
||||||
|
|
@ -130,8 +131,7 @@ public final class Constants {
|
||||||
*/
|
*/
|
||||||
public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type";
|
public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type";
|
||||||
|
|
||||||
public static final String AWS_S3_BUCKET_NAME = "resource.aws.s3.bucket.name";
|
public static final String AWS_S3_BUCKET_NAME = "aws.s3.bucket.name";
|
||||||
public static final String AWS_END_POINT = "resource.aws.s3.endpoint";
|
|
||||||
|
|
||||||
public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name";
|
public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name";
|
||||||
public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint";
|
public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint";
|
||||||
|
|
@ -704,19 +704,8 @@ public final class Constants {
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint";
|
public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint";
|
||||||
|
|
||||||
/**
|
|
||||||
* remote logging for S3
|
|
||||||
*/
|
|
||||||
public static final String REMOTE_LOGGING_S3_ACCESS_KEY_ID = "remote.logging.s3.access.key.id";
|
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_S3_ACCESS_KEY_SECRET = "remote.logging.s3.access.key.secret";
|
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name";
|
public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name";
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_S3_ENDPOINT = "remote.logging.s3.endpoint";
|
|
||||||
|
|
||||||
public static final String REMOTE_LOGGING_S3_REGION = "remote.logging.s3.region";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* remote logging for GCS
|
* remote logging for GCS
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.common.log.remote;
|
package org.apache.dolphinscheduler.common.log.remote;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
|
|
||||||
|
|
@ -26,41 +27,25 @@ import java.io.Closeable;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
|
||||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
|
||||||
import com.amazonaws.regions.Regions;
|
|
||||||
import com.amazonaws.services.s3.AmazonS3;
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
|
||||||
import com.amazonaws.services.s3.model.S3Object;
|
import com.amazonaws.services.s3.model.S3Object;
|
||||||
import com.amazonaws.services.s3.model.S3ObjectInputStream;
|
import com.amazonaws.services.s3.model.S3ObjectInputStream;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
|
public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
|
||||||
|
|
||||||
private String accessKeyId;
|
private final String bucketName;
|
||||||
|
|
||||||
private String accessKeySecret;
|
private final AmazonS3 s3Client;
|
||||||
|
|
||||||
private String region;
|
|
||||||
|
|
||||||
private String bucketName;
|
|
||||||
|
|
||||||
private String endPoint;
|
|
||||||
|
|
||||||
private AmazonS3 s3Client;
|
|
||||||
|
|
||||||
private static S3RemoteLogHandler instance;
|
private static S3RemoteLogHandler instance;
|
||||||
|
|
||||||
private S3RemoteLogHandler() {
|
private S3RemoteLogHandler() {
|
||||||
accessKeyId = readAccessKeyID();
|
|
||||||
accessKeySecret = readAccessKeySecret();
|
|
||||||
region = readRegion();
|
|
||||||
bucketName = readBucketName();
|
bucketName = readBucketName();
|
||||||
endPoint = readEndPoint();
|
|
||||||
s3Client = buildS3Client();
|
s3Client = buildS3Client();
|
||||||
checkBucketNameExists(bucketName);
|
checkBucketNameExists(bucketName);
|
||||||
}
|
}
|
||||||
|
|
@ -74,23 +59,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected AmazonS3 buildS3Client() {
|
protected AmazonS3 buildS3Client() {
|
||||||
if (StringUtils.isNotEmpty(endPoint)) {
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.s3.", "");
|
||||||
return AmazonS3ClientBuilder
|
return AmazonS3ClientFactory.createAmazonS3Client(awsProperties);
|
||||||
.standard()
|
|
||||||
.withPathStyleAccessEnabled(true)
|
|
||||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
|
|
||||||
endPoint, Regions.fromName(region).getName()))
|
|
||||||
.withCredentials(
|
|
||||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
|
|
||||||
.build();
|
|
||||||
} else {
|
|
||||||
return AmazonS3ClientBuilder
|
|
||||||
.standard()
|
|
||||||
.withCredentials(
|
|
||||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
|
|
||||||
.withRegion(Regions.fromName(region))
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -131,24 +101,8 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String readAccessKeyID() {
|
|
||||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_ID);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readAccessKeySecret() {
|
|
||||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ACCESS_KEY_SECRET);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readRegion() {
|
|
||||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_REGION);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readBucketName() {
|
protected String readBucketName() {
|
||||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_BUCKET_NAME);
|
return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME);
|
||||||
}
|
|
||||||
|
|
||||||
protected String readEndPoint() {
|
|
||||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_S3_ENDPOINT);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void checkBucketNameExists(String bucketName) {
|
public void checkBucketNameExists(String bucketName) {
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.common.utils;
|
package org.apache.dolphinscheduler.common.utils;
|
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH;
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
|
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
|
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
|
||||||
|
|
||||||
|
|
@ -42,7 +43,7 @@ public class PropertyUtils {
|
||||||
private final ImmutablePriorityPropertyDelegate propertyDelegate =
|
private final ImmutablePriorityPropertyDelegate propertyDelegate =
|
||||||
new ImmutablePriorityPropertyDelegate(
|
new ImmutablePriorityPropertyDelegate(
|
||||||
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
|
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
|
||||||
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
|
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH));
|
||||||
|
|
||||||
public static String getString(String key) {
|
public static String getString(String key) {
|
||||||
return propertyDelegate.get(key.trim());
|
return propertyDelegate.get(key.trim());
|
||||||
|
|
@ -106,6 +107,19 @@ public class PropertyUtils {
|
||||||
return matchedProperties;
|
return matchedProperties;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all properties with specified prefix, like: fs., will replace the prefix with newPrefix
|
||||||
|
*/
|
||||||
|
public static Map<String, String> getByPrefix(String prefix, String newPrefix) {
|
||||||
|
Map<String, String> matchedProperties = new HashMap<>();
|
||||||
|
for (String propName : propertyDelegate.getPropertyKeys()) {
|
||||||
|
if (propName.startsWith(prefix)) {
|
||||||
|
matchedProperties.put(propName.replace(prefix, newPrefix), propertyDelegate.get(propName));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return matchedProperties;
|
||||||
|
}
|
||||||
|
|
||||||
public static <T> Set<T> getSet(String key, Function<String, Set<T>> transformFunction, Set<T> defaultValue) {
|
public static <T> Set<T> getSet(String key, Function<String, Set<T>> transformFunction, Set<T> defaultValue) {
|
||||||
return propertyDelegate.get(key, transformFunction, defaultValue);
|
return propertyDelegate.get(key, transformFunction, defaultValue);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -39,17 +39,6 @@ resource.azure.tenant.id=minioadmin
|
||||||
# The query interval
|
# The query interval
|
||||||
resource.query.interval=10000
|
resource.query.interval=10000
|
||||||
|
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.access.key.id=minioadmin
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=minioadmin
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=cn-north-1
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=http://localhost:9000
|
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||||
|
|
@ -188,16 +177,7 @@ remote.logging.oss.access.key.secret=<access.key.secret>
|
||||||
remote.logging.oss.bucket.name=<bucket.name>
|
remote.logging.oss.bucket.name=<bucket.name>
|
||||||
# oss endpoint, required if you set remote.logging.target=OSS
|
# oss endpoint, required if you set remote.logging.target=OSS
|
||||||
remote.logging.oss.endpoint=<endpoint>
|
remote.logging.oss.endpoint=<endpoint>
|
||||||
# s3 access key id, required if you set remote.logging.target=S3
|
|
||||||
remote.logging.s3.access.key.id=<access.key.id>
|
|
||||||
# s3 access key secret, required if you set remote.logging.target=S3
|
|
||||||
remote.logging.s3.access.key.secret=<access.key.secret>
|
|
||||||
# s3 bucket name, required if you set remote.logging.target=S3
|
|
||||||
remote.logging.s3.bucket.name=<bucket.name>
|
|
||||||
# s3 endpoint, required if you set remote.logging.target=S3
|
|
||||||
remote.logging.s3.endpoint=<endpoint>
|
|
||||||
# s3 region, required if you set remote.logging.target=S3
|
|
||||||
remote.logging.s3.region=<region>
|
|
||||||
# the location of the google cloud credential, required if you set remote.logging.target=GCS
|
# the location of the google cloud credential, required if you set remote.logging.target=GCS
|
||||||
remote.logging.google.cloud.storage.credential=/path/to/credential
|
remote.logging.google.cloud.storage.credential=/path/to/credential
|
||||||
# gcs bucket name, required if you set remote.logging.target=GCS
|
# gcs bucket name, required if you set remote.logging.target=GCS
|
||||||
|
|
|
||||||
|
|
@ -34,28 +34,16 @@ remote-logging:
|
||||||
bucket.name: <bucket.name>
|
bucket.name: <bucket.name>
|
||||||
# oss endpoint, required if you set remote-logging.target=OSS
|
# oss endpoint, required if you set remote-logging.target=OSS
|
||||||
endpoint: <endpoint>
|
endpoint: <endpoint>
|
||||||
# required if you set remote-logging.target=S3
|
|
||||||
s3:
|
|
||||||
# s3 access key id, required if you set remote-logging.target=S3
|
|
||||||
access.key.id: <access.key.id>
|
|
||||||
# s3 access key secret, required if you set remote-logging.target=S3
|
|
||||||
access.key.secret: <access.key.secret>
|
|
||||||
# s3 bucket name, required if you set remote-logging.target=S3
|
|
||||||
bucket.name: <bucket.name>
|
|
||||||
# s3 endpoint, required if you set remote-logging.target=S3
|
|
||||||
endpoint: <endpoint>
|
|
||||||
# s3 region, required if you set remote-logging.target=S3
|
|
||||||
region: <region>
|
|
||||||
google.cloud.storage:
|
google.cloud.storage:
|
||||||
# the location of the google cloud credential, required if you set remote-logging.target=GCS
|
# the location of the google cloud credential, required if you set remote-logging.target=GCS
|
||||||
credential: /path/to/credential
|
credential: /path/to/credential
|
||||||
# gcs bucket name, required if you set remote-logging.target=GCS
|
# gcs bucket name, required if you set remote-logging.target=GCS
|
||||||
bucket.name: <your-bucket>
|
bucket.name: <your-bucket>
|
||||||
abs:
|
abs:
|
||||||
# abs account name, required if you set resource.storage.type=ABS
|
# abs account name, required if you set resource.storage.type=ABS
|
||||||
account.name: <your-account-name>
|
account.name: <your-account-name>
|
||||||
# abs account key, required if you set resource.storage.type=ABS
|
# abs account key, required if you set resource.storage.type=ABS
|
||||||
account.key: <your-account-key>
|
account.key: <your-account-key>
|
||||||
# abs container name, required if you set resource.storage.type=ABS
|
# abs container name, required if you set resource.storage.type=ABS
|
||||||
container.name: <your-container-name>
|
container.name: <your-container-name>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,12 +17,15 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.common.utils;
|
package org.apache.dolphinscheduler.common.utils;
|
||||||
|
|
||||||
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
@ -48,4 +51,14 @@ public class PropertyUtilsTest {
|
||||||
}, Sets.newHashSet("docker0"));
|
}, Sets.newHashSet("docker0"));
|
||||||
Assertions.assertEquals(Sets.newHashSet("docker0"), networkInterface);
|
Assertions.assertEquals(Sets.newHashSet("docker0"), networkInterface);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void getByPrefix() {
|
||||||
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("resource.aws.", "");
|
||||||
|
assertThat(awsProperties).containsEntry("access.key.id", "minioadmin");
|
||||||
|
assertThat(awsProperties).containsEntry("secret.access.key", "minioadmin");
|
||||||
|
assertThat(awsProperties).containsEntry("region", "cn-north-1");
|
||||||
|
assertThat(awsProperties).containsEntry("s3.bucket.name", "dolphinscheduler");
|
||||||
|
assertThat(awsProperties).containsEntry("endpoint", "http://localhost:9000");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,10 @@ resource.azure.tenant.id=minioadmin
|
||||||
# The query interval
|
# The query interval
|
||||||
resource.query.interval=10000
|
resource.query.interval=10000
|
||||||
|
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
aws.credentials.provider.type=AWSStaticCredentialsProvider
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||||
resource.aws.access.key.id=minioadmin
|
resource.aws.access.key.id=minioadmin
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||||
|
|
@ -54,7 +58,7 @@ resource.aws.region=cn-north-1
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
resource.aws.s3.bucket.name=dolphinscheduler
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||||
resource.aws.s3.endpoint=http://localhost:9000
|
resource.aws.endpoint=http://localhost:9000
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
aws:
|
||||||
|
s3:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: accessKey123
|
||||||
|
access.key.secret: secretKey123
|
||||||
|
region: us-east-1
|
||||||
|
bucket.name: dolphinscheduler
|
||||||
|
endpoint: http://s3:9000
|
||||||
|
emr:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
sagemaker:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
dms:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
datasync:
|
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider
|
||||||
|
access.key.id: minioadmin
|
||||||
|
access.key.secret: minioadmin
|
||||||
|
region: cn-north-1
|
||||||
|
endpoint: http://localhost:9000
|
||||||
|
|
||||||
|
|
@ -37,17 +37,6 @@ resource.azure.tenant.id=minioadmin
|
||||||
# The query interval
|
# The query interval
|
||||||
resource.query.interval=10000
|
resource.query.interval=10000
|
||||||
|
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.access.key.id=accessKey123
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=secretKey123
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=us-east-1
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=http://s3:9000
|
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,7 @@ services:
|
||||||
retries: 120
|
retries: 120
|
||||||
volumes:
|
volumes:
|
||||||
- ./common.properties:/opt/dolphinscheduler/conf/common.properties
|
- ./common.properties:/opt/dolphinscheduler/conf/common.properties
|
||||||
|
- ./aws.yaml:/opt/dolphinscheduler/conf/aws.yaml
|
||||||
depends_on:
|
depends_on:
|
||||||
s3:
|
s3:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,13 @@
|
||||||
</includes>
|
</includes>
|
||||||
<outputDirectory>conf</outputDirectory>
|
<outputDirectory>conf</outputDirectory>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*.yaml</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>conf</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
</fileSets>
|
</fileSets>
|
||||||
<dependencySets>
|
<dependencySets>
|
||||||
<dependencySet>
|
<dependencySet>
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,13 @@
|
||||||
</includes>
|
</includes>
|
||||||
<outputDirectory>conf</outputDirectory>
|
<outputDirectory>conf</outputDirectory>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*.yaml</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>conf</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
<fileSet>
|
<fileSet>
|
||||||
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
|
<directory>${basedir}/../dolphinscheduler-api/src/main/resources</directory>
|
||||||
<includes>
|
<includes>
|
||||||
|
|
|
||||||
|
|
@ -17,19 +17,18 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.plugin.storage.s3;
|
package org.apache.dolphinscheduler.plugin.storage.s3;
|
||||||
|
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.AWS_END_POINT;
|
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
|
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
|
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
|
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF;
|
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||||
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
import org.apache.dolphinscheduler.common.enums.ResUploadType;
|
||||||
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
|
||||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
|
||||||
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
@ -57,11 +56,7 @@ import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import com.amazonaws.AmazonServiceException;
|
import com.amazonaws.AmazonServiceException;
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
|
||||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
|
||||||
import com.amazonaws.services.s3.AmazonS3;
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
|
||||||
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
||||||
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
||||||
import com.amazonaws.services.s3.model.ListObjectsV2Request;
|
import com.amazonaws.services.s3.model.ListObjectsV2Request;
|
||||||
|
|
@ -79,71 +74,27 @@ import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||||
@Data
|
@Data
|
||||||
public class S3StorageOperator implements Closeable, StorageOperate {
|
public class S3StorageOperator implements Closeable, StorageOperate {
|
||||||
|
|
||||||
private String accessKeyId;
|
|
||||||
|
|
||||||
private String accessKeySecret;
|
|
||||||
|
|
||||||
private String region;
|
|
||||||
|
|
||||||
private String bucketName;
|
private String bucketName;
|
||||||
|
|
||||||
private String endPoint;
|
|
||||||
|
|
||||||
private AmazonS3 s3Client;
|
private AmazonS3 s3Client;
|
||||||
|
|
||||||
public S3StorageOperator() {
|
public S3StorageOperator() {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void init() {
|
public void init() {
|
||||||
accessKeyId = readAccessKeyID();
|
|
||||||
accessKeySecret = readAccessKeySecret();
|
|
||||||
region = readRegion();
|
|
||||||
bucketName = readBucketName();
|
bucketName = readBucketName();
|
||||||
endPoint = readEndPoint();
|
|
||||||
s3Client = buildS3Client();
|
s3Client = buildS3Client();
|
||||||
checkBucketNameExists(bucketName);
|
checkBucketNameExists(bucketName);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected AmazonS3 buildS3Client() {
|
protected AmazonS3 buildS3Client() {
|
||||||
if (!StringUtils.isEmpty(endPoint)) {
|
return AmazonS3ClientFactory.createAmazonS3Client(PropertyUtils.getByPrefix("aws.s3.", ""));
|
||||||
return AmazonS3ClientBuilder
|
|
||||||
.standard()
|
|
||||||
.withPathStyleAccessEnabled(true)
|
|
||||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
|
|
||||||
endPoint, region))
|
|
||||||
.withCredentials(
|
|
||||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
|
|
||||||
.build();
|
|
||||||
} else {
|
|
||||||
return AmazonS3ClientBuilder
|
|
||||||
.standard()
|
|
||||||
.withCredentials(
|
|
||||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
|
|
||||||
.withRegion(region)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readAccessKeyID() {
|
|
||||||
return PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readAccessKeySecret() {
|
|
||||||
return PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected String readRegion() {
|
|
||||||
return PropertyUtils.getString(TaskConstants.AWS_REGION);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String readBucketName() {
|
protected String readBucketName() {
|
||||||
return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME);
|
return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String readEndPoint() {
|
|
||||||
return PropertyUtils.getString(AWS_END_POINT);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
s3Client.shutdown();
|
s3Client.shutdown();
|
||||||
|
|
|
||||||
|
|
@ -78,13 +78,7 @@ public class S3StorageOperatorTest {
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
s3StorageOperator = Mockito.spy(new S3StorageOperator());
|
s3StorageOperator = Mockito.spy(new S3StorageOperator());
|
||||||
|
|
||||||
doReturn(ACCESS_KEY_ID_MOCK).when(s3StorageOperator)
|
|
||||||
.readAccessKeyID();
|
|
||||||
doReturn(ACCESS_KEY_SECRET_MOCK).when(s3StorageOperator)
|
|
||||||
.readAccessKeySecret();
|
|
||||||
doReturn(REGION_MOCK).when(s3StorageOperator).readRegion();
|
|
||||||
doReturn(BUCKET_NAME_MOCK).when(s3StorageOperator).readBucketName();
|
doReturn(BUCKET_NAME_MOCK).when(s3StorageOperator).readBucketName();
|
||||||
doReturn(END_POINT_MOCK).when(s3StorageOperator).readEndPoint();
|
|
||||||
Mockito.doReturn(s3Client)
|
Mockito.doReturn(s3Client)
|
||||||
.when(s3StorageOperator).buildS3Client();
|
.when(s3StorageOperator).buildS3Client();
|
||||||
Mockito.doNothing()
|
Mockito.doNothing()
|
||||||
|
|
@ -96,9 +90,6 @@ public class S3StorageOperatorTest {
|
||||||
@Test
|
@Test
|
||||||
public void testInit() {
|
public void testInit() {
|
||||||
verify(s3StorageOperator, times(1)).buildS3Client();
|
verify(s3StorageOperator, times(1)).buildS3Client();
|
||||||
Assertions.assertEquals(ACCESS_KEY_ID_MOCK, s3StorageOperator.getAccessKeyId());
|
|
||||||
Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, s3StorageOperator.getAccessKeySecret());
|
|
||||||
Assertions.assertEquals(REGION_MOCK, s3StorageOperator.getRegion());
|
|
||||||
Assertions.assertEquals(BUCKET_NAME_MOCK, s3StorageOperator.getBucketName());
|
Assertions.assertEquals(BUCKET_NAME_MOCK, s3StorageOperator.getBucketName());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -26,17 +26,6 @@ resource.storage.type=NONE
|
||||||
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||||
resource.storage.upload.base.path=/dolphinscheduler
|
resource.storage.upload.base.path=/dolphinscheduler
|
||||||
|
|
||||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.access.key.id=minioadmin
|
|
||||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.secret.access.key=minioadmin
|
|
||||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
|
||||||
resource.aws.region=cn-north-1
|
|
||||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
|
||||||
resource.aws.s3.bucket.name=dolphinscheduler
|
|
||||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
|
||||||
resource.aws.s3.endpoint=http://localhost:9000
|
|
||||||
|
|
||||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||||
|
|
|
||||||
|
|
@ -31,22 +31,18 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-spi</artifactId>
|
<artifactId>dolphinscheduler-spi</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/datasync -->
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>software.amazon.awssdk</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>datasync</artifactId>
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-common</artifactId>
|
<artifactId>dolphinscheduler-common</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,13 +17,9 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.plugin.task.datasync;
|
package org.apache.dolphinscheduler.plugin.task.datasync;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.DataSyncClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
|
||||||
|
|
||||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
|
||||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
|
||||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
|
||||||
import software.amazon.awssdk.regions.Region;
|
|
||||||
import software.amazon.awssdk.services.datasync.DataSyncClient;
|
import software.amazon.awssdk.services.datasync.DataSyncClient;
|
||||||
import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionRequest;
|
import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionRequest;
|
||||||
import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionResponse;
|
import software.amazon.awssdk.services.datasync.model.CancelTaskExecutionResponse;
|
||||||
|
|
@ -48,6 +44,7 @@ import org.apache.commons.lang3.StringUtils;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
@ -73,16 +70,8 @@ public class DatasyncHook {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected static DataSyncClient createClient() {
|
protected static DataSyncClient createClient() {
|
||||||
final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID);
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.datasync.", "");
|
||||||
final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY);
|
return DataSyncClientFactory.createDataSyncClient(awsProperties);
|
||||||
final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION);
|
|
||||||
|
|
||||||
final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey);
|
|
||||||
final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials);
|
|
||||||
|
|
||||||
// create a datasync client
|
|
||||||
return DataSyncClient.builder().region(Region.of(awsRegion)).credentialsProvider(awsCredentialsProvider)
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Boolean createDatasyncTask(DatasyncParameters parameters) {
|
public Boolean createDatasyncTask(DatasyncParameters parameters) {
|
||||||
|
|
|
||||||
|
|
@ -25,27 +25,23 @@
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<artifactId>dolphinscheduler-task-dms</artifactId>
|
<artifactId>dolphinscheduler-task-dms</artifactId>
|
||||||
<packaging>jar</packaging>
|
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-spi</artifactId>
|
<artifactId>dolphinscheduler-spi</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-common</artifactId>
|
<artifactId>dolphinscheduler-common</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.amazonaws</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>aws-java-sdk-dms</artifactId>
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
|
||||||
|
|
@ -17,9 +17,9 @@
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.plugin.task.dms;
|
package org.apache.dolphinscheduler.plugin.task.dms;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.AWSDatabaseMigrationServiceClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
|
|
||||||
|
|
@ -29,6 +29,7 @@ import java.nio.charset.StandardCharsets;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
@ -37,11 +38,7 @@ import lombok.NoArgsConstructor;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
|
||||||
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService;
|
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService;
|
||||||
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder;
|
|
||||||
import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskRequest;
|
import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskRequest;
|
||||||
import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskResult;
|
import com.amazonaws.services.databasemigrationservice.model.CreateReplicationTaskResult;
|
||||||
import com.amazonaws.services.databasemigrationservice.model.DeleteReplicationTaskRequest;
|
import com.amazonaws.services.databasemigrationservice.model.DeleteReplicationTaskRequest;
|
||||||
|
|
@ -87,17 +84,8 @@ public class DmsHook {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static AWSDatabaseMigrationService createClient() {
|
public static AWSDatabaseMigrationService createClient() {
|
||||||
final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID);
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.dms.", "");
|
||||||
final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY);
|
return AWSDatabaseMigrationServiceClientFactory.createAWSDatabaseMigrationServiceClient(awsProperties);
|
||||||
final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION);
|
|
||||||
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey);
|
|
||||||
final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials);
|
|
||||||
|
|
||||||
// create a DMS client
|
|
||||||
return AWSDatabaseMigrationServiceClientBuilder.standard()
|
|
||||||
.withCredentials(awsCredentialsProvider)
|
|
||||||
.withRegion(awsRegion)
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Boolean createReplicationTask() throws Exception {
|
public Boolean createReplicationTask() throws Exception {
|
||||||
|
|
|
||||||
|
|
@ -28,19 +28,20 @@
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-spi</artifactId>
|
<artifactId>dolphinscheduler-spi</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.amazonaws</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>aws-java-sdk-emr</artifactId>
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
|
||||||
|
|
@ -22,22 +22,19 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN
|
||||||
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
||||||
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.AmazonElasticMapReduceClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
|
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
|
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
import java.util.TimeZone;
|
import java.util.TimeZone;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
|
||||||
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
|
||||||
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
||||||
|
|
||||||
|
|
@ -94,22 +91,8 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask {
|
||||||
return emrParameters;
|
return emrParameters;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* create emr client from BasicAWSCredentials
|
|
||||||
*
|
|
||||||
* @return AmazonElasticMapReduce
|
|
||||||
*/
|
|
||||||
protected AmazonElasticMapReduce createEmrClient() {
|
protected AmazonElasticMapReduce createEmrClient() {
|
||||||
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.emr.", "");
|
||||||
final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID);
|
return AmazonElasticMapReduceClientFactory.createAmazonElasticMapReduceClient(awsProperties);
|
||||||
final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY);
|
|
||||||
final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION);
|
|
||||||
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey);
|
|
||||||
final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials);
|
|
||||||
// create an EMR client
|
|
||||||
return AmazonElasticMapReduceClientBuilder.standard()
|
|
||||||
.withCredentials(awsCredentialsProvider)
|
|
||||||
.withRegion(awsRegion)
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,26 +31,23 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-spi</artifactId>
|
<artifactId>dolphinscheduler-spi</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-task-api</artifactId>
|
<artifactId>dolphinscheduler-task-api</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-common</artifactId>
|
<artifactId>dolphinscheduler-common</artifactId>
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.amazonaws</groupId>
|
|
||||||
<artifactId>aws-java-sdk-sagemaker</artifactId>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.dolphinscheduler</groupId>
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
<artifactId>dolphinscheduler-datasource-all</artifactId>
|
<artifactId>dolphinscheduler-datasource-all</artifactId>
|
||||||
<version>${project.version}</version>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,9 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN
|
||||||
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
|
||||||
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
|
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.authentication.aws.AmazonSageMakerClientFactory;
|
||||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
|
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
|
||||||
import org.apache.dolphinscheduler.plugin.datasource.sagemaker.param.SagemakerConnectionParam;
|
import org.apache.dolphinscheduler.plugin.datasource.sagemaker.param.SagemakerConnectionParam;
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
|
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
|
||||||
|
|
@ -41,11 +43,7 @@ import java.util.Map;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
||||||
import com.amazonaws.auth.BasicAWSCredentials;
|
|
||||||
import com.amazonaws.services.sagemaker.AmazonSageMaker;
|
import com.amazonaws.services.sagemaker.AmazonSageMaker;
|
||||||
import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder;
|
|
||||||
import com.amazonaws.services.sagemaker.model.StartPipelineExecutionRequest;
|
import com.amazonaws.services.sagemaker.model.StartPipelineExecutionRequest;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
||||||
|
|
@ -186,16 +184,8 @@ public class SagemakerTask extends AbstractRemoteTask {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected AmazonSageMaker createClient() {
|
protected AmazonSageMaker createClient() {
|
||||||
final String awsAccessKeyId = parameters.getUsername();
|
Map<String, String> awsProperties = PropertyUtils.getByPrefix("aws.sagemaker.", "");
|
||||||
final String awsSecretAccessKey = parameters.getPassword();
|
return AmazonSageMakerClientFactory.createAmazonSageMakerClient(awsProperties);
|
||||||
final String awsRegion = parameters.getAwsRegion();
|
|
||||||
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey);
|
|
||||||
final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials);
|
|
||||||
// create a SageMaker client
|
|
||||||
return AmazonSageMakerClientBuilder.standard()
|
|
||||||
.withCredentials(awsCredentialsProvider)
|
|
||||||
.withRegion(awsRegion)
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,13 @@
|
||||||
</includes>
|
</includes>
|
||||||
<outputDirectory>conf</outputDirectory>
|
<outputDirectory>conf</outputDirectory>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*.yaml</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>conf</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
</fileSets>
|
</fileSets>
|
||||||
<dependencySets>
|
<dependencySets>
|
||||||
<dependencySet>
|
<dependencySet>
|
||||||
|
|
|
||||||
7
pom.xml
7
pom.xml
|
|
@ -57,6 +57,7 @@
|
||||||
<module>dolphinscheduler-storage-plugin</module>
|
<module>dolphinscheduler-storage-plugin</module>
|
||||||
<module>dolphinscheduler-extract</module>
|
<module>dolphinscheduler-extract</module>
|
||||||
<module>dolphinscheduler-dao-plugin</module>
|
<module>dolphinscheduler-dao-plugin</module>
|
||||||
|
<module>dolphinscheduler-authentication</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
|
|
@ -294,6 +295,12 @@
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.dolphinscheduler</groupId>
|
||||||
|
<artifactId>dolphinscheduler-aws-authentication</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.junit</groupId>
|
<groupId>org.junit</groupId>
|
||||||
<artifactId>junit-bom</artifactId>
|
<artifactId>junit-bom</artifactId>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue