natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Improvement][Etcd] Support SSL In Etcd And Enhance Etcd In Helm (#13924)

This commit is contained in:
旺阳 2023-06-14 13:29:50 +08:00 committed by GitHub
parent 465e7ae6ee
commit 1c0dfbb044
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 150 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
View File

@ -212,15 +212,36 @@ Create a registry environment variables.
- name: REGISTRY_TYPE
{{- if .Values.zookeeper.enabled }}
value: "zookeeper"
{{- else if .Values.etcd.enabled }}
value: "etcd"
{{- else }}
value: {{ .Values.externalRegistry.registryPluginName }}
{{- end }}
{{- if .Values.etcd.enabled }}
- name: REGISTRY_ENDPOINTS
value: {{ .Values.etcd.endpoints }}
- name: REGISTRY_NAMESPACE
value: {{ .Values.etcd.namespace }}
- name: REGISTRY_USER
value: {{ .Values.etcd.user }}
- name: REGISTRY_PASSWORD
value: {{ .Values.etcd.passWord }}
- name: REGISTRY_AUTHORITY
value: {{ .Values.etcd.authority }}
- name: REGISTRY_CERT_FILE
value: {{ .Values.etcd.ssl.certFile }}
- name: REGISTRY_KEY_CERT_CHAIN_FILE
value: {{ .Values.etcd.ssl.keyCertChainFile }}
- name: REGISTRY_KEY_FILE
value: {{ .Values.etcd.ssl.keyFile }}
{{- else }}
- name: REGISTRY_ZOOKEEPER_CONNECT_STRING
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalRegistry.registryServers }}
{{- end }}
{{- end }}
{{- end -}}
{{/*
@ -264,3 +285,31 @@ Create a fsFileResourcePersistence volumeMount.
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end -}}
{{- end -}}
{{/*
Create a etcd ssl volume.
*/}}
{{- define "dolphinscheduler.etcd.ssl.volume" -}}
{{- if .Values.etcd.ssl.enabled -}}
- name: etcd-ssl
secret:
secretName: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
{{- end -}}
{{- end -}}
{{/*
Create a etcd ssl volumeMount.
*/}}
{{- define "dolphinscheduler.etcd.ssl.volumeMount" -}}
{{- if .Values.etcd.ssl.enabled -}}
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.certFile }}
name: etcd-ssl
subPath: cert-file
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyCertChainFile }}
name: etcd-ssl
subPath: key-cert-chain-file
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyFile }}
name: etcd-ssl
subPath: key-file
{{- end -}}
{{- end -}}

2
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

@ -111,6 +111,7 @@ spec:
subPath: common_properties
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.api.persistentVolumeClaim.enabled }}
@ -124,3 +125,4 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}

30
deploy/kubernetes/dolphinscheduler/templates/secret-external-etcd-ssl.yaml Normal file
View File

@ -0,0 +1,30 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.etcd.ssl.enabled }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
type: Opaque
data:
cert-file: {{ .Files.Get .Values.etcd.ssl.certFile | b64enc | quote }}
key-cert-chain-file: {{ .Files.Get .Values.etcd.ssl.keyCertChainFile | b64enc | quote }}
key-file: {{ .Files.Get .Values.etcd.ssl.keyFile | b64enc | quote }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

@ -105,6 +105,7 @@ spec:
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common_properties
{{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-master
{{- if .Values.master.persistentVolumeClaim.enabled }}
@ -117,6 +118,7 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
{{- if .Values.master.persistentVolumeClaim.enabled }}
volumeClaimTemplates:
- metadata:

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

@ -108,6 +108,7 @@ spec:
subPath: common_properties
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
volumes:
{{- if .Values.worker.persistentVolumeClaim.enabled }}
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data
@ -135,6 +136,7 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
{{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates:
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}

14
deploy/kubernetes/dolphinscheduler/values.yaml
View File

@ -94,6 +94,20 @@ zookeeper:
size: "20Gi"
storageClass: "-"
etcd:
enabled: false
endpoints: ""
namespace: "dolphinscheduler"
user: ""
passWord: ""
authority: ""
# Please create a new folder: deploy/kubernetes/dolphinscheduler/etcd-certs
ssl:
enabled: false
certFile: "etcd-certs/ca.crt"
keyCertChainFile: "etcd-certs/client.crt"
keyFile: "etcd-certs/client.pem"
## If exists external registry and set zookeeper.enable value to false, the external registry will be used.
externalRegistry:
registryPluginName: "zookeeper"

8
dolphinscheduler-master/pom.xml
View File

@ -154,6 +154,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -250,6 +254,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

26
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md
View File

@ -12,17 +12,31 @@ registry:
endpoints: "http://etcd0:2379, http://etcd1:2379, http://etcd2:2379"
# The options below have default values
namespace: dolphinscheduler
connectionTimeout: 9s
connection-timeout: 9s
# The unit is milliseconds
retryDelay: 60ms
retryMaxDelay: 300ms
retryMaxDuration: 1500ms
# The following options are set according to personal needs
retry-delay: 60ms
retry-max-delay: 300ms
retry-max-duration: 1500ms
# The following ssl options are set according to personal needs
cert-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/ca.crt"
key-cert-chain-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.crt"
key-file: "deploy/kubernetes/dolphinscheduler/etcd-certs/client.pem"
# The following auth options are set according to personal needs
user: ""
password: ""
authority: ""
loadBalancerPolicy: ""
load-balancer-policy: ""
```
If your etcd server has configured with ssl, about certification files you can see [here](https://github.com/etcd-io/jetcd/blob/main/docs/SslConfig.md) for how to convert.
> If you need ssl certification, you need to make sure your jdk version is newer than Java 8u252 (April 2020), jdk11 works well too.
>
> By the way, the jdk version in docker images `FROM eclipse-temurin:8-jre` now is 8u362 works well, don't need change.
>
> Because after version 8u252 has native support for ALPN. Detail you can see:
>
> https://github.com/grpc/grpc-java/issues/5369#issuecomment-751885384
After do this config, you can start your DolphinScheduler cluster, your cluster will use etcd as registry center to
store server metadata.

19
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java
View File

@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.registry.api.Registry;
import org.apache.dolphinscheduler.registry.api.RegistryException;
import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
@ -36,6 +37,7 @@ import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.net.ssl.SSLException;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@ -60,6 +62,8 @@ import io.etcd.jetcd.options.PutOption;
import io.etcd.jetcd.options.WatchOption;
import io.etcd.jetcd.support.Observers;
import io.etcd.jetcd.watch.WatchEvent;
import io.grpc.netty.GrpcSslContexts;
import io.netty.handler.ssl.SslContext;
/**
* This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to
@ -80,7 +84,7 @@ public class EtcdRegistry implements Registry {
private final Map<String, Watch.Watcher> watcherMap = new ConcurrentHashMap<>();
private static final long TIME_TO_LIVE_SECONDS = 30L;
public EtcdRegistry(EtcdRegistryProperties registryProperties) {
public EtcdRegistry(EtcdRegistryProperties registryProperties) throws SSLException {
ClientBuilder clientBuilder = Client.builder()
.endpoints(Util.toURIs(Splitter.on(",").trimResults().splitToList(registryProperties.getEndpoints())))
.namespace(byteSequence(registryProperties.getNamespace()))
@ -100,6 +104,19 @@ public class EtcdRegistry implements Registry {
if (StringUtils.hasLength(registryProperties.getAuthority())) {
clientBuilder.authority(registryProperties.getAuthority());
}
if (StringUtils.hasLength(registryProperties.getCertFile())
&& StringUtils.hasLength(registryProperties.getKeyCertChainFile())
&& StringUtils.hasLength(registryProperties.getKeyFile())) {
String userDir = System.getProperty("user.dir") + "/";
File certFile = new File(userDir + registryProperties.getCertFile());
File keyCertChainFile = new File(userDir + registryProperties.getKeyCertChainFile());
File keyFile = new File(userDir + registryProperties.getKeyFile());
SslContext context = GrpcSslContexts.forClient()
.trustManager(certFile)
.keyManager(keyCertChainFile, keyFile)
.build();
clientBuilder.sslContext(context);
}
client = clientBuilder.build();
log.info("Started Etcd Registry...");
etcdConnectionStateListener = new EtcdConnectionStateListener(client);

5
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java
View File

@ -47,4 +47,9 @@ public class EtcdRegistryProperties {
// loadBalancerPolicy
private String loadBalancerPolicy;
// ssl
private String certFile;
private String keyCertChainFile;
private String keyFile;
}