natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Feature][Helm] Supports deploying only partial components (#14985) 

* [Feature][Helm] Supports deploying only partial components

* chore: add doc

* chore: miss some doc
This commit is contained in:
Gallardot 2023-10-10 20:58:49 +08:00 committed by GitHub
parent 049c1511fa
commit 4fe41ac57f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 88 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.alert.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
@ -125,3 +126,4 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.api.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
@ -134,3 +135,4 @@ spec:
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}
{{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/ingress.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.ingress.enabled }}
{{- if and .Values.api.enabled .Values.ingress.enabled }}
{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
apiVersion: networking.k8s.io/v1
{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" }}

2
deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-schema-initializer.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.api.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
@ -52,3 +53,4 @@ spec:
envFrom:
- configMapRef:
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/keda-autoscaler-worker.yaml
View File

@ -18,7 +18,7 @@
################################
## DolphinScheduler Worker KEDA Scaler
#################################
{{- if and .Values.worker.keda.enabled }}
{{- if and .Values.worker.keda.enabled .Values.worker.enabled }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:

2
deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.alert.persistentVolumeClaim.enabled }}
{{- if and .Values.alert.enabled .Values.alert.persistentVolumeClaim.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:

2
deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.api.persistentVolumeClaim.enabled }}
{{- if and .Values.api.enabled .Values.api.persistentVolumeClaim.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.master.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
@ -143,3 +144,4 @@ spec:
requests:
storage: {{ .Values.master.persistentVolumeClaim.storage | quote }}
{{- end }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.worker.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
@ -179,3 +180,4 @@ spec:
storage: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }}
{{- end }}
{{- end }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.alert.enabled }}
apiVersion: v1
kind: Service
metadata:
@ -67,3 +68,4 @@ spec:
path: {{ .Values.alert.service.serviceMonitor.path }}
interval: {{ .Values.alert.service.serviceMonitor.interval }}
{{- end }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.api.enabled }}
apiVersion: v1
kind: Service
metadata:
@ -86,3 +87,4 @@ spec:
path: {{ .Values.api.service.serviceMonitor.path }}
interval: {{ .Values.api.service.serviceMonitor.interval }}
{{- end }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.master.enabled }}
apiVersion: v1
kind: Service
metadata:
@ -67,3 +68,4 @@ spec:
path: {{ .Values.master.service.serviceMonitor.path }}
interval: {{ .Values.master.service.serviceMonitor.interval }}
{{- end }}
{{- end }}

2
deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.worker.enabled }}
apiVersion: v1
kind: Service
metadata:
@ -67,3 +68,4 @@ spec:
path: {{ .Values.worker.service.serviceMonitor.path }}
interval: {{ .Values.worker.service.serviceMonitor.interval }}
{{- end }}
{{- end }}

8
deploy/kubernetes/dolphinscheduler/values.yaml
View File

@ -300,6 +300,8 @@ common:
storage: "20Gi"
master:
## Enable or disable the Master component
enabled: true
## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
podManagementPolicy: "Parallel"
## Replicas is the desired number of replicas of the given Template.
@ -387,6 +389,8 @@ master:
annotations: {}
worker:
## Enable or disable the Worker component
enabled: true
## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
podManagementPolicy: "Parallel"
## Replicas is the desired number of replicas of the given Template.
@ -504,6 +508,8 @@ worker:
alert:
## Enable or disable the Alert-Server component
enabled: true
## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
replicas: 1
## The deployment strategy to use to replace existing pods with new ones.
@ -581,6 +587,8 @@ alert:
api:
## Enable or disable the API-Server component
enabled: true
## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
replicas: "1"
## The deployment strategy to use to replace existing pods with new ones.

29
docs/docs/en/guide/installation/kubernetes.md
View File

@ -514,6 +514,31 @@ common:
SW_GRPC_LOG_SERVER_PORT: "11800"
```
### How to deploy specific components separately?
Modify the `api.enabled`, `alert.enabled`, `master.enabled`, or `worker.enabled` configuration items in the `values.yaml` file.
For example, if you need to deploy worker to both CPU and GPU servers in a cluster, and the worker uses different images, you can do the following:
```bash
# Install master, api-server, alert-server, and other default components, but do not install worker
helm install dolphinscheduler . --set worker.enabled=false
# Disable the installation of other components, only install worker, use the self-built CPU image, deploy to CPU servers with the `x86` label through nodeselector, and use zookeeper as the external registry center
helm install dolphinscheduler-cpu-worker . \
--set minio.enabled=false --set postgresql.enabled=false --set zookeeper.enabled=false \
--set master.enabled=false --set api.enabled=false --set alert.enabled=false \
--set worker.enabled=true --set image.tag=lastes-cpu --set worker.nodeSelector.cpu="x86" \
--set externalRegistry.registryPluginName=zookeeper --set externalRegistry.registryServers=dolphinscheduler-zookeeper:2181
# Disable the installation of other components, only install worker, use the self-built GPU image, deploy to GPU servers with the `a100` label through nodeselector, and use zookeeper as the external registry center
helm install dolphinscheduler-gpu-worker . \
--set minio.enabled=false --set postgresql.enabled=false --set zookeeper.enabled=false \
--set master.enabled=false --set api.enabled=false --set alert.enabled=false \
--set worker.enabled=true --set image.tag=lastes-cpu --set worker.nodeSelector.gpu="a100" \
--set externalRegistry.registryPluginName=zookeeper --set externalRegistry.registryServers=dolphinscheduler-zookeeper:2181
```
Please note that the above steps are for reference only, and specific operations need to be adjusted according to the actual situation.
## Appendix-Configuration
| Parameter | Description | Default |
@ -590,6 +615,7 @@ common:
| `common.fsFileResourcePersistence.storageClassName` | Resource persistent volume storage class, must support the access mode: ReadWriteMany | `-` |
| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `master.enabled` | Enable or disable the Master component | true |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.annotations` | The `annotations` for master server | `{}` |
@ -624,6 +650,7 @@ common:
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `worker.enabled` | Enable or disable the Worker component | true |
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `worker.annotations` | The `annotations` for worker server | `{}` |
@ -658,6 +685,7 @@ common:
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `alert.enabled` | Enable or disable the Alert-Server component | true |
| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
@ -699,6 +727,7 @@ common:
| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `api.enabled` | Enable or disable the API-Server component | true |
| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |

29
docs/docs/zh/guide/installation/kubernetes.md
View File

@ -513,6 +513,31 @@ common:
SW_GRPC_LOG_SERVER_PORT: "11800"
```
### 如何单独部署特定组件?
修改 `values.yaml` 文件中的 `api.enabled`, `alert.enabled` `master.enabled` 或 `worker.enabled` 配置项
例如，在一个集群中需要同时将 worker 部署到 CPU 服务器和 GPU 服务器，并且 worker 使用不同的镜像，可以这样做:
```bash
# 安装 master、api-server、alert-server以及其他默认组件，但是不安装 worker
helm install dolphinscheduler . --set worker.enabled=false
# 禁用其他组件的安装，只安装 worker，使用自行建构建的 CPU镜像，通过 nodeselector部署到附带 x86标签的 CPU服务器，使用 zookeeper作为外部注册中心
helm install dolphinscheduler-cpu-worker . \
--set minio.enabled=false --set postgresql.enabled=false --set zookeeper.enabled=false \
--set master.enabled=false --set api.enabled=false --set alert.enabled=false \
--set worker.enabled=true --set image.tag=lastes-cpu --set worker.nodeSelector.cpu="x86" \
--set externalRegistry.registryPluginName=zookeeper --set externalRegistry.registryServers=dolphinscheduler-zookeeper:2181
# 禁用其他组件的安装，只安装 worker，使用自行建构建的 GPU 镜像，通过 nodeselector部署到附带 a100标签的 gpu服务器，使用zookeeper作为外部注册中心
helm install dolphinscheduler-gpu-worker . \
--set minio.enabled=false --set postgresql.enabled=false --set zookeeper.enabled=false \
--set master.enabled=false --set api.enabled=false --set alert.enabled=false \
--set worker.enabled=true --set image.tag=lastes-cpu --set worker.nodeSelector.gpu="a100" \
--set externalRegistry.registryPluginName=zookeeper --set externalRegistry.registryServers=dolphinscheduler-zookeeper:2181
```
请注意，以上步骤仅供参考，具体操作需要根据实际情况进行调整。
## 附录-配置
| Parameter | Description | Default |
@ -589,6 +614,7 @@ common:
| `common.fsFileResourcePersistence.storageClassName` | Resource persistent volume storage class, must support the access mode: ReadWriteMany | `-` |
| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `master.enabled` | Enable or disable the Master component | true |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.annotations` | The `annotations` for master server | `{}` |
@ -623,6 +649,7 @@ common:
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `worker.enabled` | Enable or disable the Worker component | true |
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `worker.annotations` | The `annotations` for worker server | `{}` |
@ -657,6 +684,7 @@ common:
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `alert.enabled` | Enable or disable the Alert-Server component | true |
| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
@ -698,6 +726,7 @@ common:
| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` size | `20Gi` |
| | | |
| `api.enabled` | Enable or disable the API-Server component | true |
| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |