!179 [I1TL9B] Update docker and k8s license headers

Merge pull request !179 from sandygao/I1TL9B_update_docker_license_header
This commit is contained in:
i-robot 2020-09-07 19:27:02 +08:00 committed by Gitee
commit 265ae56edd
16 changed files with 250 additions and 80 deletions

View File

@ -1,4 +1,4 @@
#
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -10,7 +10,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
ARG BASE_IMAGE=centos:centos7
FROM $BASE_IMAGE
LABEL maintainer="OpenLooKeng Community"

View File

@ -1,5 +1,18 @@
#!/bin/bash
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeuo pipefail
CONFIG_PATH='/usr/lib/hetu/etc'

View File

@ -1,5 +1,18 @@
#!/bin/bash
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Read Hetu port number from config file. Default to 8080.
HETU_PORT=
CONFIG_FILE='/usr/lib/hetu/etc/config.properties'

View File

@ -1,4 +1,18 @@
#!/usr/bin/env bash
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
CUSTOM_TAG=
BASE_IMAGE="centos:centos7"
TAG=$(date +%s)
@ -33,6 +47,7 @@ if [[ $IS_GIT_FOLDER = true ]] && [[ -z "$CUSTOM_TAG" ]]; then
echo "Changes not committed, couldn't retrive a unique HASH for tagging"
exit
else
# By default, tag the image using the latest git commit hash, e.g. openlookeng:23555bf9
TAG=$(git rev-parse HEAD | cut -c 1-8)
fi
fi
@ -60,8 +75,6 @@ cp -R bin default ${WORK_DIR}/hetu-server-${HETU_VERSION}
cp ../presto-cli/target/hetu-cli-${HETU_VERSION}-executable.jar ${WORK_DIR}
# tag the image using both the Hetu version and the latest git commit hash
# i.e.) hetu:316-23555bf9
docker build ${WORK_DIR} -f Dockerfile --build-arg "OPENLK_VERSION=${HETU_VERSION}" --build-arg "BASE_IMAGE=${BASE_IMAGE}" -t "openlookeng:${TAG}"
rm -r ${WORK_DIR}

View File

@ -1,5 +1,18 @@
#!/usr/bin/env bash
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euxo pipefail
CONTAINER_ID=

View File

@ -1 +1,14 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
connector.name=jmx

View File

@ -1 +1,14 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
connector.name=memory

View File

@ -1,2 +1,15 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
connector.name=tpcds
tpcds.splits-per-node=4

View File

@ -1,2 +1,15 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
connector.name=tpch
tpch.splits-per-node=4

View File

@ -1,3 +1,16 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#single node install config
coordinator=true
node-scheduler.include-coordinator=true

View File

@ -1,2 +1,15 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Enable verbose logging from Hetu
#io.hetu.core=DEBUG

View File

@ -1,3 +1,16 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
node.environment=default
node.id=
node.data-dir=/var/lib/hetu/data

View File

@ -1,40 +1,38 @@
Assuming a Hetu docker image already exists, e.g. one built by `/hetu-core/docker/build-local.sh`.
Assuming an openLooKeng docker image already exists, e.g. one built by `/hetu-core/docker/build-local.sh`.
Update the [deployment.yaml](./deployment.yaml) file and replace _all_ occurrences of `{{ imageTag }}` with the tag of Hetu docker image.
Update the [deployment.yaml](./deployment.yaml) file and replace _all_ occurrences of `{{ imageTag }}` with the tag of the docker image.
### Deploy a Hetu cluster
### Deploy an openLooKeng cluster
```
docker stack deploy -f deployment.yaml <stack_name>
```
The cluster consists of 1 coordinator and 2 worker containers in the "hetu1" network.
The cluster consists of 1 coordinator and 2 worker containers in the "openlk1" network.
### Delete the Hetu cluster
### Delete the openLooKeng cluster
```
docker stack rm <stack_name>
```
### Additional configurations
- Need a "File manager" that helps docker swarm host the new configuration files among the swarm environment,
since docker swarm doesn't have that service built-in
- New configuration files must be accessible as soon as a container starts (need to make sure files are mounted as if
one is using docker swarm `config`)
- Need a "File manager" that helps docker swarm host the new configuration files among the swarm environment, since docker swarm doesn't have that service built-in
- New configuration files must be accessible as soon as a container starts (need to make sure files are mounted as if one is using docker swarm `config`)
To supply additional configuration files to be used by Hetu (e.g. `config.properties` or `log.properties`):
To supply additional configuration files to be used (e.g. `config.properties` or `log.properties`):
- Store these filed in a folder (this folder should only contain these configuration files)
- Store these files in a folder (this folder should only contain these configuration files)
- Mount the folder of new configuration files to each container (do it in the way required by your "File Manager")
- Inform Hetu containers where to look for the mounted folder using flag `-configDir` in the `command` list
- Delete and redeploy Hetu cluster (the cluster must be deleted first for configuration changes to take place)
- Inform openLooKeng containers where to look for the mounted folder using flag `-configDir` in the `command` list
- Delete and redeploy openLooKeng cluster (the cluster must be deleted first for configuration changes to take place)
### Additional catalogs
To connect to data sources through additional catalog files:
- Place these files inside a `catalog` folder under the configuration file folder above
- Inform Hetu containers where to look for the mounted folder using flag `-configDir` in the `command` list
- Delete and redeploy Hetu cluster
- Inform openLooKeng containers where to look for the mounted folder using flag `-configDir` in the `command` list
- Delete and redeploy openLooKeng cluster
If there are other files that are needed by the new catalogs:
- Store them in a different folder (e.g. `etc`) under the above configuration folder (that is, parallel to `catalog`)

View File

@ -1,16 +1,29 @@
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Assuming this machine has a local copy of the following in the same directory as the docker-compose.yml
# - A hdfs_configs directory containing default Hive connector configuration files
version: "3.7"
services:
# Hetu Engine - coordinator node
# openLooKeng Engine - coordinator node
coordinator:
image: hetu:{{ imageTag }}
image: openlookeng:{{ imageTag }}
ports:
- "8080:8080"
# mount config files for hive connector
networks:
- hetu1
- openlk1
# configs:
# - source: hive_prop
# target: /usr/lib/hetu/etc/catalog/hive.properties
@ -18,15 +31,15 @@ services:
# target: /opt/hdfs_configs/core-site.xml
# - source: hive_hdfs_site
# target: /opt/hdfs_configs/hdfs-site.xml
command: ["-t","coordinator","-nodeEnv","hetu1","-includeCoordinator","false"]
command: ["-t","coordinator","-nodeEnv","openlk1","-includeCoordinator","false"]
# coordinator1:
# image: hetu:{{ imageTag }}
# image: openlookeng:{{ imageTag }}
# ports:
# - "8081:8080"
# # mount config files for hive connector
# networks:
# - hetu2
# - openlk2
## configs:
## - source: hive_prop
## target: /usr/lib/hetu/etc/catalog/hive.properties
@ -34,16 +47,16 @@ services:
## target: /opt/hdfs_configs/core-site.xml
## - source: hive_hdfs_site
## target: /opt/hdfs_configs/hdfs-site.xml
# command: ["-t","coordinator","-nodeEnv","hetu2","-includeCoordinator","false"]
# command: ["-t","coordinator","-nodeEnv","openlk2","-includeCoordinator","false"]
# Hetu Engine - worker node
# openLooKeng Engine - worker node
worker:
image: hetu:{{ imageTag }}
image: openlookeng:{{ imageTag }}
depends_on:
- coordinator
# mount config files for hive connector
networks:
- hetu1
- openlk1
# configs:
# - source: hive_prop
# target: /usr/lib/hetu/etc/catalog/hive.properties
@ -54,15 +67,15 @@ services:
deploy:
replicas: 2
# -discoveryURI need to be full path; hostname must align with the coordinator service node
command: ["-t", "worker","-nodeEnv","hetu1","-discoveryURI","http://coordinator:8080"]
command: ["-t", "worker","-nodeEnv","openlk1","-discoveryURI","http://coordinator:8080"]
# worker1:
# image: hetu:{{ imageTag }}
# image: openlookeng:{{ imageTag }}
# depends_on:
# - coordinator1
# # mount config files for hive connector
# networks:
# - hetu2
# - openlk2
## configs:
## - source: hive_prop
## target: /usr/lib/hetu/etc/catalog/hive.properties
@ -73,8 +86,7 @@ services:
# deploy:
# replicas: 2
# # -discoveryURI need to be full path; hostname must align with the coordinator service node
# command: ["-t", "worker","-nodeEnv","hetu2","-discoveryURI","http://coordinator1:8080"]
# command: ["-t", "worker","-nodeEnv","openlk2","-discoveryURI","http://coordinator1:8080"]
# # Hive
# # must be named as hadoop-master
@ -84,9 +96,8 @@ services:
# ports:
# - "9083:9083"
# networks:
# - hetu1
# - hetu2
# - openlk1
# - openlk2
#configs:
# hive_prop:
@ -97,9 +108,9 @@ services:
# file: ${PWD}/hdfs_configs/hdfs-site.xml
networks:
hetu1:
openlk1:
driver: overlay
attachable: true
# hetu2:
# openlk2:
# driver: overlay
# attachable: true

View File

@ -1,16 +1,16 @@
Assuming a Hetu docker image already exists, e.g. one built by `/hetu-core/docker/build-local.sh`.
Assuming an openLooKeng docker image already exists, e.g. one built by `/hetu-core/docker/build-local.sh`.
Update the [deployment.yaml](./deployment.yaml) file and replace _both_ occurrences of `{{ imageTag }}` with the tag of Hetu docker image.
Update the [deployment.yaml](./deployment.yaml) file and replace _all_ occurrences of `{{ imageTag }}` with the tag of the docker image.
### Deploy a Hetu cluster
### Deploy an openLooKeng cluster
```
kubectl create -f deployment.yaml
```
The cluster consists of 1 coordinator and 2 worker pods in the "hetu" namespace.
The cluster consists of 1 coordinator and 2 worker pods in the "openlk" namespace.
### Delete the Hetu cluster
### Delete the openLooKeng cluster
```
kubectl delete -f deployment.yaml
@ -18,18 +18,23 @@ kubectl delete -f deployment.yaml
### Additional configurations
To supply additional configuration files to be used by Hetu (e.g. `config.properties` or `log.properties`):
- Store these filed in a folder (this folder should only contain these configuration files)
To supply additional configuration files to be used (e.g. `config.properties` or `log.properties`):
- Store these files in a folder (this folder should only contain these configuration files)
- Locate comments that read `TODO: update to point to additional configuration and catalog folder`
- Update the `path` value to poin to the above folder
- Delete and redeploy Hetu cluster (the cluster must be deleted first for configuration changes to take place)
- Delete and redeploy openLooKeng cluster (the cluster must be deleted first for configuration changes to take place)
Notes:
- The above uses `hostPath` to supply configuration files from a mounted local drive. This only works when deploying to a _local_ Kubernetes cluster. To deploy to a remote Kubernetes cluster, configuration files may need to be supplied as Kubernetes `configMaps`.
- Ensure configuration files and subfolders have appropriate permissions to allow non-root users to read.
### Additional catalogs
To connect to data sources through additional catalog files:
- Place these files inside a `catalog` folder under the configuration file folder above
- Ensure `path` points to the configuration folder
- Delete and redeploy Hetu cluster
- Delete and redeploy openLooKeng cluster
If there are other files that are needed by the new catalogs:
- Store them in a different folder (e.g. `etc`) under the above configuration folder (that is, parallel to `catalog`)
@ -37,7 +42,7 @@ If there are other files that are needed by the new catalogs:
### Enable graceful shutdown
When a Hetu pod is shutdown, by default it's terminated immediately. To enable graceful shutdown, where the pod waits for all its active work to finish first:
When an openLooKeng pod is shutdown, by default it's terminated immediately. To enable graceful shutdown, where the pod waits for all its active work to finish first:
- Locate comments that read `# TODO: enable if graceful shutdown is needed`
- Uncomment lines following the above comment
- Redeploy with
@ -46,7 +51,7 @@ When a Hetu pod is shutdown, by default it's terminated immediately. To enable g
kubectl apply -f deployment.yaml
```
With this change, it'll take at least a few minutes to kill any Hetu pod, because the graceful shutdown process needs to ensure no work is affected by shutting down the node. To shorten the wait period, the following line can be added to `config.properties`. See steps above about how to use custom configuration files.
With this change, it'll take at least a few minutes to kill any openLooKeng pod, because the graceful shutdown process needs to ensure no work is affected by shutting down the node. To shorten the wait period, the following line can be added to `config.properties`. See steps above about how to use custom configuration files.
```
shutdown.grace-period=10s

View File

@ -1,17 +1,30 @@
#### Hetu namespace ####
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#### openlk namespace ####
---
apiVersion: v1
kind: Namespace
metadata:
name: hetu
name: openlk
#### Hetu service ####
#### openLooKeng service ####
---
apiVersion: v1
kind: Service
metadata:
name: hetu
namespace: hetu
name: openlk
namespace: openlk
spec:
type: NodePort
ports:
@ -19,20 +32,20 @@ spec:
targetPort: 8080
nodePort: 30321
selector:
app: hetu-coordinator
app: openlk-coordinator
#### coordinator ####
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hetu-coordinator
namespace: hetu
name: openlk-coordinator
namespace: openlk
spec:
replicas: 1
selector:
matchLabels:
app: hetu-coordinator
app: openlk-coordinator
strategy:
type: RollingUpdate
rollingUpdate:
@ -41,16 +54,16 @@ spec:
template:
metadata:
labels:
app: hetu-coordinator
app: openlk-coordinator
spec:
# TODO: enable if graceful shutdown is needed
# terminationGracePeriodSeconds: 3600
containers:
- name: hetu-coordinator
image: hetu:{{ imageTag }}
- name: openlk-coordinator
image: openlookeng:{{ imageTag }}
imagePullPolicy: IfNotPresent
# Update "jvmXmx" value according to memory allocation below. Suggest 75~80% of total memory limit.
args: ["-t", "coordinator", "-configDir", "/custom-configs", "-jvmXmx", "1500M"]
# Update "jvmXmx" value according to memory allocation below. Suggest 60~70% of total memory limit.
args: ["-t", "coordinator", "-configDir", "/custom-configs", "-jvmXmx", "1300M"]
resources:
requests:
cpu: 1
@ -71,20 +84,20 @@ spec:
- name: custom-configs-volume
hostPath:
# TODO: update to point to additional configuration and catalog folder
path: /hetu/hetu-core/hetu-samples/kubernetes/config-coordinator
path: /hetu-core/hetu-samples/kubernetes/config-coordinator
#### worker ####
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hetu-worker
namespace: hetu
name: openlk-worker
namespace: openlk
spec:
replicas: 2
selector:
matchLabels:
app: hetu-worker
app: openlk-worker
strategy:
type: RollingUpdate
rollingUpdate:
@ -93,16 +106,16 @@ spec:
template:
metadata:
labels:
app: hetu-worker
app: openlk-worker
spec:
# TODO: enable if graceful shutdown is needed
# terminationGracePeriodSeconds: 600
containers:
- name: hetu-worker
image: hetu:{{ imageTag }}
- name: openlk-worker
image: openlookeng:{{ imageTag }}
imagePullPolicy: IfNotPresent
# Update "jvmXmx" value according to memory allocation below. Suggest 75~80% of total memory limit.
args: ["-t", "worker", "-discoveryURI", "http://hetu:8080", "-configDir", "/custom-configs", "-jvmXmx", "1500M"]
# Update "jvmXmx" value according to memory allocation below. Suggest 60~70% of total memory limit.
args: ["-t", "worker", "-discoveryURI", "http://openlk:8080", "-configDir", "/custom-configs", "-jvmXmx", "1300M"]
resources:
requests:
cpu: 1
@ -123,7 +136,7 @@ spec:
- name: custom-configs-volume
hostPath:
# TODO: update to point to additional configuration and catalog folder
path: /hetu/hetu-core/hetu-samples/kubernetes/config-worker
path: /hetu-core/hetu-samples/kubernetes/config-worker
# TODO: enable for coordinator auto-scaling
#### HPA for coordinator ####
@ -131,13 +144,13 @@ spec:
# apiVersion: autoscaling/v2beta2
# kind: HorizontalPodAutoscaler
# metadata:
# name: hetu-coordinator-scaler
# namespace: hetu
# name: openlk-coordinator-scaler
# namespace: openlk
# spec:
# scaleTargetRef:
# apiVersion: apps/v1
# kind: Deployment
# name: hetu-coordinator
# name: openlk-coordinator
# minReplicas: 1
# maxReplicas: 2
# metrics:
@ -158,13 +171,13 @@ spec:
# apiVersion: autoscaling/v2beta2
# kind: HorizontalPodAutoscaler
# metadata:
# name: hetu-worker-scaler
# namespace: hetu
# name: openlk-worker-scaler
# namespace: openlk
# spec:
# scaleTargetRef:
# apiVersion: apps/v1
# kind: Deployment
# name: hetu-worker
# name: openlk-worker
# minReplicas: 1
# maxReplicas: 5
# metrics: