sonar setup test
This commit is contained in:
Vignesh Jayavel
parent
99d8276be7
commit
a235de595f
|
|
@ -1,45 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
/.github/ @SbloodyS
|
||||
/deploy/ @caishunfeng
|
||||
/dolphinscheduler-alert/ @caishunfeng
|
||||
/dolphinscheduler-e2e/ @SbloodyS
|
||||
/dolphinscheduler-api-test/ @SbloodyS
|
||||
/dolphinscheduler-registry/ @caishunfeng @ruanwenjun
|
||||
/dolphinscheduler-api/ @caishunfeng @SbloodyS
|
||||
/dolphinscheduler-dao/ @caishunfeng @SbloodyS
|
||||
/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888
|
||||
/dolphinscheduler-common/ @caishunfeng
|
||||
/dolphinscheduler-standalone-server/ @caishunfeng
|
||||
/dolphinscheduler-datasource-plugin/ @caishunfeng
|
||||
/dolphinscheduler-dist/ @caishunfeng
|
||||
/dolphinscheduler-meter/ @caishunfeng @ruanwenjun @EricGao888
|
||||
/dolphinscheduler-scheduler-plugin/ @caishunfeng
|
||||
/dolphinscheduler-master/ @caishunfeng @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-service/ @caishunfeng
|
||||
/dolphinscheduler-extract/ @caishunfeng @ruanwenjun
|
||||
/dolphinscheduler-spi/ @caishunfeng
|
||||
/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong
|
||||
/dolphinscheduler-tools/ @caishunfeng @SbloodyS @EricGao888
|
||||
/script/ @caishunfeng @SbloodyS @EricGao888
|
||||
/dolphinscheduler-ui/ @songjianet @Amy0104
|
||||
/docs/ @EricGao888
|
||||
/licenses/ @EricGao888
|
||||
/images/ @EricGao888
|
||||
/style/ @caishunfeng
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: Bug report
|
||||
title: "[Bug] [Module Name] Bug title "
|
||||
description: Problems and issues with code of Apache Dolphinscheduler
|
||||
labels: [ "bug", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
Please make sure what you are reporting is indeed a bug with reproducible steps, if you want to ask questions
|
||||
or share ideas, you can head to our
|
||||
[Discussions](https://github.com/apache/dolphinscheduler/discussions) tab, you can also
|
||||
[join our slack](https://s.apache.org/dolphinscheduler-slack)
|
||||
and send your question to channel `#troubleshooting`
|
||||
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue)
|
||||
first to see whether the same issue was reported already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found
|
||||
no similar issues.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: What happened
|
||||
description: Describe what happened.
|
||||
placeholder: >
|
||||
Please provide the context in which the problem occurred and explain what happened
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: What you expected to happen
|
||||
description: What do you think went wrong?
|
||||
placeholder: >
|
||||
Please explain why you think the behaviour is erroneous. It is extremely helpful if you copy and paste
|
||||
the fragment of logs showing the exact error messages or wrong behaviour and screenshots for
|
||||
UI problems. You can include files by dragging and dropping them here.
|
||||
|
||||
**NOTE**: please copy and paste texts instead of taking screenshots of them for easy future search.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: How to reproduce
|
||||
description: >
|
||||
What should we do to reproduce the problem? If you are not able to provide a reproducible case,
|
||||
please open a [Discussion](https://github.com/apache/dolphinscheduler/discussions) instead.
|
||||
placeholder: >
|
||||
Please make sure you provide a reproducible step-by-step case of how to reproduce the problem
|
||||
as minimally and precisely as possible. Keep in mind we do not have access to your deployment.
|
||||
Remember that non-reproducible issues will be closed! Opening a discussion is recommended as a
|
||||
first step.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Anything else
|
||||
description: Anything else we need to know?
|
||||
placeholder: >
|
||||
How often does this problem occur? (Once? Every time? Only when certain conditions are met?)
|
||||
Any relevant logs to include? Put them here inside fenced
|
||||
``` ``` blocks or inside a collapsable details tag if it's too long:
|
||||
<details><summary>x.log</summary> lots of stuff </details>
|
||||
|
||||
- type: dropdown
|
||||
id: version
|
||||
attributes:
|
||||
label: Version
|
||||
description: >
|
||||
Which version of Apache DolphinScheduler are you running? We only accept bugs report from the LTS projects.
|
||||
options:
|
||||
- dev
|
||||
- 3.1.x
|
||||
- 3.2.x
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Are you willing to submit PR?
|
||||
description: >
|
||||
This is absolutely not required, but we are happy to guide you in the contribution process
|
||||
especially if you already have a good understanding of how to implement the fix.
|
||||
Dolphinscheduler is a totally community-driven project and we love to bring new contributors in.
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: >
|
||||
I agree to follow this project's
|
||||
[Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Ask a question or get support
|
||||
url: https://github.com/apache/dolphinscheduler/discussions/
|
||||
about: Ask a question or request support for using Apache DolphinScheduler
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Documentation Related
|
||||
description: Suggest an improvement or report a bug for this project's Documentation
|
||||
title: "[Doc][Module Name] Documentation bug or improvement"
|
||||
labels: [ "document", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first
|
||||
to see whether the same feature was requested already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no
|
||||
similar feature requirement.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Description
|
||||
description: A short description why your find in our document.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Documentation Links
|
||||
description: Copy and paste one or more links of this documentation issue.
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Are you willing to submit a PR?
|
||||
description: >
|
||||
This is absolutely not required, but we are happy to guide you in the contribution process
|
||||
especially if you already have a good understanding of how to implement the improvement.
|
||||
DolphinScheduler is a totally community-driven project and we love to bring new contributors in.
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
name: DSIP
|
||||
description: Suggest an idea for this project
|
||||
title: "[DSIP-][Module Name] DSIP title"
|
||||
labels: [ "DSIP", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first
|
||||
to see whether the same DSIP was created already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no
|
||||
similar DSIP.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Motivation
|
||||
description: Why you want to do this change?
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Design Detail
|
||||
description: Your design.
|
||||
placeholder: >
|
||||
It's better to provide a detailed design, such as the design of the interface, the design of the database, etc.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Compatibility, Deprecation, and Migration Plan
|
||||
description: >
|
||||
If this feature is related to compatibility, deprecation, or migration, please describe it here.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Test Plan
|
||||
description: >
|
||||
How to test this improvement.
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
name: Feature request
|
||||
description: Suggest an idea for this project
|
||||
title: "[Feature][Module Name] Feature title"
|
||||
labels: [ "feature", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first
|
||||
to see whether the same feature was requested already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no
|
||||
similar feature requirement.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Description
|
||||
description: A short description of your feature
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Use case
|
||||
description: What do you want to happen?
|
||||
placeholder: >
|
||||
Rather than telling us how you might implement this feature, try to take a
|
||||
step back and describe what you are trying to achieve.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Related issues
|
||||
description: Is there currently another issue associated with this?
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Are you willing to submit a PR?
|
||||
description: >
|
||||
This is absolutely not required, but we are happy to guide you in the contribution process
|
||||
especially if you already have a good understanding of how to implement the feature.
|
||||
DolphinScheduler is a totally community-driven project and we love to bring new contributors in.
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
name: Improvement request
|
||||
description: Suggest an improvement for this project
|
||||
title: "[Improvement][Module Name] Improvement title"
|
||||
labels: [ "improvement", "Waiting for reply" ]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For better global communication, Please write in English.
|
||||
|
||||
If you feel the description in English is not clear, then you can append description in Chinese, thanks!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first
|
||||
to see whether the same feature was requested already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no
|
||||
similar feature requirement.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Description
|
||||
description: A short description why your want to do this improvement
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Are you willing to submit a PR?
|
||||
description: >
|
||||
This is absolutely not required, but we are happy to guide you in the contribution process
|
||||
especially if you already have a good understanding of how to implement the improvement.
|
||||
DolphinScheduler is a totally community-driven project and we love to bring new contributors in.
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: |
|
||||
The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: |
|
||||
I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
<!--Thanks very much for contributing to Apache DolphinScheduler, we are happy that you want to help us improve DolphinScheduler! -->
|
||||
|
||||
## Purpose of the pull request
|
||||
|
||||
<!--(For example: This pull request adds checkstyle plugin).-->
|
||||
|
||||
## Brief change log
|
||||
|
||||
<!--*(for example:)*
|
||||
- *Add maven-checkstyle-plugin to root pom.xml*
|
||||
-->
|
||||
|
||||
## Verify this pull request
|
||||
|
||||
<!--*(Please pick either of the following options)*-->
|
||||
|
||||
This pull request is code cleanup without any test coverage.
|
||||
|
||||
*(or)*
|
||||
|
||||
This pull request is already covered by existing tests, such as *(please describe tests)*.
|
||||
|
||||
(or)
|
||||
|
||||
This change added tests and can be verified as follows:
|
||||
|
||||
<!--*(example:)*
|
||||
- *Added dolphinscheduler-dao tests for end-to-end.*
|
||||
- *Added CronUtilsTest to verify the change.*
|
||||
- *Manually verified the change by testing locally.* -->
|
||||
|
||||
(or)
|
||||
|
||||
If your pull request contain incompatible change, you should also add it to `docs/docs/en/guide/upgrede/incompatible.md`
|
||||
|
|
@ -1 +0,0 @@
|
|||
Subproject commit 966bbbd2113de1f8008322fd1cc7674981a6b1ab
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
backend:
|
||||
- 'dolphinscheduler-alert/**/*'
|
||||
- 'dolphinscheduler-api/**/*'
|
||||
- 'dolphinscheduler-common/**/*'
|
||||
- 'dolphinscheduler-dao/**/*'
|
||||
- 'dolphinscheduler-data-quality/**/*'
|
||||
- 'dolphinscheduler-datasource-plugin/**/*'
|
||||
- 'dolphinscheduler-dist/**/*'
|
||||
- 'dolphinscheduler-master/**/*'
|
||||
- 'dolphinscheduler-registry/**/*'
|
||||
- 'dolphinscheduler-extract/**/*'
|
||||
- 'dolphinscheduler-scheduler-plugin/**/*'
|
||||
- 'dolphinscheduler-service/**/*'
|
||||
- 'dolphinscheduler-spi/**/*'
|
||||
- 'dolphinscheduler-standalone-server/**/*'
|
||||
- 'dolphinscheduler-task-plugin/**/*'
|
||||
- 'dolphinscheduler-tools/**/*'
|
||||
- 'dolphinscheduler-worker/**/*'
|
||||
- 'script/**/*'
|
||||
|
||||
document:
|
||||
- 'docs/**/*'
|
||||
|
||||
CI&CD:
|
||||
- any: ['.github/**/*']
|
||||
|
||||
docker:
|
||||
- any: ['.deploy/**/*']
|
||||
|
||||
UI:
|
||||
- any: ['dolphinscheduler-ui/**/*']
|
||||
|
||||
e2e:
|
||||
- any: ['dolphinscheduler-e2e/**/*']
|
||||
|
||||
test:
|
||||
- any: ['dolphinscheduler-api-test/**/*']
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
name: "Sanity Check"
|
||||
|
||||
description: |
|
||||
Action to perform some very basic lightweight checks, like code styles, license headers, etc.,
|
||||
and fail fast to avoid wasting resources running heavyweight checks, like unit tests, e2e tests.
|
||||
|
||||
inputs:
|
||||
token:
|
||||
description: 'The GitHub API token'
|
||||
required: false
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Check License Header
|
||||
uses: apache/skywalking-eyes/header@main
|
||||
|
||||
- shell: bash
|
||||
run: ./mvnw spotless:check
|
||||
|
|
@ -1 +0,0 @@
|
|||
Subproject commit 8c4d3b8ead41812bbe837dc5dfb8fa047cd2b548
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
---
|
||||
version: 2
|
||||
mergeable:
|
||||
# we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
|
||||
# otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
|
||||
# GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
|
||||
# which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
|
||||
- when: pull_request.opened, pull_request.reopened, pull_request.synchronize
|
||||
name: sync-sql-ddl
|
||||
validate:
|
||||
# Sql files must change synchronize
|
||||
- do: dependent
|
||||
files:
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
|
||||
message: 'Sql files not change synchronize'
|
||||
# Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
|
||||
fail:
|
||||
- do: comment
|
||||
payload:
|
||||
body: >
|
||||
:warning: This PR do not change database DDL synchronize.
|
||||
leave_old_comment: false
|
||||
- do: labels
|
||||
add: 'sql not sync'
|
||||
# Remove labels 'sql not sync' if pass
|
||||
pass:
|
||||
- do: labels
|
||||
delete: 'sql not sync'
|
||||
|
||||
- when: pull_request.*
|
||||
name: milestone-label-check
|
||||
validate:
|
||||
- do: milestone
|
||||
no_empty:
|
||||
enabled: false # Cannot be empty when true.
|
||||
message: 'Milestone is required and cannot be empty.'
|
||||
- do: label
|
||||
and:
|
||||
- must_include:
|
||||
regex: 'feature|bug|improvement|document|chore|revert'
|
||||
message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
|
||||
- must_include:
|
||||
regex: 'ready-to-merge'
|
||||
message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
name: API-Test
|
||||
|
||||
concurrency:
|
||||
group: api-test-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: API-Test-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: API-Test-Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
uses: ./.github/actions/sanity-check
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- name: Build Image
|
||||
run: |
|
||||
./mvnw -B clean install \
|
||||
-Dmaven.test.skip \
|
||||
-Dmaven.javadoc.skip \
|
||||
-Dspotless.skip=true \
|
||||
-Pdocker,release -Ddocker.tag=ci
|
||||
- name: Export Docker Images
|
||||
run: |
|
||||
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
|
||||
&& du -sh /tmp/standalone-image.tar
|
||||
- uses: actions/upload-artifact@v2
|
||||
name: Upload Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
path: /tmp/standalone-image.tar
|
||||
retention-days: 1
|
||||
api-test:
|
||||
name: ${{ matrix.case.name }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: Tenant
|
||||
class: org.apache.dolphinscheduler.api.test.cases.TenantAPITest
|
||||
- name: WorkerGroup
|
||||
class: org.apache.dolphinscheduler.api.test.cases.WorkerGroupAPITest
|
||||
- name: Project
|
||||
class: org.apache.dolphinscheduler.api.test.cases.ProjectAPITest
|
||||
- name: Workflow
|
||||
class: org.apache.dolphinscheduler.api.test.cases.ProcessDefinitionAPITest
|
||||
- name: Scheduler
|
||||
class: org.apache.dolphinscheduler.api.test.cases.SchedulerAPITest
|
||||
- name: Executor
|
||||
class: org.apache.dolphinscheduler.api.test.cases.ExecutorAPITest
|
||||
- name: ProcessInstance
|
||||
class: org.apache.dolphinscheduler.api.test.cases.ProcessInstanceAPITest
|
||||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
path: /tmp
|
||||
- name: Load Docker Images
|
||||
run: |
|
||||
docker load -i /tmp/standalone-image.tar
|
||||
- name: Run Test
|
||||
run: |
|
||||
./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dspotless.skip=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
name: recording-${{ matrix.case.name }}
|
||||
path: ${{ env.RECORDING_PATH }}
|
||||
retention-days: 1
|
||||
result:
|
||||
name: API-Test-Result
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ api-test, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip API Test!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.api-test.result }} != 'success' ]]; then
|
||||
echo "API test Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
@ -1,277 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: Backend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
paths:
|
||||
- '.github/workflows/backend.yml'
|
||||
- 'package.xml'
|
||||
- 'pom.xml'
|
||||
- 'dolphinscheduler-alert/**'
|
||||
- 'dolphinscheduler-api/**'
|
||||
- 'dolphinscheduler-common/**'
|
||||
- 'dolphinscheduler-dao/**'
|
||||
- 'dolphinscheduler-rpc/**'
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: backend-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Backend-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
db-schema: ${{ steps.filter.outputs.db-schema }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
db-schema:
|
||||
- 'dolphinscheduler-dao/src/main/resources/sql/**'
|
||||
build:
|
||||
name: Backend-Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
java: [ '8', '11' ]
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Set up JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'adopt'
|
||||
- name: Sanity Check
|
||||
uses: ./.github/actions/sanity-check
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- name: Build and Package on ${{ matrix.java }}
|
||||
run: |
|
||||
./mvnw -B clean install \
|
||||
-Prelease \
|
||||
-Dmaven.test.skip=true \
|
||||
-Dspotless.skip=true \
|
||||
-Dhttp.keepAlive=false \
|
||||
-Dmaven.wagon.http.pool=false \
|
||||
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
|
||||
- name: Check dependency license
|
||||
run: tools/dependencies/check-LICENSE.sh
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: ${{ matrix.java == '8' }}
|
||||
name: Upload Binary Package
|
||||
with:
|
||||
name: binary-package-${{ matrix.java }}
|
||||
path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
|
||||
retention-days: 1
|
||||
cluster-test:
|
||||
name: ${{ matrix.case.name }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: cluster-test-mysql-with-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-mysql-with-mysql-registry
|
||||
script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-zookeeper-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
|
||||
- name: cluster-test-postgresql-with-postgresql-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Binary Package
|
||||
with:
|
||||
# Only run cluster test on jdk8
|
||||
name: binary-package-8
|
||||
path: ./
|
||||
- name: Running cluster test
|
||||
run: |
|
||||
/bin/bash ${{ matrix.case.script }}
|
||||
schema-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
|
||||
timeout-minutes: 20
|
||||
needs: build
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:5.7
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: mysql
|
||||
MYSQL_DATABASE: dolphinscheduler_dev
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dolphinscheduler_dev
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
db: ["mysql", "postgresql"]
|
||||
version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"]
|
||||
steps:
|
||||
- name: Set up JDK 8
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 8
|
||||
distribution: 'adopt'
|
||||
- name: Install Atlas and Create Dir
|
||||
run: |
|
||||
mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
|
||||
curl -sSf https://atlasgo.sh | sh
|
||||
- name: Download Tarball
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: binary-package-8
|
||||
path: dolphinscheduler/dev
|
||||
- name: Set Env
|
||||
run: |
|
||||
VERSION=${{ matrix.version }}
|
||||
echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
|
||||
- name: Prepare
|
||||
run: |
|
||||
wget https://archive.apache.org/dist/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
|
||||
tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
|
||||
tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
|
||||
|
||||
if [[ ${{ matrix.db }} == "mysql" ]]; then
|
||||
MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
|
||||
wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
|
||||
for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
|
||||
if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
|
||||
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
|
||||
else
|
||||
for d in alert-server api-server master-server worker-server tools; do
|
||||
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
|
||||
done
|
||||
fi
|
||||
done
|
||||
docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}";
|
||||
else
|
||||
docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
|
||||
fi
|
||||
- name: Check
|
||||
run: |
|
||||
if [[ $DATABASE_VERSION -lt 300 ]]; then
|
||||
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
|
||||
else
|
||||
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
|
||||
fi
|
||||
if [[ ${{ matrix.db }} == "mysql" ]]; then
|
||||
export DATABASE="mysql"
|
||||
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
|
||||
export SPRING_DATASOURCE_USERNAME="root"
|
||||
export SPRING_DATASOURCE_PASSWORD="mysql"
|
||||
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
|
||||
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
|
||||
if [[ $DATABASE_VERSION -lt 300 ]]; then
|
||||
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
|
||||
else
|
||||
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
|
||||
fi
|
||||
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
|
||||
|
||||
atlas_result=$(atlas schema diff \
|
||||
--from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
|
||||
--to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
|
||||
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
|
||||
echo "================================================================================================"
|
||||
echo " !!!!! For Contributors !!!!!"
|
||||
echo "================================================================================================"
|
||||
echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
|
||||
echo "${atlas_result}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
export DATABASE="postgresql"
|
||||
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
|
||||
export SPRING_DATASOURCE_USERNAME="postgres"
|
||||
export SPRING_DATASOURCE_PASSWORD="postgres"
|
||||
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
|
||||
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
|
||||
if [[ $DATABASE_VERSION -lt 300 ]]; then
|
||||
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
|
||||
else
|
||||
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
|
||||
fi
|
||||
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
|
||||
|
||||
atlas_result=$(atlas schema diff \
|
||||
--from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
|
||||
--to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
|
||||
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
|
||||
echo "================================================================================================"
|
||||
echo " !!!!! For Contributors !!!!!"
|
||||
echo "================================================================================================"
|
||||
echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
|
||||
echo "${atlas_result}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
result:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ build, paths-filter, cluster-test, schema-check ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Build!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
|
||||
echo "Build Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
name: SonarCloud
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
jobs:
|
||||
build:
|
||||
name: Build and analyze
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'zulu' # Alternative distribution options are available.
|
||||
- name: Cache SonarCloud packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.sonar/cache
|
||||
key: ${{ runner.os }}-sonar
|
||||
restore-keys: ${{ runner.os }}-sonar
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
- name: Build and analyze
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=zepic-sonar_dolphinscheduler
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
|
||||
RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Create database
|
||||
mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mysql:
|
||||
container_name: mysql
|
||||
image: mysql:5.7.36
|
||||
command: --default-authentication-plugin=mysql_native_password
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: 123456
|
||||
ports:
|
||||
- "3306:3306"
|
||||
healthcheck:
|
||||
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_mysql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-mysql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export SPRING_DATASOURCE_USERNAME=root
|
||||
export SPRING_DATASOURCE_PASSWORD=123456
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=root
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=123456
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Download mysql jar
|
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
|
||||
ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
|
||||
RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
|
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Create database
|
||||
mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mysql:
|
||||
container_name: mysql
|
||||
image: mysql:5.7.36
|
||||
command: --default-authentication-plugin=mysql_native_password
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: 123456
|
||||
ports:
|
||||
- "3306:3306"
|
||||
healthcheck:
|
||||
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
||||
zoo1:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo1
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 1
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
zoo2:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo2
|
||||
ports:
|
||||
- "2182:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 2
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
zoo3:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo3
|
||||
ports:
|
||||
- "2183:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 3
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_mysql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-mysql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
|
||||
export SPRING_DATASOURCE_USERNAME=root
|
||||
export SPRING_DATASOURCE_PASSWORD=123456
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
|
||||
export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
||||
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
|
||||
zkRoot=${zkRoot:-"/dolphinscheduler"}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds mysql cluster image
|
||||
docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds mysql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
container_name: postgres
|
||||
image: postgres:14.1
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dolphinscheduler
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_postgresql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export SPRING_DATASOURCE_USERNAME=postgres
|
||||
export SPRING_DATASOURCE_PASSWORD=postgres
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=jdbc
|
||||
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export REGISTRY_HIKARI_CONFIG_USERNAME=postgres
|
||||
export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM eclipse-temurin:8-jre
|
||||
|
||||
RUN apt update ; \
|
||||
apt install -y wget sudo openssh-server netcat-traditional ;
|
||||
|
||||
COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
|
||||
RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
|
||||
|
||||
RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
|
||||
|
||||
#Setting install.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
|
||||
|
||||
#Setting dolphinscheduler_env.sh
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
|
||||
|
||||
#Deploy
|
||||
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh
|
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ]
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
|
||||
USER=root
|
||||
|
||||
#Sudo
|
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
|
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
|
||||
|
||||
#SSH
|
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
|
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
service ssh start
|
||||
|
||||
#Init schema
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
|
||||
|
||||
#Start Cluster
|
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
|
||||
|
||||
#Keep running
|
||||
tail -f /dev/null
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
container_name: postgres
|
||||
image: postgres:14.1
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dolphinscheduler
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 60s
|
||||
retries: 120
|
||||
|
||||
zoo1:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo1
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 1
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
zoo2:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo2
|
||||
ports:
|
||||
- "2182:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 2
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
zoo3:
|
||||
image: zookeeper:3.8.0
|
||||
restart: always
|
||||
hostname: zoo3
|
||||
ports:
|
||||
- "2183:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 3
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
|
||||
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
ds:
|
||||
container_name: ds
|
||||
image: jdk8:ds_postgresql_cluster
|
||||
restart: always
|
||||
ports:
|
||||
- "12345:12345"
|
||||
- "5679:5679"
|
||||
- "1235:1235"
|
||||
- "50053:50053"
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server
|
||||
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
|
||||
|
||||
# Database related configuration, set database type, username and password
|
||||
export DATABASE=${DATABASE:-postgresql}
|
||||
export SPRING_PROFILES_ACTIVE=${DATABASE}
|
||||
export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
|
||||
export SPRING_DATASOURCE_USERNAME=postgres
|
||||
export SPRING_DATASOURCE_PASSWORD=postgres
|
||||
|
||||
# DolphinScheduler server related configuration
|
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
|
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
|
||||
|
||||
# Registry center configuration, determines the type and link of the registry center
|
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
|
||||
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
|
||||
export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms
|
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks.
|
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
|
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
|
||||
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
|
||||
export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3}
|
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
|
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
|
||||
export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
|
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
|
||||
|
||||
export MASTER_RESERVED_MEMORY=0.01
|
||||
export WORKER_RESERVED_MEMORY=0.01
|
||||
|
||||
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
|
||||
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
|
||||
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
|
||||
#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
|
||||
#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
|
||||
#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# INSTALL MACHINE
|
||||
# ---------------------------------------------------------
|
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
|
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
|
||||
# mode, just write a pseudo-distributed hostname
|
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
|
||||
ips=${ips:-"localhost"}
|
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
|
||||
# modify it if you use different ssh port
|
||||
sshPort=${sshPort:-"22"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
|
||||
masters=${masters:-"localhost"}
|
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
|
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
|
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
|
||||
workers=${workers:-"localhost:default"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
|
||||
alertServer=${alertServer:-"localhost"}
|
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it
|
||||
# must be a subset of configuration `ips`.
|
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
|
||||
apiServers=${apiServers:-"localhost"}
|
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
|
||||
# Do not set this configuration same as the current path (pwd)
|
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
|
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
|
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
|
||||
# to be created by this user
|
||||
deployUser=${deployUser:-"dolphinscheduler"}
|
||||
|
||||
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
|
||||
zkRoot=${zkRoot:-"/dolphinscheduler"}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -x
|
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
|
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
|
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
|
||||
ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health"
|
||||
|
||||
#Cluster start health check
|
||||
TIMEOUT=180
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
|
||||
for ((i=1; i<=TIMEOUT; i++))
|
||||
do
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then
|
||||
START_HEALTHCHECK_EXITCODE=0
|
||||
else
|
||||
START_HEALTHCHECK_EXITCODE=2
|
||||
fi
|
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
|
||||
echo "cluster start health check success"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out"
|
||||
echo "master start health check failed"
|
||||
fi
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out"
|
||||
echo "worker start health check failed"
|
||||
fi
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out"
|
||||
echo "api start health check failed"
|
||||
fi
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log"
|
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out"
|
||||
echo "alert start health check failed"
|
||||
fi
|
||||
exit $START_HEALTHCHECK_EXITCODE
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
#Stop Cluster
|
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
|
||||
|
||||
#Cluster stop health check
|
||||
sleep 5
|
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
|
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "master stop health check success"
|
||||
else
|
||||
echo "master stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
|
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
|
||||
echo "worker stop health check success"
|
||||
else
|
||||
echo "worker stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
|
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then
|
||||
echo "api stop health check success"
|
||||
else
|
||||
echo "api stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND")
|
||||
if [[ $ALERT_HTTP_STATUS -ne 200 ]];then
|
||||
echo "alert stop health check success"
|
||||
else
|
||||
echo "alert stop health check failed"
|
||||
exit 3
|
||||
fi
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euox pipefail
|
||||
|
||||
#Start base service containers
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d
|
||||
|
||||
#Build ds postgresql cluster image
|
||||
docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile .
|
||||
|
||||
#Start ds postgresql cluster container
|
||||
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d
|
||||
|
||||
#Running tests
|
||||
/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
|
||||
|
||||
#Cleanup
|
||||
docker rm -f $(docker ps -aq)
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'dev'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'dev'
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule')
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: java
|
||||
queries: +security-and-quality
|
||||
|
||||
- run: |
|
||||
./mvnw -B clean install \
|
||||
-Dmaven.test.skip \
|
||||
-Dmaven.javadoc.skip \
|
||||
-Dspotless.skip=true \
|
||||
-Prelease
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Docs
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: '0 18 * * *' # TimeZone: UTC 0
|
||||
|
||||
concurrency:
|
||||
group: doc-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
style:
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
run: ./mvnw spotless:check
|
||||
img-check:
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Run Dev Relative Reference
|
||||
run: python img_utils.py -v dev-syntax
|
||||
- name: Run Image Check
|
||||
run: python img_utils.py -v check
|
||||
dead-link:
|
||||
if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: sudo npm install -g markdown-link-check@3.11.2
|
||||
- run: sudo apt install plocate -y
|
||||
# NOTE: Change command from `find . -name "*.md"` to `find . -not -path "*/node_modules/*" -not -path "*/.tox/*" -name "*.md"`
|
||||
# if you want to run check locally
|
||||
- run: |
|
||||
for file in $(locate "$PWD*/*.md" | grep -v ./deploy/terraform/aws/README.md); do
|
||||
markdown-link-check -c .dlc.json -q "$file" &
|
||||
done
|
||||
wait
|
||||
paths-filter:
|
||||
name: Helm-Doc-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
helm-doc: ${{ steps.filter.outputs.helm-doc }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
helm-doc:
|
||||
- 'deploy/**'
|
||||
helm-doc:
|
||||
name: Helm-Doc-Execute
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.helm-doc == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Generating helm-doc
|
||||
run: |
|
||||
./mvnw validate -P helm-doc -pl :dolphinscheduler
|
||||
- name: Check helm-doc
|
||||
run: |
|
||||
DIFF=$(git diff ${GITHUB_WORKSPACE}/deploy/kubernetes/*md)
|
||||
if [ ! -z "$DIFF" ]; then
|
||||
echo "###### ERROR: helm-doc is not up to date ######"
|
||||
echo "Please execute './mvnw validate -P helm-doc -pl :dolphinscheduler' in your clone, of your fork, of the project, and commit an updated deploy/kubernetes/README.md for the chart."
|
||||
echo "###### ERROR: helm-doc is not up to date ######"
|
||||
fi
|
||||
git diff --exit-code
|
||||
result:
|
||||
name: Docs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs:
|
||||
- style
|
||||
- img-check
|
||||
- dead-link
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ contains(needs.*.result, 'failure') }} == 'true' || ${{ contains(needs.*.result, 'cancelled') }} == 'true' ]]; then
|
||||
echo "Build Failed!"
|
||||
exit 1
|
||||
else
|
||||
echo "Build Success!"
|
||||
fi
|
||||
|
|
@ -1,147 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
name: E2E-K8S
|
||||
|
||||
concurrency:
|
||||
group: E2E-K8S-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: E2E-K8S-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
e2e-k8s:
|
||||
name: E2E-K8S-Execute
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Build Image
|
||||
run: |
|
||||
./mvnw -B clean package \
|
||||
-Dmaven.test.skip \
|
||||
-Dmaven.javadoc.skip \
|
||||
-Dspotless.skip=true \
|
||||
-Dmaven.checkstyle.skip \
|
||||
-Dmaven.deploy.skip \
|
||||
-Ddocker.push.skip=true \
|
||||
-Pdocker,release -Ddocker.tag=ci \
|
||||
-pl org.apache.dolphinscheduler:dolphinscheduler-alert-server \
|
||||
-pl dolphinscheduler-tools \
|
||||
-pl dolphinscheduler-api \
|
||||
-pl dolphinscheduler-master \
|
||||
-pl dolphinscheduler-worker -am
|
||||
- name: Create k8s Kind Cluster
|
||||
run: |
|
||||
# install kubectl
|
||||
curl -LO "https://dl.k8s.io/release/v1.28.3/bin/linux/amd64/kubectl"
|
||||
sudo chmod +x kubectl
|
||||
sudo mv kubectl /usr/local/bin/kubectl
|
||||
|
||||
# install kind
|
||||
curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64
|
||||
sudo chmod +x kind-linux-amd64
|
||||
sudo mv kind-linux-amd64 /usr/local/bin/kind
|
||||
kind version
|
||||
|
||||
# create kind cluster
|
||||
kind_node_image="kindest/node:v1.23.17"
|
||||
echo "Kubernetes version: ${kind_node_image}"
|
||||
kind create cluster --name dolphinscheduler --image ${kind_node_image}
|
||||
kubectl version
|
||||
kubectl get all --all-namespaces
|
||||
- name: Load images
|
||||
run: |
|
||||
components=("master" "worker" "api" "tools" "alert-server")
|
||||
for component in "${components[@]}"; do
|
||||
kind load docker-image apache/dolphinscheduler-${component}:ci --name dolphinscheduler
|
||||
done
|
||||
- name: Helm install dolphinscheduler
|
||||
working-directory: ${{ github.workspace }}/deploy/kubernetes/dolphinscheduler
|
||||
run: |
|
||||
# install helm
|
||||
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
||||
|
||||
# install ds chart via helm
|
||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
helm dependency update .
|
||||
helm upgrade --install --create-namespace -n dolphinscheduler dolphinscheduler . -f - <<EOF
|
||||
image:
|
||||
registry: apache
|
||||
tag: ci
|
||||
master:
|
||||
replicas: 1
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
worker:
|
||||
replicas: 1
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
alert:
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
api:
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
EOF
|
||||
|
||||
- name: Wait for pods
|
||||
run: |
|
||||
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
|
||||
components=("master" "worker" "api" "alert")
|
||||
for component in "${components[@]}"; do
|
||||
until kubectl -n dolphinscheduler get pods -l app.kubernetes.io/component=${component} -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
|
||||
echo "waiting for dolphinscheduler ${component} to be available"
|
||||
sleep 10
|
||||
kubectl get pods --all-namespaces
|
||||
kubectl get events --all-namespaces
|
||||
for pod in $(kubectl get pods -n dolphinscheduler -o jsonpath='{.items[*].metadata.name}'); do
|
||||
echo "logs for pod $pod:"
|
||||
kubectl logs --tail=1000 -n dolphinscheduler $pod
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,219 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
name: E2E
|
||||
|
||||
concurrency:
|
||||
group: e2e-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: E2E-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: E2E-Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
uses: ./.github/actions/sanity-check
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- name: Build Image
|
||||
run: |
|
||||
./mvnw -B clean install \
|
||||
-Dmaven.test.skip \
|
||||
-Dmaven.javadoc.skip \
|
||||
-Dspotless.skip=true \
|
||||
-Pdocker,release -Ddocker.tag=ci \
|
||||
-pl dolphinscheduler-standalone-server -am
|
||||
- name: Export Docker Images
|
||||
run: |
|
||||
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
|
||||
&& du -sh /tmp/standalone-image.tar
|
||||
- uses: actions/upload-artifact@v2
|
||||
name: Upload Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
path: /tmp/standalone-image.tar
|
||||
retention-days: 1
|
||||
e2e:
|
||||
name: ${{ matrix.case.name }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: Tenant
|
||||
class: org.apache.dolphinscheduler.e2e.cases.TenantE2ETest
|
||||
- name: User
|
||||
class: org.apache.dolphinscheduler.e2e.cases.UserE2ETest
|
||||
- name: WorkerGroup
|
||||
class: org.apache.dolphinscheduler.e2e.cases.WorkerGroupE2ETest
|
||||
- name: Project
|
||||
class: org.apache.dolphinscheduler.e2e.cases.ProjectE2ETest
|
||||
- name: Queue
|
||||
class: org.apache.dolphinscheduler.e2e.cases.QueueE2ETest
|
||||
- name: Environment
|
||||
class: org.apache.dolphinscheduler.e2e.cases.EnvironmentE2ETest
|
||||
- name: Cluster
|
||||
class: org.apache.dolphinscheduler.e2e.cases.ClusterE2ETest
|
||||
- name: Token
|
||||
class: org.apache.dolphinscheduler.e2e.cases.TokenE2ETest
|
||||
- name: Workflow
|
||||
class: org.apache.dolphinscheduler.e2e.cases.WorkflowE2ETest
|
||||
- name: WorkflowHttp
|
||||
class: org.apache.dolphinscheduler.e2e.cases.WorkflowHttpTaskE2ETest
|
||||
- name: WorkflowJava
|
||||
class: org.apache.dolphinscheduler.e2e.cases.WorkflowJavaTaskE2ETest
|
||||
# - name: WorkflowForSwitch
|
||||
# class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest
|
||||
- name: FileManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest
|
||||
- name: UdfManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.UdfManageE2ETest
|
||||
- name: FunctionManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.FunctionManageE2ETest
|
||||
- name: MysqlDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest
|
||||
- name: ClickhouseDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
|
||||
- name: PostgresDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
|
||||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
path: /tmp
|
||||
- name: Load Docker Images
|
||||
run: |
|
||||
docker load -i /tmp/standalone-image.tar
|
||||
- name: Run Test
|
||||
run: |
|
||||
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
name: recording-${{ matrix.case.name }}
|
||||
path: ${{ env.RECORDING_PATH }}
|
||||
retention-days: 1
|
||||
e2e-optional:
|
||||
name: ${{ matrix.case.name }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
case:
|
||||
- name: SqlServerDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
|
||||
- name: HiveDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest
|
||||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
path: /tmp
|
||||
- name: Load Docker Images
|
||||
run: |
|
||||
docker load -i /tmp/standalone-image.tar
|
||||
- name: Run Test
|
||||
run: |
|
||||
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
name: recording-${{ matrix.case.name }}
|
||||
path: ${{ env.RECORDING_PATH }}
|
||||
retention-days: 1
|
||||
result:
|
||||
name: E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ e2e, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip E2E!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.e2e.result }} != 'success' ]]; then
|
||||
echo "E2E Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: Frontend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
paths:
|
||||
- '.github/workflows/frontend.yml'
|
||||
- 'dolphinscheduler-ui/**'
|
||||
pull_request:
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: dolphinscheduler-ui
|
||||
|
||||
concurrency:
|
||||
group: frontend-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Frontend-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
build:
|
||||
name: Build
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ ubuntu-latest, macos-latest ]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
name: Sanity Check
|
||||
uses: ./.github/actions/sanity-check
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 16
|
||||
- name: Compile and Build on ${{ matrix.os }}
|
||||
run: |
|
||||
set -x
|
||||
npm install pnpm@7 -g
|
||||
pnpm install
|
||||
pnpm run lint
|
||||
pnpm run build:prod
|
||||
result:
|
||||
name: Frontend Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ build, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Frontend Build!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.build.result }} != 'success' ]]; then
|
||||
echo "Frontend Build Failed!"
|
||||
exit -1
|
||||
fi
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: issue-robot
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
jobs:
|
||||
issueRobot:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }}"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
|
||||
- name: "Translation into English in issue"
|
||||
uses: ./.github/actions/translate-on-issue
|
||||
with:
|
||||
translate-title: true
|
||||
translate-body: true
|
||||
|
||||
- name: "Comment in issue"
|
||||
uses: ./.github/actions/comment-on-issue
|
||||
with:
|
||||
message: |
|
||||
Thank you for your feedback, we have received your issue, Please wait patiently for a reply.
|
||||
* In order for us to understand your request as soon as possible, please provide detailed information, version or pictures.
|
||||
* If you haven't received a reply for a long time, you can [join our slack](https://s.apache.org/dolphinscheduler-slack) and send your question to channel `#troubleshooting`
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: OWASP Dependency Check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/pom.xml'
|
||||
env:
|
||||
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Set up JDK 8
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 8
|
||||
distribution: 'adopt'
|
||||
- name: Run OWASP Dependency Check
|
||||
run: ./mvnw -B clean install verify dependency-check:check -DskipDepCheck=false -Dmaven.test.skip=true -Dspotless.skip=true
|
||||
- name: Upload report
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ cancelled() || failure() }}
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: dependency report
|
||||
path: target/dependency-check-report.html
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: publish-docker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
release:
|
||||
types:
|
||||
- released
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'apache/dolphinscheduler'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-docker
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
if [[ ${{ github.event_name }} == "release" ]]; then
|
||||
echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV
|
||||
echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV
|
||||
echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV
|
||||
echo "HUB=apache" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
|
||||
else
|
||||
echo "DOCKER_REGISTRY=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
|
||||
echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV
|
||||
echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
|
||||
echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAG=${{ github.sha }}" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ${{ env.DOCKER_REGISTRY }}
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_PASSWORD }}
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Build and push docker images
|
||||
run: |
|
||||
./mvnw -B clean deploy \
|
||||
-Dmaven.test.skip \
|
||||
-Dmaven.javadoc.skip \
|
||||
-Dspotless.skip=true \
|
||||
-Dmaven.deploy.skip \
|
||||
-Ddocker.tag=${{ env.DOCKER_TAG }} \
|
||||
-Ddocker.hub=${{ env.HUB }} \
|
||||
-Pdocker,release
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: publish-helm-chart
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
release:
|
||||
types:
|
||||
- released
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'apache/dolphinscheduler'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
if [[ ${{ github.event_name }} == "release" ]]; then
|
||||
echo "HUB=registry-1.docker.io/apache" >> $GITHUB_ENV
|
||||
echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV
|
||||
echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV
|
||||
echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV
|
||||
else
|
||||
echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
|
||||
echo "DOCKER_REGISTRY=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV
|
||||
echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV
|
||||
echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
|
||||
fi
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ${{ env.DOCKER_REGISTRY }}
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_PASSWORD }}
|
||||
- name: Publish Helm Chart
|
||||
working-directory: deploy/kubernetes
|
||||
run: |
|
||||
if [[ ${{ env.HUB }} == "ghcr.io/apache/dolphinscheduler" ]]; then
|
||||
VERSION=0.0.0-$(git rev-parse --short HEAD)
|
||||
sed -i "s/^version: .*/version: $VERSION/" dolphinscheduler/Chart.yaml
|
||||
fi
|
||||
helm dep up dolphinscheduler
|
||||
helm package dolphinscheduler
|
||||
helm push dolphinscheduler-helm-*.tgz oci://${{ env.HUB }}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: "pull-request-robot"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
|
||||
jobs:
|
||||
labelRobot:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }}"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
|
||||
- name: "Label in pull request"
|
||||
uses: actions/labeler@v4
|
||||
with:
|
||||
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
configuration-path: .github/actions/labeler/labeler.yml
|
||||
sync-labels: true
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# https://github.com/actions/stale
|
||||
name: 'Close stale issues and PRs'
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
permissions:
|
||||
# Stale recommended permissions
|
||||
pull-requests: write
|
||||
issues: write
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@v4
|
||||
with:
|
||||
# Stale Issues
|
||||
days-before-issue-stale: 30
|
||||
days-before-issue-close: 7
|
||||
# We do not stale Issues with label `Waiting for reply`, `feature` and `DSIP`
|
||||
exempt-issue-labels: 'Waiting for reply,feature,DSIP,security,roadmap'
|
||||
stale-issue-message: >
|
||||
This issue has been automatically marked as stale because it has not had recent activity
|
||||
for 30 days. It will be closed in next 7 days if no further activity occurs.
|
||||
close-issue-message: >
|
||||
This issue has been closed because it has not received response for too long time. You could
|
||||
reopen it if you encountered similar problems in the future.
|
||||
# Stale PRs
|
||||
days-before-pr-stale: 120
|
||||
days-before-pr-close: 7
|
||||
stale-pr-message: >
|
||||
This pull request has been automatically marked as stale because it has not had recent
|
||||
activity for 120 days. It will be closed in 7 days if no further activity occurs.
|
||||
close-pr-message: >
|
||||
This pull request has been closed because it has not had recent activity. You could reopen it
|
||||
if you try to continue your work, and anyone who are interested in it are encouraged to continue
|
||||
work on this pull request.
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
- 'dolphinscheduler-ui'
|
||||
branches:
|
||||
- dev
|
||||
|
||||
env:
|
||||
LOG_DIR: /tmp/dolphinscheduler
|
||||
|
||||
concurrency:
|
||||
group: unit-test-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
paths-filter:
|
||||
name: Unit-Test-Path-Filter
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
not-ignore:
|
||||
- '!(docs/**)'
|
||||
unit-test:
|
||||
name: Unit-Test
|
||||
needs: paths-filter
|
||||
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
java: ['8', '11']
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
uses: ./.github/actions/sanity-check
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Set up JDK ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'adopt'
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-unit-test
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
|
||||
- name: Run Unit tests
|
||||
run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false
|
||||
- name: Upload coverage report to codecov
|
||||
run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
# Set up JDK 17 for SonarCloud.
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'adopt'
|
||||
- name: Run SonarCloud Analysis
|
||||
run: >
|
||||
./mvnw --batch-mode verify sonar:sonar
|
||||
-Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml
|
||||
-Dmaven.test.skip=true
|
||||
-Dspotless.skip=true
|
||||
-Dsonar.host.url=https://sonarcloud.io
|
||||
-Dsonar.organization=apache
|
||||
-Dsonar.core.codeCoveragePlugin=jacoco
|
||||
-Dsonar.projectKey=apache-dolphinscheduler
|
||||
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
|
||||
-Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/*
|
||||
-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
|
||||
-DskipUT=true -DskipIT=true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
|
||||
- name: Collect logs
|
||||
continue-on-error: true
|
||||
run: |
|
||||
mkdir -p ${LOG_DIR}
|
||||
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
|
||||
|
||||
- name: Upload logs
|
||||
uses: actions/upload-artifact@v2
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: unit-test-logs
|
||||
path: ${LOG_DIR}
|
||||
result:
|
||||
name: Unit Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ unit-test, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
echo "Skip Unit Test!"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${{ needs.unit-test.result }} != 'success' ]]; then
|
||||
echo "Unit Test Failed!"
|
||||
exit -1
|
||||
fi
|
||||
6
pom.xml
6
pom.xml
|
|
@ -17,6 +17,10 @@
|
|||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<properties>
|
||||
<sonar.organization>zepic-sonar</sonar.organization>
|
||||
<sonar.host.url>https://sonarcloud.io</sonar.host.url>
|
||||
</properties>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache</groupId>
|
||||
|
|
@ -60,6 +64,8 @@
|
|||
</modules>
|
||||
|
||||
<properties>
|
||||
<sonar.organization>zepic-sonar</sonar.organization>
|
||||
<sonar.host.url>https://sonarcloud.io</sonar.host.url>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<spring.boot.version>2.6.1</spring.boot.version>
|
||||
|
|
|
|||
Loading…
Reference in New Issue