diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index e4d5c7dc75..0000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,45 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -/.github/ @SbloodyS -/deploy/ @caishunfeng -/dolphinscheduler-alert/ @caishunfeng -/dolphinscheduler-e2e/ @SbloodyS -/dolphinscheduler-api-test/ @SbloodyS -/dolphinscheduler-registry/ @caishunfeng @ruanwenjun -/dolphinscheduler-api/ @caishunfeng @SbloodyS -/dolphinscheduler-dao/ @caishunfeng @SbloodyS -/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888 -/dolphinscheduler-common/ @caishunfeng -/dolphinscheduler-standalone-server/ @caishunfeng -/dolphinscheduler-datasource-plugin/ @caishunfeng -/dolphinscheduler-dist/ @caishunfeng -/dolphinscheduler-meter/ @caishunfeng @ruanwenjun @EricGao888 -/dolphinscheduler-scheduler-plugin/ @caishunfeng -/dolphinscheduler-master/ @caishunfeng @SbloodyS @ruanwenjun -/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun -/dolphinscheduler-service/ @caishunfeng -/dolphinscheduler-extract/ @caishunfeng @ruanwenjun -/dolphinscheduler-spi/ @caishunfeng -/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong -/dolphinscheduler-tools/ @caishunfeng @SbloodyS @EricGao888 -/script/ @caishunfeng @SbloodyS @EricGao888 -/dolphinscheduler-ui/ @songjianet @Amy0104 -/docs/ @EricGao888 -/licenses/ @EricGao888 -/images/ @EricGao888 -/style/ @caishunfeng diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml deleted file mode 100644 index 1bd1e44c98..0000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ /dev/null @@ -1,130 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Bug report -title: "[Bug] [Module Name] Bug title " -description: Problems and issues with code of Apache Dolphinscheduler -labels: [ "bug", "Waiting for reply" ] -body: - - type: markdown - attributes: - value: > - Please make sure what you are reporting is indeed a bug with reproducible steps, if you want to ask questions - or share ideas, you can head to our - [Discussions](https://github.com/apache/dolphinscheduler/discussions) tab, you can also - [join our slack](https://s.apache.org/dolphinscheduler-slack) - and send your question to channel `#troubleshooting` - - For better global communication, Please write in English. - - If you feel the description in English is not clear, then you can append description in Chinese, thanks! - - - type: checkboxes - attributes: - label: Search before asking - description: > - Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) - first to see whether the same issue was reported already. - options: - - label: > - I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found - no similar issues. - required: true - - - type: textarea - attributes: - label: What happened - description: Describe what happened. - placeholder: > - Please provide the context in which the problem occurred and explain what happened - validations: - required: true - - - type: textarea - attributes: - label: What you expected to happen - description: What do you think went wrong? - placeholder: > - Please explain why you think the behaviour is erroneous. It is extremely helpful if you copy and paste - the fragment of logs showing the exact error messages or wrong behaviour and screenshots for - UI problems. You can include files by dragging and dropping them here. - - **NOTE**: please copy and paste texts instead of taking screenshots of them for easy future search. - validations: - required: true - - - type: textarea - attributes: - label: How to reproduce - description: > - What should we do to reproduce the problem? If you are not able to provide a reproducible case, - please open a [Discussion](https://github.com/apache/dolphinscheduler/discussions) instead. - placeholder: > - Please make sure you provide a reproducible step-by-step case of how to reproduce the problem - as minimally and precisely as possible. Keep in mind we do not have access to your deployment. - Remember that non-reproducible issues will be closed! Opening a discussion is recommended as a - first step. - validations: - required: true - - - type: textarea - attributes: - label: Anything else - description: Anything else we need to know? - placeholder: > - How often does this problem occur? (Once? Every time? Only when certain conditions are met?) - Any relevant logs to include? Put them here inside fenced - ``` ``` blocks or inside a collapsable details tag if it's too long: -
x.log lots of stuff
- - - type: dropdown - id: version - attributes: - label: Version - description: > - Which version of Apache DolphinScheduler are you running? We only accept bugs report from the LTS projects. - options: - - dev - - 3.1.x - - 3.2.x - validations: - required: true - - - type: checkboxes - attributes: - label: Are you willing to submit PR? - description: > - This is absolutely not required, but we are happy to guide you in the contribution process - especially if you already have a good understanding of how to implement the fix. - Dolphinscheduler is a totally community-driven project and we love to bring new contributors in. - options: - - label: Yes I am willing to submit a PR! - - - type: checkboxes - attributes: - label: Code of Conduct - description: | - The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. - options: - - label: > - I agree to follow this project's - [Code of Conduct](https://www.apache.org/foundation/policies/conduct) - required: true - - - type: markdown - attributes: - value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 0be9e8268c..0000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -blank_issues_enabled: false -contact_links: - - name: Ask a question or get support - url: https://github.com/apache/dolphinscheduler/discussions/ - about: Ask a question or request support for using Apache DolphinScheduler diff --git a/.github/ISSUE_TEMPLATE/document.yml b/.github/ISSUE_TEMPLATE/document.yml deleted file mode 100644 index 0430691e21..0000000000 --- a/.github/ISSUE_TEMPLATE/document.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Documentation Related -description: Suggest an improvement or report a bug for this project's Documentation -title: "[Doc][Module Name] Documentation bug or improvement" -labels: [ "document", "Waiting for reply" ] -body: - - type: markdown - attributes: - value: | - For better global communication, Please write in English. - - If you feel the description in English is not clear, then you can append description in Chinese, thanks! - - - type: checkboxes - attributes: - label: Search before asking - description: > - Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first - to see whether the same feature was requested already. - options: - - label: > - I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no - similar feature requirement. - required: true - - - type: textarea - attributes: - label: Description - description: A short description why your find in our document. - - - type: textarea - attributes: - label: Documentation Links - description: Copy and paste one or more links of this documentation issue. - - - type: checkboxes - attributes: - label: Are you willing to submit a PR? - description: > - This is absolutely not required, but we are happy to guide you in the contribution process - especially if you already have a good understanding of how to implement the improvement. - DolphinScheduler is a totally community-driven project and we love to bring new contributors in. - options: - - label: Yes I am willing to submit a PR! - - - type: checkboxes - attributes: - label: Code of Conduct - description: | - The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. - options: - - label: | - I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) - required: true - - - type: markdown - attributes: - value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/dsip-request.yml b/.github/ISSUE_TEMPLATE/dsip-request.yml deleted file mode 100644 index f54421b066..0000000000 --- a/.github/ISSUE_TEMPLATE/dsip-request.yml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: DSIP -description: Suggest an idea for this project -title: "[DSIP-][Module Name] DSIP title" -labels: [ "DSIP", "Waiting for reply" ] -body: - - type: markdown - attributes: - value: | - For better global communication, Please write in English. - - If you feel the description in English is not clear, then you can append description in Chinese, thanks! - - - type: checkboxes - attributes: - label: Search before asking - description: > - Please make sure to search in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) first - to see whether the same DSIP was created already. - options: - - label: > - I had searched in the [DSIP](https://github.com/apache/dolphinscheduler/issues/14102) and found no - similar DSIP. - required: true - - - type: textarea - attributes: - label: Motivation - description: Why you want to do this change? - - - type: textarea - attributes: - label: Design Detail - description: Your design. - placeholder: > - It's better to provide a detailed design, such as the design of the interface, the design of the database, etc. - - - type: textarea - attributes: - label: Compatibility, Deprecation, and Migration Plan - description: > - If this feature is related to compatibility, deprecation, or migration, please describe it here. - - - type: textarea - attributes: - label: Test Plan - description: > - How to test this improvement. - - - type: checkboxes - attributes: - label: Code of Conduct - description: | - The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. - options: - - label: | - I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) - required: true - - - type: markdown - attributes: - value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml deleted file mode 100644 index 585df6a550..0000000000 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ /dev/null @@ -1,81 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: Feature request -description: Suggest an idea for this project -title: "[Feature][Module Name] Feature title" -labels: [ "feature", "Waiting for reply" ] -body: - - type: markdown - attributes: - value: | - For better global communication, Please write in English. - - If you feel the description in English is not clear, then you can append description in Chinese, thanks! - - - type: checkboxes - attributes: - label: Search before asking - description: > - Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first - to see whether the same feature was requested already. - options: - - label: > - I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no - similar feature requirement. - required: true - - - type: textarea - attributes: - label: Description - description: A short description of your feature - - - type: textarea - attributes: - label: Use case - description: What do you want to happen? - placeholder: > - Rather than telling us how you might implement this feature, try to take a - step back and describe what you are trying to achieve. - - - type: textarea - attributes: - label: Related issues - description: Is there currently another issue associated with this? - - - type: checkboxes - attributes: - label: Are you willing to submit a PR? - description: > - This is absolutely not required, but we are happy to guide you in the contribution process - especially if you already have a good understanding of how to implement the feature. - DolphinScheduler is a totally community-driven project and we love to bring new contributors in. - options: - - label: Yes I am willing to submit a PR! - - - type: checkboxes - attributes: - label: Code of Conduct - description: | - The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. - options: - - label: | - I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) - required: true - - - type: markdown - attributes: - value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/improvement-report.yml b/.github/ISSUE_TEMPLATE/improvement-report.yml deleted file mode 100644 index 1a3e88fca8..0000000000 --- a/.github/ISSUE_TEMPLATE/improvement-report.yml +++ /dev/null @@ -1,68 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: Improvement request -description: Suggest an improvement for this project -title: "[Improvement][Module Name] Improvement title" -labels: [ "improvement", "Waiting for reply" ] -body: - - type: markdown - attributes: - value: | - For better global communication, Please write in English. - - If you feel the description in English is not clear, then you can append description in Chinese, thanks! - - - type: checkboxes - attributes: - label: Search before asking - description: > - Please make sure to search in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) first - to see whether the same feature was requested already. - options: - - label: > - I had searched in the [issues](https://github.com/apache/dolphinscheduler/issues?q=is%3Aissue) and found no - similar feature requirement. - required: true - - - type: textarea - attributes: - label: Description - description: A short description why your want to do this improvement - - - type: checkboxes - attributes: - label: Are you willing to submit a PR? - description: > - This is absolutely not required, but we are happy to guide you in the contribution process - especially if you already have a good understanding of how to implement the improvement. - DolphinScheduler is a totally community-driven project and we love to bring new contributors in. - options: - - label: Yes I am willing to submit a PR! - - - type: checkboxes - attributes: - label: Code of Conduct - description: | - The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. - options: - - label: | - I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) - required: true - - - type: markdown - attributes: - value: "Thanks for completing our form!" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 652e41a3bd..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,34 +0,0 @@ - - -## Purpose of the pull request - - - -## Brief change log - - - -## Verify this pull request - - - -This pull request is code cleanup without any test coverage. - -*(or)* - -This pull request is already covered by existing tests, such as *(please describe tests)*. - -(or) - -This change added tests and can be verified as follows: - - - -(or) - -If your pull request contain incompatible change, you should also add it to `docs/docs/en/guide/upgrede/incompatible.md` diff --git a/.github/actions/comment-on-issue b/.github/actions/comment-on-issue deleted file mode 160000 index 966bbbd211..0000000000 --- a/.github/actions/comment-on-issue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 966bbbd2113de1f8008322fd1cc7674981a6b1ab diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml deleted file mode 100644 index b4776198e2..0000000000 --- a/.github/actions/labeler/labeler.yml +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -backend: - - 'dolphinscheduler-alert/**/*' - - 'dolphinscheduler-api/**/*' - - 'dolphinscheduler-common/**/*' - - 'dolphinscheduler-dao/**/*' - - 'dolphinscheduler-data-quality/**/*' - - 'dolphinscheduler-datasource-plugin/**/*' - - 'dolphinscheduler-dist/**/*' - - 'dolphinscheduler-master/**/*' - - 'dolphinscheduler-registry/**/*' - - 'dolphinscheduler-extract/**/*' - - 'dolphinscheduler-scheduler-plugin/**/*' - - 'dolphinscheduler-service/**/*' - - 'dolphinscheduler-spi/**/*' - - 'dolphinscheduler-standalone-server/**/*' - - 'dolphinscheduler-task-plugin/**/*' - - 'dolphinscheduler-tools/**/*' - - 'dolphinscheduler-worker/**/*' - - 'script/**/*' - -document: - - 'docs/**/*' - -CI&CD: - - any: ['.github/**/*'] - -docker: - - any: ['.deploy/**/*'] - -UI: - - any: ['dolphinscheduler-ui/**/*'] - -e2e: - - any: ['dolphinscheduler-e2e/**/*'] - -test: - - any: ['dolphinscheduler-api-test/**/*'] diff --git a/.github/actions/sanity-check/action.yml b/.github/actions/sanity-check/action.yml deleted file mode 100644 index f7b1287f13..0000000000 --- a/.github/actions/sanity-check/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Sanity Check" - -description: | - Action to perform some very basic lightweight checks, like code styles, license headers, etc., - and fail fast to avoid wasting resources running heavyweight checks, like unit tests, e2e tests. - -inputs: - token: - description: 'The GitHub API token' - required: false - -runs: - using: "composite" - steps: - - name: Check License Header - uses: apache/skywalking-eyes/header@main - - - shell: bash - run: ./mvnw spotless:check diff --git a/.github/actions/translate-on-issue b/.github/actions/translate-on-issue deleted file mode 160000 index 8c4d3b8ead..0000000000 --- a/.github/actions/translate-on-issue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8c4d3b8ead41812bbe837dc5dfb8fa047cd2b548 diff --git a/.github/mergeable.yml b/.github/mergeable.yml deleted file mode 100644 index a1df2e7410..0000000000 --- a/.github/mergeable.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -version: 2 -mergeable: - # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643, - # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as - # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like, - # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened - - when: pull_request.opened, pull_request.reopened, pull_request.synchronize - name: sync-sql-ddl - validate: - # Sql files must change synchronize - - do: dependent - files: - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql' - - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql' - message: 'Sql files not change synchronize' - # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize - fail: - - do: comment - payload: - body: > - :warning: This PR do not change database DDL synchronize. - leave_old_comment: false - - do: labels - add: 'sql not sync' - # Remove labels 'sql not sync' if pass - pass: - - do: labels - delete: 'sql not sync' - - - when: pull_request.* - name: milestone-label-check - validate: - - do: milestone - no_empty: - enabled: false # Cannot be empty when true. - message: 'Milestone is required and cannot be empty.' - - do: label - and: - - must_include: - regex: 'feature|bug|improvement|document|chore|revert' - message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`' - - must_include: - regex: 'ready-to-merge' - message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.' diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml deleted file mode 100644 index fd39bd0977..0000000000 --- a/.github/workflows/api-test.yml +++ /dev/null @@ -1,153 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - -name: API-Test - -concurrency: - group: api-test-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - - -jobs: - paths-filter: - name: API-Test-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: API-Test-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./mvnw -B clean install \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ - -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci - - name: Export Docker Images - run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar - - uses: actions/upload-artifact@v2 - name: Upload Docker Images - with: - name: standalone-image - path: /tmp/standalone-image.tar - retention-days: 1 - api-test: - name: ${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: Tenant - class: org.apache.dolphinscheduler.api.test.cases.TenantAPITest - - name: WorkerGroup - class: org.apache.dolphinscheduler.api.test.cases.WorkerGroupAPITest - - name: Project - class: org.apache.dolphinscheduler.api.test.cases.ProjectAPITest - - name: Workflow - class: org.apache.dolphinscheduler.api.test.cases.ProcessDefinitionAPITest - - name: Scheduler - class: org.apache.dolphinscheduler.api.test.cases.SchedulerAPITest - - name: Executor - class: org.apache.dolphinscheduler.api.test.cases.ExecutorAPITest - - name: ProcessInstance - class: org.apache.dolphinscheduler.api.test.cases.ProcessInstanceAPITest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test - restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v2 - name: Download Docker Images - with: - name: standalone-image - path: /tmp - - name: Load Docker Images - run: | - docker load -i /tmp/standalone-image.tar - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \ - -DfailIfNoTests=false \ - -Dspotless.skip=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v2 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - result: - name: API-Test-Result - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ api-test, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip API Test!" - exit 0 - fi - if [[ ${{ needs.api-test.result }} != 'success' ]]; then - echo "API test Failed!" - exit -1 - fi diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml deleted file mode 100644 index 671223d286..0000000000 --- a/.github/workflows/backend.yml +++ /dev/null @@ -1,277 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Backend - -on: - push: - branches: - - dev - paths: - - '.github/workflows/backend.yml' - - 'package.xml' - - 'pom.xml' - - 'dolphinscheduler-alert/**' - - 'dolphinscheduler-api/**' - - 'dolphinscheduler-common/**' - - 'dolphinscheduler-dao/**' - - 'dolphinscheduler-rpc/**' - pull_request: - -concurrency: - group: backend-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - paths-filter: - name: Backend-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - db-schema: ${{ steps.filter.outputs.db-schema }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - db-schema: - - 'dolphinscheduler-dao/src/main/resources/sql/**' - build: - name: Backend-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - strategy: - matrix: - java: [ '8', '11' ] - timeout-minutes: 30 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.java }} - distribution: 'adopt' - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend - restore-keys: ${{ runner.os }}-maven- - - name: Build and Package on ${{ matrix.java }} - run: | - ./mvnw -B clean install \ - -Prelease \ - -Dmaven.test.skip=true \ - -Dspotless.skip=true \ - -Dhttp.keepAlive=false \ - -Dmaven.wagon.http.pool=false \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - - name: Check dependency license - run: tools/dependencies/check-LICENSE.sh - - uses: actions/upload-artifact@v2 - if: ${{ matrix.java == '8' }} - name: Upload Binary Package - with: - name: binary-package-${{ matrix.java }} - path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz - retention-days: 1 - cluster-test: - name: ${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - matrix: - case: - - name: cluster-test-mysql-with-zookeeper-registry - script: .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh - - name: cluster-test-mysql-with-mysql-registry - script: .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh - - name: cluster-test-postgresql-zookeeper-registry - script: .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh - - name: cluster-test-postgresql-with-postgresql-registry - script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: actions/download-artifact@v2 - name: Download Binary Package - with: - # Only run cluster test on jdk8 - name: binary-package-8 - path: ./ - - name: Running cluster test - run: | - /bin/bash ${{ matrix.case.script }} - schema-check: - runs-on: ubuntu-latest - if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }} - timeout-minutes: 20 - needs: build - services: - mysql: - image: mysql:5.7 - env: - MYSQL_ROOT_PASSWORD: mysql - MYSQL_DATABASE: dolphinscheduler_dev - ports: - - 3306:3306 - options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dolphinscheduler_dev - ports: - - 5432:5432 - options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5 - strategy: - fail-fast: false - matrix: - db: ["mysql", "postgresql"] - version: ["2.0.9", "3.0.6", "3.1.9", "3.2.0"] - steps: - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - java-version: 8 - distribution: 'adopt' - - name: Install Atlas and Create Dir - run: | - mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }} - curl -sSf https://atlasgo.sh | sh - - name: Download Tarball - uses: actions/download-artifact@v2 - with: - name: binary-package-8 - path: dolphinscheduler/dev - - name: Set Env - run: | - VERSION=${{ matrix.version }} - echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV - - name: Prepare - run: | - wget https://archive.apache.org/dist/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }} - tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1 - tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1 - - if [[ ${{ matrix.db }} == "mysql" ]]; then - MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" - MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar" - wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR} - for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do - if [[ $base_dir == *"dolphinscheduler/2"* ]]; then - cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib - else - for d in alert-server api-server master-server worker-server tools; do - cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs - done - fi - done - docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}"; - else - docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};" - fi - - name: Check - run: | - if [[ $DATABASE_VERSION -lt 300 ]]; then - chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh - else - chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh - fi - if [[ ${{ matrix.db }} == "mysql" ]]; then - export DATABASE="mysql" - export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver" - export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false" - export SPRING_DATASOURCE_USERNAME="root" - export SPRING_DATASOURCE_PASSWORD="mysql" - bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh - - export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false" - if [[ $DATABASE_VERSION -lt 300 ]]; then - bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh - else - bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh - fi - bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh - - atlas_result=$(atlas schema diff \ - --from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \ - --to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev") - if [[ ${atlas_result} != *"Schemas are synced"* ]]; then - echo "================================================================================================" - echo " !!!!! For Contributors !!!!!" - echo "================================================================================================" - echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory" - echo "${atlas_result}" - exit 1 - fi - else - export DATABASE="postgresql" - export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" - export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev" - export SPRING_DATASOURCE_USERNAME="postgres" - export SPRING_DATASOURCE_PASSWORD="postgres" - bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh - - export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}" - if [[ $DATABASE_VERSION -lt 300 ]]; then - bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh - else - bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh - fi - bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh - - atlas_result=$(atlas schema diff \ - --from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \ - --to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable") - if [[ ${atlas_result} != *"Schemas are synced"* ]]; then - echo "================================================================================================" - echo " !!!!! For Contributors !!!!!" - echo "================================================================================================" - echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory" - echo "${atlas_result}" - exit 1 - fi - fi - result: - name: Build - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ build, paths-filter, cluster-test, schema-check ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip Build!" - exit 0 - fi - if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then - echo "Build Failed!" - exit -1 - fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..09220ddf40 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +name: SonarCloud +on: + push: + branches: + - dev + pull_request: + types: [opened, synchronize, reopened] +jobs: + build: + name: Build and analyze + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'zulu' # Alternative distribution options are available. + - name: Cache SonarCloud packages + uses: actions/cache@v3 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Build and analyze + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=zepic-sonar_dolphinscheduler diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile deleted file mode 100644 index 12c7db3c18..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jre - -RUN apt update ; \ - apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ; - -COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root -RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ - -RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -#Setting install.sh -COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh - -#Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh - -#Download mysql jar -ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" -ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" -RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER - -#Deploy -COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh - -CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh deleted file mode 100644 index 72b2a630fa..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - - -USER=root - -#Create database -mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" - -#Sudo -sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers -sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers - -#SSH -ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -service ssh start - -#Init schema -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh - -#Start Cluster -/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh - -#Keep running -tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml deleted file mode 100644 index d59e3c868c..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - mysql: - container_name: mysql - image: mysql:5.7.36 - command: --default-authentication-plugin=mysql_native_password - restart: always - environment: - MYSQL_ROOT_PASSWORD: 123456 - ports: - - "3306:3306" - healthcheck: - test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD - interval: 5s - timeout: 60s - retries: 120 diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml deleted file mode 100644 index 7343c8eee7..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - ds: - container_name: ds - image: jdk8:ds_mysql_cluster - restart: always - ports: - - "12345:12345" - - "5679:5679" - - "1235:1235" - - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh deleted file mode 100755 index 8536eb0905..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} - -# Database related configuration, set database type, username and password -export DATABASE=${DATABASE:-mysql} -export SPRING_PROFILES_ACTIVE=${DATABASE} -export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" -export SPRING_DATASOURCE_USERNAME=root -export SPRING_DATASOURCE_PASSWORD=123456 - -# DolphinScheduler server related configuration -export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} -export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} - -# Registry center configuration, determines the type and link of the registry center -export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc} -export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" -export REGISTRY_HIKARI_CONFIG_USERNAME=root -export REGISTRY_HIKARI_CONFIG_PASSWORD=123456 - -# Tasks related configurations, need to change the configuration if you use the related tasks. -export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} -export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} -export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} -export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} -export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} -export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH - -export MASTER_RESERVED_MEMORY=0.01 -export WORKER_RESERVED_MEMORY=0.01 - -# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log -#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* -#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH -#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS -#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS -#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh deleted file mode 100644 index cd660febf8..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh deleted file mode 100644 index 7582c3ccc5..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -x - - -API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" -MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" -WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" -ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" - -#Cluster start health check -TIMEOUT=180 -START_HEALTHCHECK_EXITCODE=0 - -for ((i=1; i<=TIMEOUT; i++)) -do - MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") - WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") - API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") - ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") - if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then - START_HEALTHCHECK_EXITCODE=0 - else - START_HEALTHCHECK_EXITCODE=2 - fi - - if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then - echo "cluster start health check success" - break - fi - - if [[ $i -eq $TIMEOUT ]];then - if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" - echo "master start health check failed" - fi - if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" - echo "worker start health check failed" - fi - if [[ $API_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" - echo "api start health check failed" - fi - if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" - echo "alert start health check failed" - fi - exit $START_HEALTHCHECK_EXITCODE - fi - sleep 1 -done - -#Stop Cluster -docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" - -#Cluster stop health check -sleep 5 -MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") -if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - echo "master stop health check success" -else - echo "master stop health check failed" - exit 3 -fi - -WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") -if [[ $WORKER_HTTP_STATUS -ne 200 ]];then - echo "worker stop health check success" -else - echo "worker stop health check failed" - exit 3 -fi - -API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") -if [[ $API_HTTP_STATUS -ne 200 ]];then - echo "api stop health check success" -else - echo "api stop health check failed" - exit 3 -fi - -ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") -if [[ $ALERT_HTTP_STATUS -ne 200 ]];then - echo "alert stop health check success" -else - echo "alert stop health check failed" - exit 3 -fi diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh deleted file mode 100644 index 0ce48c64ae..0000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - -#Start base service containers -docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml up -d - -#Build ds mysql cluster image -docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile . - -#Start ds mysql cluster container -docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml up -d - -#Running tests -/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh - -#Cleanup -docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile deleted file mode 100644 index 574c059442..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jre - -RUN apt update ; \ - apt install -y wget default-mysql-client sudo openssh-server netcat-traditional ; - -COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root -RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ - -RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -#Setting install.sh -COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh - -#Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh - -#Download mysql jar -ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" -ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" -RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER - -#Deploy -COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh - -CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh deleted file mode 100644 index 722a4b849f..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - - -USER=root - -#Create database -mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" - -#Sudo -sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers -sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers - -#SSH -ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -service ssh start - -#Init schema -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh - -#Start Cluster -/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh - -#Keep running -tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml deleted file mode 100644 index 251b72672d..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - mysql: - container_name: mysql - image: mysql:5.7.36 - command: --default-authentication-plugin=mysql_native_password - restart: always - environment: - MYSQL_ROOT_PASSWORD: 123456 - ports: - - "3306:3306" - healthcheck: - test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD - interval: 5s - timeout: 60s - retries: 120 - - zoo1: - image: zookeeper:3.8.0 - restart: always - hostname: zoo1 - ports: - - "2181:2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - - zoo2: - image: zookeeper:3.8.0 - restart: always - hostname: zoo2 - ports: - - "2182:2181" - environment: - ZOO_MY_ID: 2 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - - zoo3: - image: zookeeper:3.8.0 - restart: always - hostname: zoo3 - ports: - - "2183:2181" - environment: - ZOO_MY_ID: 3 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml deleted file mode 100644 index 7343c8eee7..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - ds: - container_name: ds - image: jdk8:ds_mysql_cluster - restart: always - ports: - - "12345:12345" - - "5679:5679" - - "1235:1235" - - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh deleted file mode 100755 index f64e59b768..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} - -# Database related configuration, set database type, username and password -export DATABASE=${DATABASE:-mysql} -export SPRING_PROFILES_ACTIVE=${DATABASE} -export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" -export SPRING_DATASOURCE_USERNAME=root -export SPRING_DATASOURCE_PASSWORD=123456 - -# DolphinScheduler server related configuration -export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} -export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} - -# Registry center configuration, determines the type and link of the registry center -export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} -export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183} -export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms - -# Tasks related configurations, need to change the configuration if you use the related tasks. -export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} -export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} -export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} -export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} -export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} -export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH - -export MASTER_RESERVED_MEMORY=0.01 -export WORKER_RESERVED_MEMORY=0.01 - -# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log -#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* -#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH -#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS -#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS -#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh deleted file mode 100644 index e43813479b..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh +++ /dev/null @@ -1,61 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} - -# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. -zkRoot=${zkRoot:-"/dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh deleted file mode 100644 index 7582c3ccc5..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -x - - -API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" -MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" -WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" -ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" - -#Cluster start health check -TIMEOUT=180 -START_HEALTHCHECK_EXITCODE=0 - -for ((i=1; i<=TIMEOUT; i++)) -do - MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") - WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") - API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") - ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") - if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then - START_HEALTHCHECK_EXITCODE=0 - else - START_HEALTHCHECK_EXITCODE=2 - fi - - if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then - echo "cluster start health check success" - break - fi - - if [[ $i -eq $TIMEOUT ]];then - if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" - echo "master start health check failed" - fi - if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" - echo "worker start health check failed" - fi - if [[ $API_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" - echo "api start health check failed" - fi - if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" - echo "alert start health check failed" - fi - exit $START_HEALTHCHECK_EXITCODE - fi - sleep 1 -done - -#Stop Cluster -docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" - -#Cluster stop health check -sleep 5 -MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") -if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - echo "master stop health check success" -else - echo "master stop health check failed" - exit 3 -fi - -WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") -if [[ $WORKER_HTTP_STATUS -ne 200 ]];then - echo "worker stop health check success" -else - echo "worker stop health check failed" - exit 3 -fi - -API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") -if [[ $API_HTTP_STATUS -ne 200 ]];then - echo "api stop health check success" -else - echo "api stop health check failed" - exit 3 -fi - -ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") -if [[ $ALERT_HTTP_STATUS -ne 200 ]];then - echo "alert stop health check success" -else - echo "alert stop health check failed" - exit 3 -fi diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh deleted file mode 100644 index db8d23147e..0000000000 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - -#Start base service containers -docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml up -d - -#Build ds mysql cluster image -docker build -t jdk8:ds_mysql_cluster -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile . - -#Start ds mysql cluster container -docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml up -d - -#Running tests -/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh - -#Cleanup -docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile deleted file mode 100644 index bb2d9a5383..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jre - -RUN apt update ; \ - apt install -y wget sudo openssh-server netcat-traditional ; - -COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root -RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ - -RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -#Setting install.sh -COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh - -#Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh - -#Deploy -COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh /root/deploy.sh - -CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh deleted file mode 100644 index 37bf3433c0..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - - -USER=root - -#Sudo -sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers -sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers - -#SSH -ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -service ssh start - -#Init schema -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh - -#Start Cluster -/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh - -#Keep running -tail -f /dev/null diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml deleted file mode 100644 index 1793d94f39..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - postgres: - container_name: postgres - image: postgres:14.1 - restart: always - environment: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dolphinscheduler - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 60s - retries: 120 - diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml deleted file mode 100644 index 9ab79ea44d..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - ds: - container_name: ds - image: jdk8:ds_postgresql_cluster - restart: always - ports: - - "12345:12345" - - "5679:5679" - - "1235:1235" - - "50053:50053" diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh deleted file mode 100644 index 29f8570319..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} - -# Database related configuration, set database type, username and password -export DATABASE=${DATABASE:-postgresql} -export SPRING_PROFILES_ACTIVE=${DATABASE} -export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" -export SPRING_DATASOURCE_USERNAME=postgres -export SPRING_DATASOURCE_PASSWORD=postgres - -# DolphinScheduler server related configuration -export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} -export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} - -# Registry center configuration, determines the type and link of the registry center -export REGISTRY_TYPE=jdbc -export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" -export REGISTRY_HIKARI_CONFIG_USERNAME=postgres -export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres - -# Tasks related configurations, need to change the configuration if you use the related tasks. -export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} -export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} -export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} -export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} -export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} -export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH - -export MASTER_RESERVED_MEMORY=0.01 -export WORKER_RESERVED_MEMORY=0.01 - -# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log -#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* -#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH -#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS -#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS -#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh deleted file mode 100644 index cd660febf8..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh deleted file mode 100644 index 0bc861c389..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -x - - -API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" -MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" -WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" -ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" - -#Cluster start health check -TIMEOUT=180 -START_HEALTHCHECK_EXITCODE=0 - -for ((i=1; i<=TIMEOUT; i++)) -do - MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") - WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") - API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") - ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") - if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then - START_HEALTHCHECK_EXITCODE=0 - else - START_HEALTHCHECK_EXITCODE=2 - fi - - if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then - echo "cluster start health check success" - break - fi - - if [[ $i -eq $TIMEOUT ]];then - if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" - echo "master start health check failed" - fi - if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" - echo "worker start health check failed" - fi - if [[ $API_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" - echo "api start health check failed" - fi - if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" - echo "alert start health check failed" - fi - exit $START_HEALTHCHECK_EXITCODE - fi - - sleep 1 -done - -#Stop Cluster -docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" - -#Cluster stop health check -sleep 5 -MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") -if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - echo "master stop health check success" -else - echo "master stop health check failed" - exit 3 -fi - -WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") -if [[ $WORKER_HTTP_STATUS -ne 200 ]];then - echo "worker stop health check success" -else - echo "worker stop health check failed" - exit 3 -fi - -API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") -if [[ $API_HTTP_STATUS -ne 200 ]];then - echo "api stop health check success" -else - echo "api stop health check failed" - exit 3 -fi - -ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") -if [[ $ALERT_HTTP_STATUS -ne 200 ]];then - echo "alert stop health check success" -else - echo "alert stop health check failed" - exit 3 -fi diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh deleted file mode 100644 index e2b6b630e8..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - -#Start base service containers -docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml up -d - -#Build ds postgresql cluster image -docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile . - -#Start ds postgresql cluster container -docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml up -d - -#Running tests -/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh - -#Cleanup -docker rm -f $(docker ps -aq) diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile deleted file mode 100644 index 077b5c97b8..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jre - -RUN apt update ; \ - apt install -y wget sudo openssh-server netcat-traditional ; - -COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root -RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ - -RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin - -#Setting install.sh -COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh - -#Setting dolphinscheduler_env.sh -COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh - -#Deploy -COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh /root/deploy.sh - -CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh deleted file mode 100644 index f7547d40b4..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - - -USER=root - -#Sudo -sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers -sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers - -#SSH -ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -service ssh start - -#Init schema -/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh - -#Start Cluster -/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh - -#Keep running -tail -f /dev/null diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml deleted file mode 100644 index 8cc7692d71..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - postgres: - container_name: postgres - image: postgres:14.1 - restart: always - environment: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dolphinscheduler - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 60s - retries: 120 - - zoo1: - image: zookeeper:3.8.0 - restart: always - hostname: zoo1 - ports: - - "2181:2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - - zoo2: - image: zookeeper:3.8.0 - restart: always - hostname: zoo2 - ports: - - "2182:2181" - environment: - ZOO_MY_ID: 2 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - - zoo3: - image: zookeeper:3.8.0 - restart: always - hostname: zoo3 - ports: - - "2183:2181" - environment: - ZOO_MY_ID: 3 - ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 - diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml deleted file mode 100644 index 9ab79ea44d..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: "3" - -services: - ds: - container_name: ds - image: jdk8:ds_postgresql_cluster - restart: always - ports: - - "12345:12345" - - "5679:5679" - - "1235:1235" - - "50053:50053" diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh deleted file mode 100644 index 6851716058..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} - -# Database related configuration, set database type, username and password -export DATABASE=${DATABASE:-postgresql} -export SPRING_PROFILES_ACTIVE=${DATABASE} -export SPRING_DATASOURCE_URL="jdbc:postgresql://postgres:5432/dolphinscheduler" -export SPRING_DATASOURCE_USERNAME=postgres -export SPRING_DATASOURCE_PASSWORD=postgres - -# DolphinScheduler server related configuration -export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} -export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} - -# Registry center configuration, determines the type and link of the registry center -export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} -export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183} -export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms - -# Tasks related configurations, need to change the configuration if you use the related tasks. -export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} -export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark} -export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python/bin/python3} -export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} -export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} -export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py} - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH - -export MASTER_RESERVED_MEMORY=0.01 -export WORKER_RESERVED_MEMORY=0.01 - -# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log -#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* -#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH -#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS -#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS -#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh deleted file mode 100644 index e43813479b..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh +++ /dev/null @@ -1,61 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} - -# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. -zkRoot=${zkRoot:-"/dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh deleted file mode 100644 index 0bc861c389..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -x - - -API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" -MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" -WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" -ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" - -#Cluster start health check -TIMEOUT=180 -START_HEALTHCHECK_EXITCODE=0 - -for ((i=1; i<=TIMEOUT; i++)) -do - MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") - WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") - API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") - ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") - if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 && $ALERT_HTTP_STATUS -eq 200 ]];then - START_HEALTHCHECK_EXITCODE=0 - else - START_HEALTHCHECK_EXITCODE=2 - fi - - if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then - echo "cluster start health check success" - break - fi - - if [[ $i -eq $TIMEOUT ]];then - if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" - echo "master start health check failed" - fi - if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" - echo "worker start health check failed" - fi - if [[ $API_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" - echo "api start health check failed" - fi - if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" - echo "alert start health check failed" - fi - exit $START_HEALTHCHECK_EXITCODE - fi - - sleep 1 -done - -#Stop Cluster -docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh" - -#Cluster stop health check -sleep 5 -MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") -if [[ $MASTER_HTTP_STATUS -ne 200 ]];then - echo "master stop health check success" -else - echo "master stop health check failed" - exit 3 -fi - -WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") -if [[ $WORKER_HTTP_STATUS -ne 200 ]];then - echo "worker stop health check success" -else - echo "worker stop health check failed" - exit 3 -fi - -API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") -if [[ $API_HTTP_STATUS -ne 200 ]];then - echo "api stop health check success" -else - echo "api stop health check failed" - exit 3 -fi - -ALERT_HTTP_STATUS=$(eval "$ALERT_HEALTHCHECK_COMMAND") -if [[ $ALERT_HTTP_STATUS -ne 200 ]];then - echo "alert stop health check success" -else - echo "alert stop health check failed" - exit 3 -fi diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh deleted file mode 100644 index fe755c97f1..0000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euox pipefail - -#Start base service containers -docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml up -d - -#Build ds postgresql cluster image -docker build -t jdk8:ds_postgresql_cluster -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile . - -#Start ds postgresql cluster container -docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml up -d - -#Running tests -/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh - -#Cleanup -docker rm -f $(docker ps -aq) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml deleted file mode 100644 index 8810a33c96..0000000000 --- a/.github/workflows/codeql.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "CodeQL" - -on: - push: - branches: - - 'dev' - pull_request: - branches: - - 'dev' - -concurrency: - group: codeql-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - analyze: - if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule') - name: Analyze - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: true - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: java - queries: +security-and-quality - - - run: | - ./mvnw -B clean install \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ - -Dspotless.skip=true \ - -Prelease - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 591bb0a65b..0000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,119 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Docs - -on: - pull_request: - schedule: - - cron: '0 18 * * *' # TimeZone: UTC 0 - -concurrency: - group: doc-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - style: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Style Check - run: ./mvnw spotless:check - img-check: - timeout-minutes: 15 - runs-on: ubuntu-latest - defaults: - run: - working-directory: docs - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Run Dev Relative Reference - run: python img_utils.py -v dev-syntax - - name: Run Image Check - run: python img_utils.py -v check - dead-link: - if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule') - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v2 - - run: sudo npm install -g markdown-link-check@3.11.2 - - run: sudo apt install plocate -y - # NOTE: Change command from `find . -name "*.md"` to `find . -not -path "*/node_modules/*" -not -path "*/.tox/*" -name "*.md"` - # if you want to run check locally - - run: | - for file in $(locate "$PWD*/*.md" | grep -v ./deploy/terraform/aws/README.md); do - markdown-link-check -c .dlc.json -q "$file" & - done - wait - paths-filter: - name: Helm-Doc-Path-Filter - runs-on: ubuntu-latest - outputs: - helm-doc: ${{ steps.filter.outputs.helm-doc }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - helm-doc: - - 'deploy/**' - helm-doc: - name: Helm-Doc-Execute - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.helm-doc == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Generating helm-doc - run: | - ./mvnw validate -P helm-doc -pl :dolphinscheduler - - name: Check helm-doc - run: | - DIFF=$(git diff ${GITHUB_WORKSPACE}/deploy/kubernetes/*md) - if [ ! -z "$DIFF" ]; then - echo "###### ERROR: helm-doc is not up to date ######" - echo "Please execute './mvnw validate -P helm-doc -pl :dolphinscheduler' in your clone, of your fork, of the project, and commit an updated deploy/kubernetes/README.md for the chart." - echo "###### ERROR: helm-doc is not up to date ######" - fi - git diff --exit-code - result: - name: Docs - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: - - style - - img-check - - dead-link - if: always() - steps: - - name: Status - run: | - if [[ ${{ contains(needs.*.result, 'failure') }} == 'true' || ${{ contains(needs.*.result, 'cancelled') }} == 'true' ]]; then - echo "Build Failed!" - exit 1 - else - echo "Build Success!" - fi diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml deleted file mode 100644 index fb402b3c79..0000000000 --- a/.github/workflows/e2e-k8s.yml +++ /dev/null @@ -1,147 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - -name: E2E-K8S - -concurrency: - group: E2E-K8S-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - - -jobs: - paths-filter: - name: E2E-K8S-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - e2e-k8s: - name: E2E-K8S-Execute - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Build Image - run: | - ./mvnw -B clean package \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ - -Dspotless.skip=true \ - -Dmaven.checkstyle.skip \ - -Dmaven.deploy.skip \ - -Ddocker.push.skip=true \ - -Pdocker,release -Ddocker.tag=ci \ - -pl org.apache.dolphinscheduler:dolphinscheduler-alert-server \ - -pl dolphinscheduler-tools \ - -pl dolphinscheduler-api \ - -pl dolphinscheduler-master \ - -pl dolphinscheduler-worker -am - - name: Create k8s Kind Cluster - run: | - # install kubectl - curl -LO "https://dl.k8s.io/release/v1.28.3/bin/linux/amd64/kubectl" - sudo chmod +x kubectl - sudo mv kubectl /usr/local/bin/kubectl - - # install kind - curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64 - sudo chmod +x kind-linux-amd64 - sudo mv kind-linux-amd64 /usr/local/bin/kind - kind version - - # create kind cluster - kind_node_image="kindest/node:v1.23.17" - echo "Kubernetes version: ${kind_node_image}" - kind create cluster --name dolphinscheduler --image ${kind_node_image} - kubectl version - kubectl get all --all-namespaces - - name: Load images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - kind load docker-image apache/dolphinscheduler-${component}:ci --name dolphinscheduler - done - - name: Helm install dolphinscheduler - working-directory: ${{ github.workspace }}/deploy/kubernetes/dolphinscheduler - run: | - # install helm - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - # install ds chart via helm - helm repo add bitnami https://charts.bitnami.com/bitnami - helm dependency update . - helm upgrade --install --create-namespace -n dolphinscheduler dolphinscheduler . -f - <&1 | grep -q "Ready=True"; do - echo "waiting for dolphinscheduler ${component} to be available" - sleep 10 - kubectl get pods --all-namespaces - kubectl get events --all-namespaces - for pod in $(kubectl get pods -n dolphinscheduler -o jsonpath='{.items[*].metadata.name}'); do - echo "logs for pod $pod:" - kubectl logs --tail=1000 -n dolphinscheduler $pod - done - done - done diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index 60a2f44865..0000000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,219 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - -name: E2E - -concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - - -jobs: - paths-filter: - name: E2E-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: E2E-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./mvnw -B clean install \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ - -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci \ - -pl dolphinscheduler-standalone-server -am - - name: Export Docker Images - run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar - - uses: actions/upload-artifact@v2 - name: Upload Docker Images - with: - name: standalone-image - path: /tmp/standalone-image.tar - retention-days: 1 - e2e: - name: ${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: Tenant - class: org.apache.dolphinscheduler.e2e.cases.TenantE2ETest - - name: User - class: org.apache.dolphinscheduler.e2e.cases.UserE2ETest - - name: WorkerGroup - class: org.apache.dolphinscheduler.e2e.cases.WorkerGroupE2ETest - - name: Project - class: org.apache.dolphinscheduler.e2e.cases.ProjectE2ETest - - name: Queue - class: org.apache.dolphinscheduler.e2e.cases.QueueE2ETest - - name: Environment - class: org.apache.dolphinscheduler.e2e.cases.EnvironmentE2ETest - - name: Cluster - class: org.apache.dolphinscheduler.e2e.cases.ClusterE2ETest - - name: Token - class: org.apache.dolphinscheduler.e2e.cases.TokenE2ETest - - name: Workflow - class: org.apache.dolphinscheduler.e2e.cases.WorkflowE2ETest - - name: WorkflowHttp - class: org.apache.dolphinscheduler.e2e.cases.WorkflowHttpTaskE2ETest - - name: WorkflowJava - class: org.apache.dolphinscheduler.e2e.cases.WorkflowJavaTaskE2ETest -# - name: WorkflowForSwitch -# class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest - - name: FileManage - class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest - - name: UdfManage - class: org.apache.dolphinscheduler.e2e.cases.UdfManageE2ETest - - name: FunctionManage - class: org.apache.dolphinscheduler.e2e.cases.FunctionManageE2ETest - - name: MysqlDataSource - class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest - - name: ClickhouseDataSource - class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest - - name: PostgresDataSource - class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v2 - name: Download Docker Images - with: - name: standalone-image - path: /tmp - - name: Load Docker Images - run: | - docker load -i /tmp/standalone-image.tar - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v2 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - e2e-optional: - name: ${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: SqlServerDataSource - class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - - name: HiveDataSource - class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v2 - name: Download Docker Images - with: - name: standalone-image - path: /tmp - - name: Load Docker Images - run: | - docker load -i /tmp/standalone-image.tar - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v2 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - result: - name: E2E - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ e2e, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip E2E!" - exit 0 - fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "E2E Failed!" - exit -1 - fi diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index 502d669b66..0000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,95 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Frontend - -on: - push: - branches: - - dev - paths: - - '.github/workflows/frontend.yml' - - 'dolphinscheduler-ui/**' - pull_request: - -defaults: - run: - working-directory: dolphinscheduler-ui - -concurrency: - group: frontend-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - paths-filter: - name: Frontend-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - if: matrix.os == 'ubuntu-latest' - name: Sanity Check - uses: ./.github/actions/sanity-check - - name: Set up Node.js - uses: actions/setup-node@v2 - with: - node-version: 16 - - name: Compile and Build on ${{ matrix.os }} - run: | - set -x - npm install pnpm@7 -g - pnpm install - pnpm run lint - pnpm run build:prod - result: - name: Frontend Build - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ build, paths-filter ] - if: always() - steps: - - uses: actions/checkout@v2 - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip Frontend Build!" - exit 0 - fi - if [[ ${{ needs.build.result }} != 'success' ]]; then - echo "Frontend Build Failed!" - exit -1 - fi diff --git a/.github/workflows/issue-robot.yml b/.github/workflows/issue-robot.yml deleted file mode 100644 index ab00b34681..0000000000 --- a/.github/workflows/issue-robot.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: issue-robot - -on: - issues: - types: [opened] - -jobs: - issueRobot: - runs-on: ubuntu-latest - steps: - - name: "Checkout ${{ github.ref }}" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: true - - - name: "Translation into English in issue" - uses: ./.github/actions/translate-on-issue - with: - translate-title: true - translate-body: true - - - name: "Comment in issue" - uses: ./.github/actions/comment-on-issue - with: - message: | - Thank you for your feedback, we have received your issue, Please wait patiently for a reply. - * In order for us to understand your request as soon as possible, please provide detailed information, version or pictures. - * If you haven't received a reply for a long time, you can [join our slack](https://s.apache.org/dolphinscheduler-slack) and send your question to channel `#troubleshooting` - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/owasp-dependency-check.yaml b/.github/workflows/owasp-dependency-check.yaml deleted file mode 100644 index 54e51aafed..0000000000 --- a/.github/workflows/owasp-dependency-check.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: OWASP Dependency Check - -on: - push: - branches: - - dev - pull_request: - paths: - - '**/pom.xml' -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - java-version: 8 - distribution: 'adopt' - - name: Run OWASP Dependency Check - run: ./mvnw -B clean install verify dependency-check:check -DskipDepCheck=false -Dmaven.test.skip=true -Dspotless.skip=true - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml deleted file mode 100644 index 06aa8c6950..0000000000 --- a/.github/workflows/publish-docker.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: publish-docker - -on: - push: - branches: - - dev - release: - types: - - released - -jobs: - build: - if: github.repository == 'apache/dolphinscheduler' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - timeout-minutes: 30 - steps: - - uses: actions/checkout@v2 - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-docker - restore-keys: ${{ runner.os }}-maven- - - name: Set environment variables - run: | - if [[ ${{ github.event_name }} == "release" ]]; then - echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV - echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV - echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - echo "HUB=apache" >> $GITHUB_ENV - echo "DOCKER_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - else - echo "DOCKER_REGISTRY=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV - echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV - echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV - echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV - echo "DOCKER_TAG=${{ github.sha }}" >> $GITHUB_ENV - fi - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_PASSWORD }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build and push docker images - run: | - ./mvnw -B clean deploy \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ - -Dspotless.skip=true \ - -Dmaven.deploy.skip \ - -Ddocker.tag=${{ env.DOCKER_TAG }} \ - -Ddocker.hub=${{ env.HUB }} \ - -Pdocker,release diff --git a/.github/workflows/publish-helm-chart.yaml b/.github/workflows/publish-helm-chart.yaml deleted file mode 100644 index e383373692..0000000000 --- a/.github/workflows/publish-helm-chart.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: publish-helm-chart - -on: - push: - branches: - - dev - release: - types: - - released - -jobs: - build: - if: github.repository == 'apache/dolphinscheduler' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - timeout-minutes: 30 - steps: - - uses: actions/checkout@v2 - - name: Set environment variables - run: | - if [[ ${{ github.event_name }} == "release" ]]; then - echo "HUB=registry-1.docker.io/apache" >> $GITHUB_ENV - echo "DOCKER_REGISTRY=docker.io" >> $GITHUB_ENV - echo "DOCKER_USERNAME=${{ secrets.DOCKERHUB_USER }}" >> $GITHUB_ENV - echo "DOCKER_PASSWORD=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV - else - echo "HUB=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV - echo "DOCKER_REGISTRY=ghcr.io/apache/dolphinscheduler" >> $GITHUB_ENV - echo "DOCKER_USERNAME=${{ github.actor }}" >> $GITHUB_ENV - echo "DOCKER_PASSWORD=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV - fi - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_PASSWORD }} - - name: Publish Helm Chart - working-directory: deploy/kubernetes - run: | - if [[ ${{ env.HUB }} == "ghcr.io/apache/dolphinscheduler" ]]; then - VERSION=0.0.0-$(git rev-parse --short HEAD) - sed -i "s/^version: .*/version: $VERSION/" dolphinscheduler/Chart.yaml - fi - helm dep up dolphinscheduler - helm package dolphinscheduler - helm push dolphinscheduler-helm-*.tgz oci://${{ env.HUB }} diff --git a/.github/workflows/pull-request-robot.yml b/.github/workflows/pull-request-robot.yml deleted file mode 100644 index 731849e46d..0000000000 --- a/.github/workflows/pull-request-robot.yml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: "pull-request-robot" - -on: - pull_request_target: - -jobs: - labelRobot: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - name: "Checkout ${{ github.ref }}" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: true - - - name: "Label in pull request" - uses: actions/labeler@v4 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - configuration-path: .github/actions/labeler/labeler.yml - sync-labels: true diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 8746672ac7..0000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# https://github.com/actions/stale -name: 'Close stale issues and PRs' -on: - schedule: - - cron: '0 0 * * *' -permissions: - # Stale recommended permissions - pull-requests: write - issues: write -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v4 - with: - # Stale Issues - days-before-issue-stale: 30 - days-before-issue-close: 7 - # We do not stale Issues with label `Waiting for reply`, `feature` and `DSIP` - exempt-issue-labels: 'Waiting for reply,feature,DSIP,security,roadmap' - stale-issue-message: > - This issue has been automatically marked as stale because it has not had recent activity - for 30 days. It will be closed in next 7 days if no further activity occurs. - close-issue-message: > - This issue has been closed because it has not received response for too long time. You could - reopen it if you encountered similar problems in the future. - # Stale PRs - days-before-pr-stale: 120 - days-before-pr-close: 7 - stale-pr-message: > - This pull request has been automatically marked as stale because it has not had recent - activity for 120 days. It will be closed in 7 days if no further activity occurs. - close-pr-message: > - This pull request has been closed because it has not had recent activity. You could reopen it - if you try to continue your work, and anyone who are interested in it are encouraged to continue - work on this pull request. diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index 6c9f41d7a3..0000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,135 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Test - -on: - pull_request: - push: - paths-ignore: - - '**/*.md' - - 'dolphinscheduler-ui' - branches: - - dev - -env: - LOG_DIR: /tmp/dolphinscheduler - -concurrency: - group: unit-test-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - paths-filter: - name: Unit-Test-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v2 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - unit-test: - name: Unit-Test - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - strategy: - matrix: - java: ['8', '11'] - timeout-minutes: 45 - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.java }} - distribution: 'adopt' - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-unit-test - restore-keys: ${{ runner.os }}-maven- - - - name: Run Unit tests - run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false - - name: Upload coverage report to codecov - run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - - # Set up JDK 17 for SonarCloud. - - name: Set up JDK 17 - uses: actions/setup-java@v2 - with: - java-version: 17 - distribution: 'adopt' - - name: Run SonarCloud Analysis - run: > - ./mvnw --batch-mode verify sonar:sonar - -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml - -Dmaven.test.skip=true - -Dspotless.skip=true - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.organization=apache - -Dsonar.core.codeCoveragePlugin=jacoco - -Dsonar.projectKey=apache-dolphinscheduler - -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 - -Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/* - -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - -DskipUT=true -DskipIT=true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - - name: Collect logs - continue-on-error: true - run: | - mkdir -p ${LOG_DIR} - docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt - - - name: Upload logs - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: unit-test-logs - path: ${LOG_DIR} - result: - name: Unit Test - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ unit-test, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip Unit Test!" - exit 0 - fi - if [[ ${{ needs.unit-test.result }} != 'success' ]]; then - echo "Unit Test Failed!" - exit -1 - fi diff --git a/pom.xml b/pom.xml index dd4cb1adf1..a9b8656c7a 100755 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,10 @@ --> + + zepic-sonar + https://sonarcloud.io + 4.0.0 org.apache @@ -60,6 +64,8 @@ + zepic-sonar + https://sonarcloud.io UTF-8 UTF-8 2.6.1