natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[dev] Easier release: cherry-pick, changelog, contributor (#11478) (#12634) 

Co-authored-by: Jiajie Zhong <zhongjiajie955@gmail.com>
This commit is contained in:
Eric Gao 2022-11-01 11:53:32 +08:00 committed by GitHub
parent d158ae55b9
commit aecff85df7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 574 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
tools/release/README.md Normal file
View File

@ -0,0 +1,38 @@
# Tools Release
A tools for convenient release DolphinScheduler.
## Prepare
* python: python 3.6 or above
* pip: latest version of pip is better
To install dependence, you should run command
```shell
python -m pip install -r requirements.txt
```
## Usage
### Export Environment Variable
You can create new token in [create token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token),
it is only need all permission under `repo`
```shell
export GH_ACCESS_TOKEN="<YOUR-GITHUB-TOKEN-WITH-REPO-ACCESS>"
export GH_REPO_MILESTONE="<YOUR-MILESTONE>"
```
### Help
```shell
python release.py -h
```
### Action
* Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick
* Generate changelog: `python release.py changelog`
* Generate contributor: `python release.py contributor`

16
tools/release/github/__init__.py Normal file
View File

@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

151
tools/release/github/changelog.py Normal file
View File

@ -0,0 +1,151 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Github utils for release changelog."""
from typing import Dict, List
class Changelog:
"""Generate changelog according specific pull requests list.
Each pull requests will only once in final result. If pull requests have more than one label we need,
will classify to high priority label type, currently priority is
`feature > bug > improvement > document > chore`. pr will into feature section if it with both `feature`,
`improvement`, `document` label.
:param prs: pull requests list.
"""
key_number = "number"
key_labels = "labels"
key_name = "name"
label_feature = "feature"
label_bug = "bug"
label_improvement = "improvement"
label_document = "document"
label_chore = "chore"
changelog_prefix = "\n\n<details><summary>Click to expand</summary>\n\n"
changelog_suffix = "\n\n</details>\n"
def __init__(self, prs: List[Dict]):
self.prs = prs
self.features = []
self.bugfixs = []
self.improvements = []
self.documents = []
self.chores = []
def generate(self) -> str:
"""Generate changelog."""
self.classify()
final = []
if self.features:
detail = f"## Feature{self.changelog_prefix}{self._convert(self.features)}{self.changelog_suffix}"
final.append(detail)
if self.improvements:
detail = (
f"## Improvement{self.changelog_prefix}"
f"{self._convert(self.improvements)}{self.changelog_suffix}"
)
final.append(detail)
if self.bugfixs:
detail = f"## Bugfix{self.changelog_prefix}{self._convert(self.bugfixs)}{self.changelog_suffix}"
final.append(detail)
if self.documents:
detail = (
f"## Document{self.changelog_prefix}"
f"{self._convert(self.documents)}{self.changelog_suffix}"
)
final.append(detail)
if self.chores:
detail = f"## Chore{self.changelog_prefix}{self._convert(self.chores)}{self.changelog_suffix}"
final.append(detail)
return "\n".join(final)
@staticmethod
def _convert(prs: List[Dict]) -> str:
"""Convert pull requests into changelog item text."""
return "\n".join(
[f"- {pr['title']} (#{pr['number']}) @{pr['user']['login']}" for pr in prs]
)
def classify(self) -> None:
"""Classify pull requests different kinds of section in changelog.
Each pull requests only belongs to one single classification.
"""
for pr in self.prs:
if self.key_labels not in pr:
raise KeyError("PR %s do not have labels", pr[self.key_number])
if self._is_feature(pr):
self.features.append(pr)
elif self._is_bugfix(pr):
self.bugfixs.append(pr)
elif self._is_improvement(pr):
self.improvements.append(pr)
elif self._is_document(pr):
self.documents.append(pr)
elif self._is_chore(pr):
self.chores.append(pr)
else:
raise KeyError(
"There must at least one of labels `feature|bug|improvement|document|chore`"
"but it do not, pr: %s",
pr["html_url"],
)
def _is_feature(self, pr: Dict) -> bool:
"""Belong to feature pull requests."""
return any(
[
label[self.key_name] == self.label_feature
for label in pr[self.key_labels]
]
)
def _is_bugfix(self, pr: Dict) -> bool:
"""Belong to bugfix pull requests."""
return any(
[label[self.key_name] == self.label_bug for label in pr[self.key_labels]]
)
def _is_improvement(self, pr: Dict) -> bool:
"""Belong to improvement pull requests."""
return any(
[
label[self.key_name] == self.label_improvement
for label in pr[self.key_labels]
]
)
def _is_document(self, pr: Dict) -> bool:
"""Belong to document pull requests."""
return any(
[
label[self.key_name] == self.label_document
for label in pr[self.key_labels]
]
)
def _is_chore(self, pr: Dict) -> bool:
"""Belong to chore pull requests."""
return any(
[label[self.key_name] == self.label_chore for label in pr[self.key_labels]]
)

70
tools/release/github/git.py Normal file
View File

@ -0,0 +1,70 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Github utils for git operations."""
from pathlib import Path
from typing import Dict, Optional
from git import Repo
git_dir_path: Path = Path(__file__).parent.parent.parent.parent.joinpath(".git")
class Git:
"""Operator to handle git object.
:param path: git repository path
:param branch: branch you want to query
"""
def __init__(
self, path: Optional[str] = git_dir_path, branch: Optional[str] = None
):
self.path = path
self.branch = branch
@property
def repo(self) -> Repo:
"""Get git repo object."""
return Repo(self.path)
def has_commit_current(self, sha: str) -> bool:
"""Whether SHA in current branch."""
branches = self.repo.git.branch("--contains", sha)
return f"* {self.repo.active_branch.name}" in branches
def has_commit_global(self, sha: str) -> bool:
"""Whether SHA in all branches."""
try:
self.repo.commit(sha)
return True
except ValueError:
return False
def cherry_pick_pr(self, pr: Dict) -> None:
"""Run command `git cherry-pick -x <SHA>`."""
sha = pr["merge_commit_sha"]
if not self.has_commit_global(sha):
raise RuntimeError(
"Cherry-pick SHA %s error because SHA not exists,"
"please make sure you local default branch is up-to-date",
sha,
)
if self.has_commit_current(sha):
print("SHA %s already in current active branch, skip it.", sha)
self.repo.git.cherry_pick("-x", sha)

64
tools/release/github/pull_request.py Normal file
View File

@ -0,0 +1,64 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import Optional, List, Dict
from github.resp_get import RespGet
class PullRequest:
"""Pull request to filter the by specific condition.
:param token: token to request GitHub API entrypoint.
:param repo: GitHub repository identify, use `user/repo` or `org/repo`.
"""
url_search = "https://api.github.com/search/issues"
url_pr = "https://api.github.com/repos/{}/pulls/{}"
def __init__(self, token: str, repo: Optional[str] = "apache/dolphinscheduler"):
self.token = token
self.repo = repo
self.headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"token {token}",
}
def get_merged_detail(self, number: str) -> Dict:
"""Get all merged pull requests detail by pr number.
:param number: pull requests number you want to get detail.
"""
return RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
def get_merged_detail_by_milestone(self, milestone: str) -> List[Dict]:
"""Get all merged requests pull request detail by specific milestone.
:param milestone: query by specific milestone.
"""
detail = []
numbers = {pr.get("number") for pr in self.search_merged_by_milestone(milestone)}
for number in numbers:
pr_dict = RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
detail.append(pr_dict)
return detail
def search_merged_by_milestone(self, milestone: str) -> List[Dict]:
"""Get all merged requests pull request by specific milestone.
:param milestone: query by specific milestone.
"""
params = {"q": f"repo:{self.repo} is:pr is:merged milestone:{milestone}"}
return RespGet(url=self.url_search, headers=self.headers, param=params).get_total()

67
tools/release/github/resp_get.py Normal file
View File

@ -0,0 +1,67 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Github utils get HTTP response."""
import copy
import json
from typing import Dict, List, Optional
import requests
class RespGet:
"""Get response from GitHub restful API.
:param url: URL to requests GET method.
:param headers: headers for HTTP requests.
:param param: param for HTTP requests.
"""
def __init__(self, url: str, headers: dict, param: Optional[dict] = None):
self.url = url
self.headers = headers
self.param = param
@staticmethod
def get(url: str, headers: dict, params: Optional[dict] = None) -> Dict:
"""Get single response dict from HTTP requests by given condition."""
resp = requests.get(url=url, headers=headers, params=params)
if not resp.ok:
raise ValueError("Requests error with", resp.reason)
return json.loads(resp.content)
def get_single(self) -> Dict:
"""Get single response dict from HTTP requests by given condition."""
return self.get(url=self.url, headers=self.headers, params=self.param)
def get_total(self) -> List[Dict]:
"""Get all response dict from HTTP requests by given condition.
Will change page number until no data return.
"""
total = []
curr_param = copy.deepcopy(self.param)
while True:
curr_param["page"] = curr_param.setdefault("page", 0) + 1
content_dict = self.get(
url=self.url, headers=self.headers, params=curr_param
)
data = content_dict.get("items")
if not data:
return total
total.extend(data)

43
tools/release/github/user.py Normal file
View File

@ -0,0 +1,43 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Github utils for user."""
from typing import Dict, List, Set
class User:
"""Get users according specific pull requests list.
:param prs: pull requests list.
"""
def __init__(self, prs: List[Dict]):
self.prs = prs
def contribution_num(self) -> Dict:
"""Get unique contributor with name and commit number."""
res = dict()
for pr in self.prs:
user_id = pr["user"]["login"]
res[user_id] = res.setdefault(user_id, 0) + 1
return res
def contributors(self) -> Set[str]:
"""Get unique contributor with name."""
cn = self.contribution_num()
return {contributor for contributor in cn}

106
tools/release/release.py Normal file
View File

@ -0,0 +1,106 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Main function for releasing."""
import argparse
import os
from github.changelog import Changelog
from github.git import Git
from github.pull_request import PullRequest
from github.user import User
def get_changelog(access_token: str, milestone: str) -> str:
"""Get changelog in specific milestone from GitHub Restful API."""
pr = PullRequest(token=access_token)
pr_merged = pr.search_merged_by_milestone(milestone)
# Sort according to merged time ascending
pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
changelog = Changelog(pr_merged_sort)
changelog_text = changelog.generate()
return changelog_text
def get_contributor(access_token: str, milestone: str) -> str:
"""Get contributor in specific milestone from GitHub Restful API."""
pr = PullRequest(token=access_token)
pr_merged = pr.search_merged_by_milestone(milestone)
users = User(prs=pr_merged)
contributor = users.contributors()
# Sort according alphabetical
return ", ".join(sorted(contributor))
def auto_cherry_pick(access_token: str, milestone: str) -> None:
"""Do git cherry-pick in specific milestone, require update dev branch."""
pr = PullRequest(token=access_token)
pr_merged = pr.search_merged_by_milestone(milestone)
# Sort according to merged time ascending
pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
for p in pr_merged_sort:
pr_detail = pr.get_merged_detail(p["number"])
print(f"git cherry-pick -x {pr_detail['merge_commit_sha']}")
Git().cherry_pick_pr(pr_detail)
def build_argparse() -> argparse.ArgumentParser:
"""Build argparse.ArgumentParser with specific configuration."""
parser = argparse.ArgumentParser(prog="release")
subparsers = parser.add_subparsers(
title="subcommands",
dest="subcommand",
help="Choose one of the subcommand you want to run.",
)
parser_check = subparsers.add_parser(
"changelog", help="Generate changelog from specific milestone."
)
parser_check.set_defaults(func=get_changelog)
parser_prune = subparsers.add_parser(
"contributor", help="List all contributors from specific milestone."
)
parser_prune.set_defaults(func=get_contributor)
parser_prune = subparsers.add_parser(
"cherry-pick",
help="Auto cherry pick pr to current branch from specific milestone.",
)
parser_prune.set_defaults(func=auto_cherry_pick)
return parser
if __name__ == "__main__":
arg_parser = build_argparse()
# args = arg_parser.parse_args(["cherry-pick"])
args = arg_parser.parse_args()
ENV_ACCESS_TOKEN = os.environ.get("GH_ACCESS_TOKEN", None)
ENV_MILESTONE = os.environ.get("GH_REPO_MILESTONE", None)
if ENV_ACCESS_TOKEN is None or ENV_MILESTONE is None:
raise RuntimeError(
"Environment variable `GH_ACCESS_TOKEN` and `GH_REPO_MILESTONE` must provider"
)
print(args.func(ENV_ACCESS_TOKEN, ENV_MILESTONE))

19
tools/release/requirements.txt Normal file
View File

@ -0,0 +1,19 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
requests~=2.28
GitPython~=3.1