91 lines
3.0 KiB
Python
91 lines
3.0 KiB
Python
import requests
|
||
import pymysql
|
||
from util import config
|
||
|
||
|
||
# res 7499
|
||
# 通过仓库贡献者的数量是否大于N，结果存入数据库
|
||
def refine_repos_by_contributors():
|
||
# 打开数据库连接
|
||
db = pymysql.connect(**config.db_config)
|
||
# 使用 cursor() 方法创建一个游标对象 cursor
|
||
cursor = db.cursor()
|
||
# SQL查询语句
|
||
sql_query = "SELECT id,name,owner,is_refine2 FROM repos WHERE refine1=1"
|
||
try:
|
||
# 执行SQL语句
|
||
cursor.execute(sql_query)
|
||
# 获取所有记录列表
|
||
results = cursor.fetchall()
|
||
print('length:')
|
||
print(len(results))
|
||
for i in range(0, len(results)):
|
||
# 若该项已经检测过；则本次跳过
|
||
if results[i][3] == 1:
|
||
continue
|
||
# 拼接出 owner/name
|
||
owner_name = results[i][2] + '/' + results[i][1]
|
||
black_list = ['tipeio/tipe', 'jayshah19949596/CodingInterviews', 'microsoft/WSL2-Linux-Kernel',
|
||
'raspberrypi/linux', 'chromium/chromium', 'WorkerLivesMatter/WorkingTime'
|
||
, 'v8/v8', 'torvalds/linux']
|
||
if owner_name in black_list:
|
||
continue
|
||
# 为避免api访问受限，利用多个token进行轮换使用
|
||
# 这里N取为2
|
||
flag = get_repos_contributors_numbers(owner_name, config.token_list[(i//500) % 8],2)
|
||
# SQL更新语句
|
||
sql_update = "UPDATE repos SET is_refine2 = 1,refine2 = '%d' WHERE id = '%d'" % (flag, results[i][0])
|
||
try:
|
||
# 执行SQL语句
|
||
cursor.execute(sql_update)
|
||
# 提交到数据库执行
|
||
db.commit()
|
||
except Exception as e:
|
||
# 如果发生错误则回滚
|
||
db.rollback()
|
||
print("ERR0R1:")
|
||
print(e)
|
||
except Exception as e:
|
||
# 如果发生错误则回滚
|
||
db.rollback()
|
||
print("ERR0R2:")
|
||
print(e)
|
||
finally:
|
||
# 关闭数据库连接
|
||
db.close()
|
||
|
||
|
||
# 功能：通过查询仓库贡献者数量
|
||
# 返回：0 仓库贡献者数量小于N
|
||
# 1 仓库贡献者数量大于N
|
||
def get_repos_contributors_numbers(owner_name, token,N):
|
||
url = 'https://api.github.com/repos/{owner_name}/contributors'
|
||
url = url.format(owner_name=owner_name)
|
||
headers = {'User-Agent': 'Mozilla/5.0',
|
||
'Authorization': 'token '+token,
|
||
'Content-Type': 'application/json',
|
||
'Accept': 'application/json'
|
||
}
|
||
params = {
|
||
'page': 1,
|
||
'per_page': 100,
|
||
}
|
||
response = requests.get(url, headers=headers, params=params)
|
||
if response.status_code != 200:
|
||
print(response.json())
|
||
print(url)
|
||
print('get_repos_contributors_numbers error: fail to request')
|
||
exit(0)
|
||
response = response.json()
|
||
if len(response) > N:
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def run():
|
||
refine_repos_by_contributors()
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run()
|
||
#print('hello world') |