perf(health): parallel fetch with errgroup and global rate limiter
- Replace sequential state loops with 5 parallel goroutines (errgroup) - Global rate limiter (1000 reqs/10min, burst 5) replaces per-page sleep - sharedState with sync.Mutex for cross-goroutine dedup and counting - Fix getOrCreateUser/getOrCreateTag race conditions (INSERT OR IGNORE) - Remove sleep() and API aggregates verbose output
This commit is contained in:
parent
0a578bbab0
commit
41b89976aa
4
go.mod
4
go.mod
|
|
@ -5,8 +5,11 @@ go 1.26.1
|
|||
require (
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/zalando/go-keyring v0.2.8
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/term v0.41.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.50.1
|
||||
)
|
||||
|
||||
require (
|
||||
|
|
@ -25,5 +28,4 @@ require (
|
|||
modernc.org/libc v1.72.3 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.50.1 // indirect
|
||||
)
|
||||
|
|
|
|||
32
go.sum
32
go.sum
|
|
@ -8,8 +8,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
|||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ=
|
||||
github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
|
|
@ -40,21 +44,49 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
|
|||
github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs=
|
||||
github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY=
|
||||
modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ=
|
||||
modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||
modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU=
|
||||
modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w=
|
||||
modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package health
|
|||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/gitlink-org/gitlink-cli/shortcuts/common"
|
||||
)
|
||||
|
|
@ -59,7 +58,3 @@ func fetchIssueListPage(ctx *common.RuntimeContext, owner, repo, state string, p
|
|||
issues, _ := data["issues"].([]interface{})
|
||||
return issues, data
|
||||
}
|
||||
|
||||
func sleep() {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,17 +57,15 @@ func getOrCreateUser(db *sql.DB, username string) (int, error) {
|
|||
if username == "" {
|
||||
return 0, nil
|
||||
}
|
||||
var id int
|
||||
err := db.QueryRow("SELECT id FROM users WHERE user_name = ?", username).Scan(&id)
|
||||
if err == nil {
|
||||
return id, nil
|
||||
}
|
||||
res, err := db.Exec("INSERT INTO users (user_name) VALUES (?)", username)
|
||||
if err != nil {
|
||||
if _, err := db.Exec("INSERT OR IGNORE INTO users (user_name) VALUES (?)", username); err != nil {
|
||||
return 0, fmt.Errorf("insert user %q: %w", username, err)
|
||||
}
|
||||
lastID, _ := res.LastInsertId()
|
||||
return int(lastID), nil
|
||||
var id int
|
||||
err := db.QueryRow("SELECT id FROM users WHERE user_name = ?", username).Scan(&id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("get user %q: %w", username, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func getOrCreateRepo(db *sql.DB, repoName, owner string) (int, error) {
|
||||
|
|
@ -89,17 +87,15 @@ func getOrCreateRepo(db *sql.DB, repoName, owner string) (int, error) {
|
|||
}
|
||||
|
||||
func getOrCreateTag(db *sql.DB, repoID int, tagName string) (int, error) {
|
||||
var id int
|
||||
err := db.QueryRow("SELECT id FROM tags WHERE repo_id = ? AND name = ?", repoID, tagName).Scan(&id)
|
||||
if err == nil {
|
||||
return id, nil
|
||||
}
|
||||
res, err := db.Exec("INSERT INTO tags (repo_id, name) VALUES (?, ?)", repoID, tagName)
|
||||
if err != nil {
|
||||
if _, err := db.Exec("INSERT OR IGNORE INTO tags (repo_id, name) VALUES (?, ?)", repoID, tagName); err != nil {
|
||||
return 0, fmt.Errorf("insert tag %q for repo %d: %w", tagName, repoID, err)
|
||||
}
|
||||
lastID, _ := res.LastInsertId()
|
||||
return int(lastID), nil
|
||||
var id int
|
||||
err := db.QueryRow("SELECT id FROM tags WHERE repo_id = ? AND name = ?", repoID, tagName).Scan(&id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("get tag %q for repo %d: %w", tagName, repoID, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func savePullTags(db *sql.DB, pullID int, tagIDs []int) {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,18 @@
|
|||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/gitlink-org/gitlink-cli/internal/i18n"
|
||||
"github.com/gitlink-org/gitlink-cli/shortcuts/common"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut {
|
||||
|
|
@ -24,7 +29,6 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut {
|
|||
return err
|
||||
}
|
||||
|
||||
// maxPages: 0 means unlimited
|
||||
maxPages := 0
|
||||
if v := ctx.Arg("max-pages"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
|
|
@ -51,109 +55,145 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut {
|
|||
return fmt.Errorf("create repo: %w", err)
|
||||
}
|
||||
|
||||
// ── Fetch PRs ──
|
||||
limiter := rate.NewLimiter(rate.Limit(1.0/0.6), 5)
|
||||
|
||||
s := &sharedState{
|
||||
seenPRs: make(map[int]bool),
|
||||
seenIssues: make(map[int]bool),
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n=== Fetching Pull Requests ===\n")
|
||||
seenPRIDs := make(map[int]bool)
|
||||
var prAgg map[string]interface{}
|
||||
fmt.Fprintf(os.Stderr, "=== Fetching Issues ===\n")
|
||||
|
||||
g, egCtx := errgroup.WithContext(context.Background())
|
||||
|
||||
for _, state := range []string{"open", "closed", "merged"} {
|
||||
page := 1
|
||||
for maxPages == 0 || page <= maxPages {
|
||||
fmt.Fprintf(os.Stderr, " PR list: state=%s, page=%d...\n", state, page)
|
||||
prs, agg := fetchPRListPage(ctx, state, page, 20)
|
||||
if agg != nil {
|
||||
prAgg = agg
|
||||
}
|
||||
if len(prs) == 0 {
|
||||
break
|
||||
}
|
||||
for _, item := range prs {
|
||||
pr, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var prID int
|
||||
if v, ok := pr["pull_request_id"].(float64); ok && v > 0 {
|
||||
prID = int(v)
|
||||
} else if v, ok := pr["id"].(float64); ok {
|
||||
prID = int(v)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
if seenPRIDs[prID] {
|
||||
continue
|
||||
}
|
||||
seenPRIDs[prID] = true
|
||||
savePull(db, repoID, pr)
|
||||
}
|
||||
if len(prs) < 20 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
sleep()
|
||||
}
|
||||
state := state
|
||||
g.Go(func() error {
|
||||
return fetchPRs(ctx, db, repoID, state, maxPages, limiter, egCtx, s)
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, " Total PRs: %d\n", len(seenPRIDs))
|
||||
if prAgg != nil {
|
||||
fmt.Fprintf(os.Stderr, " API aggregates: total=%v, merged=%v, open=%v, closed=%v\n",
|
||||
prAgg["search_count"], prAgg["merged_issues_size"],
|
||||
prAgg["open_count"], prAgg["close_count"])
|
||||
}
|
||||
|
||||
// ── Fetch Issues ──
|
||||
fmt.Fprintf(os.Stderr, "\n=== Fetching Issues ===\n")
|
||||
seenIssueIDs := make(map[int]bool)
|
||||
var issueAgg map[string]interface{}
|
||||
|
||||
for _, state := range []string{"open", "closed"} {
|
||||
page := 1
|
||||
for maxPages == 0 || page <= maxPages {
|
||||
fmt.Fprintf(os.Stderr, " Issue list: state=%s, page=%d...\n", state, page)
|
||||
issues, agg := fetchIssueListPage(ctx, ctx.Owner, ctx.Repo, state, page, 20)
|
||||
if agg != nil {
|
||||
issueAgg = agg
|
||||
}
|
||||
if len(issues) == 0 {
|
||||
break
|
||||
}
|
||||
for _, item := range issues {
|
||||
issue, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
issueID, ok := issue["id"].(float64)
|
||||
if !ok || seenIssueIDs[int(issueID)] {
|
||||
continue
|
||||
}
|
||||
seenIssueIDs[int(issueID)] = true
|
||||
|
||||
// Get issue number (project_issues_index) from list item
|
||||
issueNumber := int(issueID)
|
||||
if v, ok := issue["project_issues_index"].(float64); ok && v > 0 {
|
||||
issueNumber = int(v)
|
||||
}
|
||||
|
||||
listUpdatedAt, _ := issue["updated_at"].(string)
|
||||
saveIssue(db, repoID, issue, issueNumber, listUpdatedAt)
|
||||
}
|
||||
if len(issues) < 20 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
sleep()
|
||||
}
|
||||
state := state
|
||||
g.Go(func() error {
|
||||
return fetchIssues(ctx, db, repoID, state, maxPages, limiter, egCtx, s)
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, " Total Issues: %d\n", len(seenIssueIDs))
|
||||
if issueAgg != nil {
|
||||
fmt.Fprintf(os.Stderr, " API aggregates: total=%v, open=%v, closed=%v\n",
|
||||
issueAgg["total_count"], issueAgg["opened_count"], issueAgg["closed_count"])
|
||||
if err := g.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, " Total PRs: %d\n", s.prCount)
|
||||
|
||||
fmt.Fprintf(os.Stderr, " Total Issues: %d\n", s.issueCount)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\nData saved to %s\n", dbPath)
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type sharedState struct {
|
||||
mu sync.Mutex
|
||||
seenPRs map[int]bool
|
||||
seenIssues map[int]bool
|
||||
prCount int
|
||||
issueCount int
|
||||
}
|
||||
|
||||
func fetchPRs(ctx *common.RuntimeContext, db *sql.DB, repoID int, state string, maxPages int, limiter *rate.Limiter, egCtx context.Context, s *sharedState) error {
|
||||
page := 1
|
||||
for maxPages == 0 || page <= maxPages {
|
||||
if egCtx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if err := limiter.Wait(egCtx); err != nil {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " PR list: state=%s, page=%d...\n", state, page)
|
||||
prs, _ := fetchPRListPage(ctx, state, page, 20)
|
||||
if len(prs) == 0 {
|
||||
break
|
||||
}
|
||||
for _, item := range prs {
|
||||
pr, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var prID int
|
||||
if v, ok := pr["pull_request_id"].(float64); ok && v > 0 {
|
||||
prID = int(v)
|
||||
} else if v, ok := pr["id"].(float64); ok {
|
||||
prID = int(v)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
dup := s.seenPRs[prID]
|
||||
if !dup {
|
||||
s.seenPRs[prID] = true
|
||||
s.prCount++
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
savePull(db, repoID, pr)
|
||||
}
|
||||
if len(prs) < 20 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchIssues(ctx *common.RuntimeContext, db *sql.DB, repoID int, state string, maxPages int, limiter *rate.Limiter, egCtx context.Context, s *sharedState) error {
|
||||
page := 1
|
||||
for maxPages == 0 || page <= maxPages {
|
||||
if egCtx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if err := limiter.Wait(egCtx); err != nil {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " Issue list: state=%s, page=%d...\n", state, page)
|
||||
issues, _ := fetchIssueListPage(ctx, ctx.Owner, ctx.Repo, state, page, 20)
|
||||
if len(issues) == 0 {
|
||||
break
|
||||
}
|
||||
for _, item := range issues {
|
||||
issue, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
issueID, ok := issue["id"].(float64)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
dup := s.seenIssues[int(issueID)]
|
||||
if !dup {
|
||||
s.seenIssues[int(issueID)] = true
|
||||
s.issueCount++
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
issueNumber := int(issueID)
|
||||
if v, ok := issue["project_issues_index"].(float64); ok && v > 0 {
|
||||
issueNumber = int(v)
|
||||
}
|
||||
listUpdatedAt, _ := issue["updated_at"].(string)
|
||||
saveIssue(db, repoID, issue, issueNumber, listUpdatedAt)
|
||||
}
|
||||
if len(issues) < 20 {
|
||||
break
|
||||
}
|
||||
page++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue