From 41b89976aaf49d5084de946e935e7e10a12750a0 Mon Sep 17 00:00:00 2001 From: Yingjie Shang Date: Tue, 26 May 2026 07:30:09 +0800 Subject: [PATCH] perf(health): parallel fetch with errgroup and global rate limiter - Replace sequential state loops with 5 parallel goroutines (errgroup) - Global rate limiter (1000 reqs/10min, burst 5) replaces per-page sleep - sharedState with sync.Mutex for cross-goroutine dedup and counting - Fix getOrCreateUser/getOrCreateTag race conditions (INSERT OR IGNORE) - Remove sleep() and API aggregates verbose output --- go.mod | 4 +- go.sum | 32 ++++++ shortcuts/health/api.go | 5 - shortcuts/health/db.go | 32 +++--- shortcuts/health/health.go | 224 ++++++++++++++++++++++--------------- 5 files changed, 181 insertions(+), 116 deletions(-) diff --git a/go.mod b/go.mod index 7bc2245..476d9ee 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,11 @@ go 1.26.1 require ( github.com/spf13/cobra v1.10.2 github.com/zalando/go-keyring v0.2.8 + golang.org/x/sync v0.20.0 golang.org/x/term v0.41.0 + golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.50.1 ) require ( @@ -25,5 +28,4 @@ require ( modernc.org/libc v1.72.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.50.1 // indirect ) diff --git a/go.sum b/go.sum index 3027683..bd8c592 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -40,21 +44,49 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w= modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/shortcuts/health/api.go b/shortcuts/health/api.go index 6918537..7692e13 100644 --- a/shortcuts/health/api.go +++ b/shortcuts/health/api.go @@ -3,7 +3,6 @@ package health import ( "fmt" "net/url" - "time" "github.com/gitlink-org/gitlink-cli/shortcuts/common" ) @@ -59,7 +58,3 @@ func fetchIssueListPage(ctx *common.RuntimeContext, owner, repo, state string, p issues, _ := data["issues"].([]interface{}) return issues, data } - -func sleep() { - time.Sleep(300 * time.Millisecond) -} diff --git a/shortcuts/health/db.go b/shortcuts/health/db.go index 3ccbcf7..e5c0ea7 100644 --- a/shortcuts/health/db.go +++ b/shortcuts/health/db.go @@ -57,17 +57,15 @@ func getOrCreateUser(db *sql.DB, username string) (int, error) { if username == "" { return 0, nil } - var id int - err := db.QueryRow("SELECT id FROM users WHERE user_name = ?", username).Scan(&id) - if err == nil { - return id, nil - } - res, err := db.Exec("INSERT INTO users (user_name) VALUES (?)", username) - if err != nil { + if _, err := db.Exec("INSERT OR IGNORE INTO users (user_name) VALUES (?)", username); err != nil { return 0, fmt.Errorf("insert user %q: %w", username, err) } - lastID, _ := res.LastInsertId() - return int(lastID), nil + var id int + err := db.QueryRow("SELECT id FROM users WHERE user_name = ?", username).Scan(&id) + if err != nil { + return 0, fmt.Errorf("get user %q: %w", username, err) + } + return id, nil } func getOrCreateRepo(db *sql.DB, repoName, owner string) (int, error) { @@ -89,17 +87,15 @@ func getOrCreateRepo(db *sql.DB, repoName, owner string) (int, error) { } func getOrCreateTag(db *sql.DB, repoID int, tagName string) (int, error) { - var id int - err := db.QueryRow("SELECT id FROM tags WHERE repo_id = ? AND name = ?", repoID, tagName).Scan(&id) - if err == nil { - return id, nil - } - res, err := db.Exec("INSERT INTO tags (repo_id, name) VALUES (?, ?)", repoID, tagName) - if err != nil { + if _, err := db.Exec("INSERT OR IGNORE INTO tags (repo_id, name) VALUES (?, ?)", repoID, tagName); err != nil { return 0, fmt.Errorf("insert tag %q for repo %d: %w", tagName, repoID, err) } - lastID, _ := res.LastInsertId() - return int(lastID), nil + var id int + err := db.QueryRow("SELECT id FROM tags WHERE repo_id = ? AND name = ?", repoID, tagName).Scan(&id) + if err != nil { + return 0, fmt.Errorf("get tag %q for repo %d: %w", tagName, repoID, err) + } + return id, nil } func savePullTags(db *sql.DB, pullID int, tagIDs []int) { diff --git a/shortcuts/health/health.go b/shortcuts/health/health.go index fede76f..516e6d7 100644 --- a/shortcuts/health/health.go +++ b/shortcuts/health/health.go @@ -1,13 +1,18 @@ package health import ( + "context" + "database/sql" "fmt" "os" "path/filepath" "strconv" + "sync" "github.com/gitlink-org/gitlink-cli/internal/i18n" "github.com/gitlink-org/gitlink-cli/shortcuts/common" + "golang.org/x/sync/errgroup" + "golang.org/x/time/rate" ) func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { @@ -24,7 +29,6 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { return err } - // maxPages: 0 means unlimited maxPages := 0 if v := ctx.Arg("max-pages"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { @@ -51,109 +55,145 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { return fmt.Errorf("create repo: %w", err) } - // ── Fetch PRs ── + limiter := rate.NewLimiter(rate.Limit(1.0/0.6), 5) + + s := &sharedState{ + seenPRs: make(map[int]bool), + seenIssues: make(map[int]bool), + } + fmt.Fprintf(os.Stderr, "\n=== Fetching Pull Requests ===\n") - seenPRIDs := make(map[int]bool) - var prAgg map[string]interface{} + fmt.Fprintf(os.Stderr, "=== Fetching Issues ===\n") + + g, egCtx := errgroup.WithContext(context.Background()) for _, state := range []string{"open", "closed", "merged"} { - page := 1 - for maxPages == 0 || page <= maxPages { - fmt.Fprintf(os.Stderr, " PR list: state=%s, page=%d...\n", state, page) - prs, agg := fetchPRListPage(ctx, state, page, 20) - if agg != nil { - prAgg = agg - } - if len(prs) == 0 { - break - } - for _, item := range prs { - pr, ok := item.(map[string]interface{}) - if !ok { - continue - } - var prID int - if v, ok := pr["pull_request_id"].(float64); ok && v > 0 { - prID = int(v) - } else if v, ok := pr["id"].(float64); ok { - prID = int(v) - } else { - continue - } - if seenPRIDs[prID] { - continue - } - seenPRIDs[prID] = true - savePull(db, repoID, pr) - } - if len(prs) < 20 { - break - } - page++ - sleep() - } + state := state + g.Go(func() error { + return fetchPRs(ctx, db, repoID, state, maxPages, limiter, egCtx, s) + }) } - - fmt.Fprintf(os.Stderr, " Total PRs: %d\n", len(seenPRIDs)) - if prAgg != nil { - fmt.Fprintf(os.Stderr, " API aggregates: total=%v, merged=%v, open=%v, closed=%v\n", - prAgg["search_count"], prAgg["merged_issues_size"], - prAgg["open_count"], prAgg["close_count"]) - } - - // ── Fetch Issues ── - fmt.Fprintf(os.Stderr, "\n=== Fetching Issues ===\n") - seenIssueIDs := make(map[int]bool) - var issueAgg map[string]interface{} - for _, state := range []string{"open", "closed"} { - page := 1 - for maxPages == 0 || page <= maxPages { - fmt.Fprintf(os.Stderr, " Issue list: state=%s, page=%d...\n", state, page) - issues, agg := fetchIssueListPage(ctx, ctx.Owner, ctx.Repo, state, page, 20) - if agg != nil { - issueAgg = agg - } - if len(issues) == 0 { - break - } - for _, item := range issues { - issue, ok := item.(map[string]interface{}) - if !ok { - continue - } - issueID, ok := issue["id"].(float64) - if !ok || seenIssueIDs[int(issueID)] { - continue - } - seenIssueIDs[int(issueID)] = true - - // Get issue number (project_issues_index) from list item - issueNumber := int(issueID) - if v, ok := issue["project_issues_index"].(float64); ok && v > 0 { - issueNumber = int(v) - } - - listUpdatedAt, _ := issue["updated_at"].(string) - saveIssue(db, repoID, issue, issueNumber, listUpdatedAt) - } - if len(issues) < 20 { - break - } - page++ - sleep() - } + state := state + g.Go(func() error { + return fetchIssues(ctx, db, repoID, state, maxPages, limiter, egCtx, s) + }) } - fmt.Fprintf(os.Stderr, " Total Issues: %d\n", len(seenIssueIDs)) - if issueAgg != nil { - fmt.Fprintf(os.Stderr, " API aggregates: total=%v, open=%v, closed=%v\n", - issueAgg["total_count"], issueAgg["opened_count"], issueAgg["closed_count"]) + if err := g.Wait(); err != nil { + return err } + fmt.Fprintf(os.Stderr, " Total PRs: %d\n", s.prCount) + + fmt.Fprintf(os.Stderr, " Total Issues: %d\n", s.issueCount) + fmt.Fprintf(os.Stderr, "\nData saved to %s\n", dbPath) return nil }, }, } } + +type sharedState struct { + mu sync.Mutex + seenPRs map[int]bool + seenIssues map[int]bool + prCount int + issueCount int +} + +func fetchPRs(ctx *common.RuntimeContext, db *sql.DB, repoID int, state string, maxPages int, limiter *rate.Limiter, egCtx context.Context, s *sharedState) error { + page := 1 + for maxPages == 0 || page <= maxPages { + if egCtx.Err() != nil { + return nil + } + if err := limiter.Wait(egCtx); err != nil { + return nil + } + fmt.Fprintf(os.Stderr, " PR list: state=%s, page=%d...\n", state, page) + prs, _ := fetchPRListPage(ctx, state, page, 20) + if len(prs) == 0 { + break + } + for _, item := range prs { + pr, ok := item.(map[string]interface{}) + if !ok { + continue + } + var prID int + if v, ok := pr["pull_request_id"].(float64); ok && v > 0 { + prID = int(v) + } else if v, ok := pr["id"].(float64); ok { + prID = int(v) + } else { + continue + } + s.mu.Lock() + dup := s.seenPRs[prID] + if !dup { + s.seenPRs[prID] = true + s.prCount++ + } + s.mu.Unlock() + if dup { + continue + } + savePull(db, repoID, pr) + } + if len(prs) < 20 { + break + } + page++ + } + return nil +} + +func fetchIssues(ctx *common.RuntimeContext, db *sql.DB, repoID int, state string, maxPages int, limiter *rate.Limiter, egCtx context.Context, s *sharedState) error { + page := 1 + for maxPages == 0 || page <= maxPages { + if egCtx.Err() != nil { + return nil + } + if err := limiter.Wait(egCtx); err != nil { + return nil + } + fmt.Fprintf(os.Stderr, " Issue list: state=%s, page=%d...\n", state, page) + issues, _ := fetchIssueListPage(ctx, ctx.Owner, ctx.Repo, state, page, 20) + if len(issues) == 0 { + break + } + for _, item := range issues { + issue, ok := item.(map[string]interface{}) + if !ok { + continue + } + issueID, ok := issue["id"].(float64) + if !ok { + continue + } + s.mu.Lock() + dup := s.seenIssues[int(issueID)] + if !dup { + s.seenIssues[int(issueID)] = true + s.issueCount++ + } + s.mu.Unlock() + if dup { + continue + } + issueNumber := int(issueID) + if v, ok := issue["project_issues_index"].(float64); ok && v > 0 { + issueNumber = int(v) + } + listUpdatedAt, _ := issue["updated_at"].(string) + saveIssue(db, repoID, issue, issueNumber, listUpdatedAt) + } + if len(issues) < 20 { + break + } + page++ + } + return nil +}