feat(attachment): +upload 支持多文件并发上传(-c 并发数,默认 3)
This commit is contained in:
parent
34c8a0f0b6
commit
682604f238
|
|
@ -481,6 +481,9 @@ gitlink-cli release +delete --owner Gitlink --repo forgeplus -i <version_id> --d
|
|||
# Upload a local file as a platform attachment (returns the attachment id)
|
||||
gitlink-cli attachment +upload -f ./dist/app-v1.0.0.tar.gz -d "v1.0.0 release asset"
|
||||
|
||||
# Upload several files concurrently (comma-separated; -c sets the worker count, default 3)
|
||||
gitlink-cli attachment +upload -f ./dist/app.tar.gz,./dist/app.sha256,./dist/CHANGELOG.md -c 3
|
||||
|
||||
# Download an attachment by id to a local file
|
||||
gitlink-cli attachment +download -i <attachment_id> -o ./app-v1.0.0.tar.gz
|
||||
|
||||
|
|
|
|||
|
|
@ -472,6 +472,9 @@ gitlink-cli release +delete --owner Gitlink --repo forgeplus -i <version_id> --d
|
|||
# 上传本地文件为平台附件(返回附件 id)
|
||||
gitlink-cli attachment +upload -f ./dist/app-v1.0.0.tar.gz -d "v1.0.0 发布产物"
|
||||
|
||||
# 多文件并发上传(逗号分隔;-c 指定并发数,默认 3)
|
||||
gitlink-cli attachment +upload -f ./dist/app.tar.gz,./dist/app.sha256,./dist/CHANGELOG.md -c 3
|
||||
|
||||
# 按 id 下载附件到本地文件
|
||||
gitlink-cli attachment +download -i <attachment_id> -o ./app-v1.0.0.tar.gz
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ type Client struct {
|
|||
HTTP *http.Client
|
||||
BaseURL string
|
||||
Debug bool
|
||||
// NoProgress suppresses per-byte transfer progress on stderr; used when
|
||||
// several transfers run concurrently and interleaved lines would garble.
|
||||
NoProgress bool
|
||||
}
|
||||
|
||||
type APIError struct {
|
||||
|
|
|
|||
|
|
@ -30,6 +30,14 @@ type progressReporter struct {
|
|||
out io.Writer
|
||||
}
|
||||
|
||||
// transferProgress builds a progress reporter honoring Client.NoProgress.
|
||||
func (c *Client) transferProgress(verb, name string, total int64) *progressReporter {
|
||||
if c.NoProgress {
|
||||
return newProgressReporterTo(verb, name, total, io.Discard)
|
||||
}
|
||||
return newProgressReporter(verb, name, total)
|
||||
}
|
||||
|
||||
func newProgressReporter(verb, name string, total int64) *progressReporter {
|
||||
return &progressReporter{verb: verb, name: name, total: total, lastPct: -1, out: os.Stderr}
|
||||
}
|
||||
|
|
@ -110,7 +118,7 @@ func (c *Client) PostMultipartFile(path, filePath, fileField string, fields map[
|
|||
// are never buffered in memory.
|
||||
pr, pw := io.Pipe()
|
||||
writer := multipart.NewWriter(pw)
|
||||
progress := newProgressReporter("uploading", filepath.Base(filePath), info.Size())
|
||||
progress := c.transferProgress("uploading", filepath.Base(filePath), info.Size())
|
||||
go func() {
|
||||
part, err := writer.CreateFormFile(fileField, filepath.Base(filePath))
|
||||
if err != nil {
|
||||
|
|
@ -243,7 +251,7 @@ func (c *Client) DownloadFile(path, destPath string) (int64, error) {
|
|||
}
|
||||
defer out.Close()
|
||||
|
||||
progress := newProgressReporter("downloading", filepath.Base(destPath), resp.ContentLength)
|
||||
progress := c.transferProgress("downloading", filepath.Base(destPath), resp.ContentLength)
|
||||
n, err := io.Copy(out, io.TeeReader(resp.Body, progress))
|
||||
progress.Close()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@
|
|||
"flag.api.body_stdin": "Read request body JSON from stdin",
|
||||
"flag.api.header": "Additional headers (key:value)",
|
||||
"flag.api.query": "Query parameters (key=val&key2=val2)",
|
||||
"flag.attachment.concurrency": "Concurrent uploads when passing multiple comma-separated files (default 3)",
|
||||
"flag.attachment.description": "Attachment description",
|
||||
"flag.attachment.file": "Path of the local file to upload",
|
||||
"flag.attachment.id": "Attachment ID",
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@
|
|||
"flag.api.body_stdin": "从标准输入读取 JSON 请求体",
|
||||
"flag.api.header": "附加请求头(key:value)",
|
||||
"flag.api.query": "查询参数(key=val&key2=val2)",
|
||||
"flag.attachment.concurrency": "多文件(逗号分隔)上传时的并发数(默认 3)",
|
||||
"flag.attachment.description": "附件描述",
|
||||
"flag.attachment.file": "要上传的本地文件路径",
|
||||
"flag.attachment.id": "附件 ID",
|
||||
|
|
|
|||
|
|
@ -12,11 +12,70 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gitlink-org/gitlink-cli/internal/i18n"
|
||||
"github.com/gitlink-org/gitlink-cli/internal/output"
|
||||
"github.com/gitlink-org/gitlink-cli/shortcuts/common"
|
||||
)
|
||||
|
||||
// splitFiles parses a comma-separated file list, trimming blanks.
|
||||
func splitFiles(arg string) []string {
|
||||
var files []string
|
||||
for _, f := range strings.Split(arg, ",") {
|
||||
if f = strings.TrimSpace(f); f != "" {
|
||||
files = append(files, f)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// uploadConcurrently uploads several files with a bounded worker pool.
|
||||
// Per-byte progress is suppressed (interleaved lines would garble); instead
|
||||
// one line per completed file goes to stderr. Results keep input order.
|
||||
func uploadConcurrently(ctx *common.RuntimeContext, files []string, fields map[string]string, concurrency int) ([]interface{}, error) {
|
||||
quiet := *ctx.Client
|
||||
quiet.NoProgress = true
|
||||
|
||||
type result struct {
|
||||
env *output.Envelope
|
||||
err error
|
||||
}
|
||||
results := make([]result, len(files))
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
for i, file := range files {
|
||||
wg.Add(1)
|
||||
go func(i int, file string) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
env, err := quiet.PostMultipartFile("/attachments", file, "file", fields)
|
||||
results[i] = result{env: env, err: err}
|
||||
mu.Lock()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "uploaded %s: error: %v\n", filepath.Base(file), err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "uploaded %s\n", filepath.Base(file))
|
||||
}
|
||||
mu.Unlock()
|
||||
}(i, file)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
out := make([]interface{}, 0, len(files))
|
||||
for i, r := range results {
|
||||
if r.err != nil {
|
||||
return nil, fmt.Errorf("upload %q failed: %w", files[i], r.err)
|
||||
}
|
||||
out = append(out, map[string]interface{}{"file": files[i], "result": r.env.Data})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Shortcuts returns attachment upload/download shortcuts.
|
||||
func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut {
|
||||
tr := i18n.Default()
|
||||
|
|
@ -32,27 +91,42 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut {
|
|||
Flags: []common.Flag{
|
||||
{Name: "file", Short: "f", Usage: tr.T("flag.attachment.file"), Required: true},
|
||||
{Name: "description", Short: "d", Usage: tr.T("flag.attachment.description")},
|
||||
{Name: "concurrency", Short: "c", Usage: tr.T("flag.attachment.concurrency"), Default: "3"},
|
||||
},
|
||||
Run: func(ctx *common.RuntimeContext) error {
|
||||
file, err := ctx.RequireArg("file")
|
||||
fileArg, err := ctx.RequireArg("file")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot access file %q: %w", file, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("%q is a directory, expected a file", file)
|
||||
files := splitFiles(fileArg)
|
||||
for _, file := range files {
|
||||
info, err := os.Stat(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot access file %q: %w", file, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("%q is a directory, expected a file", file)
|
||||
}
|
||||
}
|
||||
fields := map[string]string{
|
||||
"description": ctx.Arg("description"),
|
||||
}
|
||||
env, err := ctx.Client.PostMultipartFile("/attachments", file, "file", fields)
|
||||
if len(files) == 1 {
|
||||
env, err := ctx.Client.PostMultipartFile("/attachments", files[0], "file", fields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Output(env)
|
||||
}
|
||||
concurrency, err := strconv.Atoi(ctx.Arg("concurrency"))
|
||||
if err != nil || concurrency < 1 {
|
||||
concurrency = 3
|
||||
}
|
||||
results, err := uploadConcurrently(ctx, files, fields, concurrency)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Output(env)
|
||||
return ctx.OutputData(results)
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ package attachment
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gitlink-org/gitlink-cli/internal/client"
|
||||
|
|
@ -211,3 +214,57 @@ func TestDeleteAttachment(t *testing.T) {
|
|||
t.Fatalf("path = %q, want /attachments/abc-uuid.json", gotPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitFiles(t *testing.T) {
|
||||
got := splitFiles(" a.txt, b.bin ,,c ")
|
||||
want := []string{"a.txt", "b.bin", "c"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("splitFiles = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("splitFiles[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMultipleFilesConcurrently(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var files []string
|
||||
for _, name := range []string{"one.txt", "two.txt", "three.txt"} {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte("data-"+name), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files = append(files, p)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
seen := map[string]bool{}
|
||||
ctx := newTestContext(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/attachments" && r.URL.Path != "/attachments.json" {
|
||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
_, hdr, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatalf("form file: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
seen[hdr.Filename] = true
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"id":%q,"msg":"success"}`, hdr.Filename)
|
||||
}, map[string]string{
|
||||
"file": strings.Join(files, ","),
|
||||
"concurrency": "2",
|
||||
})
|
||||
if err := findShortcut(t, "upload").Run(ctx); err != nil {
|
||||
t.Fatalf("multi-file upload failed: %v", err)
|
||||
}
|
||||
if len(seen) != 3 {
|
||||
t.Fatalf("uploaded %d files, want 3: %v", len(seen), seen)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue