diff --git a/drivers/139/driver.go b/drivers/139/driver.go
index a647fed94a..7af8c7d238 100644
--- a/drivers/139/driver.go
+++ b/drivers/139/driver.go
@@ -802,32 +802,31 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return err
}
- // 先上传前100个分片
- err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, ss, p)
- if err != nil {
- return err
- }
-
- // 如果还有剩余分片,分批获取上传地址并上传
- for i := 100; i < len(partInfos); i += 100 {
- end := min(i+100, len(partInfos))
- batchPartInfos := partInfos[i:end]
- moredata := base.Json{
+ fetchUploadURLs := func(parts []PartInfo) ([]PersonalPartInfo, error) {
+ payload := base.Json{
"fileId": resp.Data.FileId,
"uploadId": resp.Data.UploadId,
- "partInfos": batchPartInfos,
+ "partInfos": parts,
"commonAccountInfo": base.Json{
"account": d.getAccount(),
"accountType": 1,
},
}
- var moreresp PersonalUploadUrlResp
- _, err = d.newPost(getUploadUrlPath, moredata, &moreresp)
+ var urls PersonalUploadUrlResp
+ _, err := d.newPost(getUploadUrlPath, payload, &urls)
+ return urls.Data.PartInfos, err
+ }
+
+ if err := d.uploadPersonalParts(ctx, firstPartInfos, resp.Data.PartInfos, fetchUploadURLs, ss, p); err != nil {
+ return err
+ }
+ for i := len(firstPartInfos); i < len(partInfos); i += 100 {
+ batch := partInfos[i:min(i+100, len(partInfos))]
+ urls, err := fetchUploadURLs(batch)
if err != nil {
return err
}
- err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, ss, p)
- if err != nil {
+ if err := d.uploadPersonalParts(ctx, batch, urls, fetchUploadURLs, ss, p); err != nil {
return err
}
}
diff --git a/drivers/139/upload_test.go b/drivers/139/upload_test.go
new file mode 100644
index 0000000000..bef0613271
--- /dev/null
+++ b/drivers/139/upload_test.go
@@ -0,0 +1,116 @@
+package _139
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/OpenListTeam/OpenList/v4/drivers/base"
+ "github.com/OpenListTeam/OpenList/v4/internal/driver"
+)
+
+type testUploadSections struct {
+ *strings.Reader
+}
+
+func (s testUploadSections) GetSectionReader(offset, length int64) (io.ReadSeeker, error) {
+ return io.NewSectionReader(s.Reader, offset, length), nil
+}
+
+func (testUploadSections) FreeSectionReader(io.ReadSeeker) {}
+
+func (testUploadSections) DiscardSection(int64, int64) error { return nil }
+
+func TestUploadPersonalPartsRenewsExpiredBatch(t *testing.T) {
+ var mu sync.Mutex
+ var requests []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Error(err)
+ }
+ mu.Lock()
+ requests = append(requests, fmt.Sprintf("%s %s %s", r.Method, r.URL.Path, body))
+ mu.Unlock()
+ if r.URL.Path == "/expired" {
+ w.WriteHeader(http.StatusForbidden)
+ fmt.Fprint(w, "AccessDeniedRequest has expired")
+ return
+ }
+ if r.URL.Path == "/stale-third" {
+ t.Error("the third part must use the renewed URL")
+ w.WriteHeader(http.StatusBadRequest)
+ }
+ }))
+ defer server.Close()
+ useUploadTestClient(t, server.Client())
+
+ parts := []PartInfo{
+ {PartNumber: 101, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 0}},
+ {PartNumber: 102, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 2}},
+ {PartNumber: 103, PartSize: 1, ParallelHashCtx: ParallelHashCtx{PartOffset: 4}},
+ }
+ urls := []PersonalPartInfo{
+ {PartNumber: 102, UploadUrl: server.URL + "/expired"},
+ {PartNumber: 103, UploadUrl: server.URL + "/stale-third"},
+ {PartNumber: 101, UploadUrl: server.URL + "/first"},
+ }
+ refreshes := 0
+ fetchURLs := func(pending []PartInfo) ([]PersonalPartInfo, error) {
+ refreshes++
+ if !reflect.DeepEqual(pending, parts[1:]) {
+ t.Fatalf("renewal requested the wrong parts: %+v", pending)
+ }
+ return []PersonalPartInfo{
+ {PartNumber: 103, UploadUrl: server.URL + "/fresh-third"},
+ {PartNumber: 102, UploadUrl: server.URL + "/fresh-second"},
+ }, nil
+ }
+ progress := driver.NewProgress(5, func(float64) {})
+ sections := testUploadSections{strings.NewReader("abcde")}
+ if err := (&Yun139{}).uploadPersonalParts(context.Background(), parts, urls, fetchURLs, sections, progress); err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"PUT /first ab", "PUT /expired cd", "PUT /fresh-second cd", "PUT /fresh-third e"}
+ mu.Lock()
+ defer mu.Unlock()
+ if !reflect.DeepEqual(requests, want) || refreshes != 1 || progress.Done != 5 {
+ t.Fatalf("requests = %v, refreshes = %d, uploaded bytes = %d", requests, refreshes, progress.Done)
+ }
+}
+
+func TestPutPersonalPartDistinguishesExpiredAndDenied(t *testing.T) {
+ for _, tt := range []struct {
+ name, body string
+ expired bool
+ }{
+ {"expired", "RequestExpired", true},
+ {"denied", "AccessDeniedAccess denied", false},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ fmt.Fprint(w, tt.body)
+ }))
+ defer server.Close()
+ useUploadTestClient(t, server.Client())
+ err := putPersonalPart(context.Background(), server.URL, 1, strings.NewReader("x"))
+ if err == nil || (err == errUploadURLExpired) != tt.expired {
+ t.Fatalf("got %v, want expired = %t", err, tt.expired)
+ }
+ })
+ }
+}
+
+func useUploadTestClient(t *testing.T, client *http.Client) {
+ t.Helper()
+ previous := base.HttpClient
+ base.HttpClient = client
+ t.Cleanup(func() { base.HttpClient = previous })
+}
diff --git a/drivers/139/util.go b/drivers/139/util.go
index a53f99e951..7eff4389e7 100644
--- a/drivers/139/util.go
+++ b/drivers/139/util.go
@@ -1078,70 +1078,102 @@ func (d *Yun139) getGroupCloudHost() string {
return d.GroupCloudHost
}
-func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, ss streamPkg.StreamSectionReader, p *driver.Progress) error {
- // 确保数组以 PartNumber 从小到大排序
- sort.Slice(uploadPartInfos, func(i, j int) bool {
- return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber
- })
+var errUploadURLExpired = errors.New("139 upload URL expired")
- for _, uploadPartInfo := range uploadPartInfos {
- index := uploadPartInfo.PartNumber - 1
- if index < 0 || index >= len(partInfos) {
- return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos))
+func matchUploadURLs(parts []PartInfo, urls []PersonalPartInfo) error {
+ if len(parts) != len(urls) {
+ return fmt.Errorf("139 returned %d upload URLs for %d parts", len(urls), len(parts))
+ }
+ sort.Slice(urls, func(i, j int) bool {
+ return urls[i].PartNumber < urls[j].PartNumber
+ })
+ for i, part := range parts {
+ if int64(urls[i].PartNumber) != part.PartNumber || urls[i].UploadUrl == "" {
+ return fmt.Errorf("139 did not return a valid upload URL for part %d", part.PartNumber)
}
- partSize := partInfos[index].PartSize
- offset := partInfos[index].ParallelHashCtx.PartOffset
- log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos))
+ }
+ return nil
+}
- rd, getErr := ss.GetSectionReader(offset, partSize)
- if getErr != nil {
- return getErr
+func (d *Yun139) uploadPersonalParts(ctx context.Context, parts []PartInfo, urls []PersonalPartInfo, fetchURLs func([]PartInfo) ([]PersonalPartInfo, error), ss streamPkg.StreamSectionReader, p *driver.Progress) error {
+ if err := matchUploadURLs(parts, urls); err != nil {
+ return err
+ }
+ for i, part := range parts {
+ rd, err := ss.GetSectionReader(part.ParallelHashCtx.PartOffset, part.PartSize)
+ if err != nil {
+ return err
}
- // Save progress before this part so retries don't double-count bytes
- partDoneStart := p.Done
- err := retry.Do(
- func() error {
- // Reset progress to the start of this part on each attempt
- p.Done = partDoneStart
- if _, seekErr := rd.Seek(0, io.SeekStart); seekErr != nil {
- return seekErr
- }
- req, reqErr := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, io.TeeReader(rd, p))
- if reqErr != nil {
- return reqErr
- }
- req.Header.Set("Content-Type", "application/octet-stream")
- req.Header.Set("Content-Length", fmt.Sprint(partSize))
- req.Header.Set("Origin", "https://yun.139.com")
- req.Header.Set("Referer", "https://yun.139.com/")
- req.ContentLength = partSize
-
- res, doErr := base.HttpClient.Do(req)
- if doErr != nil {
- return doErr
+ start := p.Done
+ upload := func(uploadURL string) error {
+ return retry.Do(func() error {
+ p.Done = start
+ if _, err := rd.Seek(0, io.SeekStart); err != nil {
+ return retry.Unrecoverable(err)
}
- defer res.Body.Close()
- log.Debugf("[139] uploaded: %+v", res)
- if res.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(res.Body)
- return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body))
+ err := putPersonalPart(ctx, uploadURL, part.PartSize, io.TeeReader(rd, p))
+ if errors.Is(err, errUploadURLExpired) {
+ return retry.Unrecoverable(err)
}
- return nil
- },
- retry.Context(ctx),
- retry.Attempts(3),
- retry.DelayType(retry.BackOffDelay),
- retry.Delay(time.Second),
- )
+ return err
+ }, retry.Context(ctx), retry.Attempts(3), retry.LastErrorOnly(true), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second))
+ }
+
+ err = upload(urls[i].UploadUrl)
+ if errors.Is(err, errUploadURLExpired) && ctx.Err() == nil {
+ p.Done = start
+ // Renew this part and the rest of the batch under the same upload ID.
+ var refreshed []PersonalPartInfo
+ refreshed, err = fetchURLs(parts[i:])
+ if err == nil {
+ err = matchUploadURLs(parts[i:], refreshed)
+ }
+ if err == nil {
+ copy(urls[i:], refreshed)
+ err = upload(urls[i].UploadUrl)
+ }
+ }
ss.FreeSectionReader(rd)
if err != nil {
- return err
+ return fmt.Errorf("upload 139 part %d: %w", part.PartNumber, err)
}
}
return nil
}
+func putPersonalPart(ctx context.Context, uploadURL string, size int64, body io.Reader) error {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, body)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", "application/octet-stream")
+ req.Header.Set("Origin", "https://yun.139.com")
+ req.Header.Set("Referer", "https://yun.139.com/")
+ req.ContentLength = size
+
+ res, err := base.HttpClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+ if res.StatusCode == http.StatusOK {
+ return nil
+ }
+ raw, _ := io.ReadAll(res.Body)
+ if res.StatusCode == http.StatusForbidden {
+ var eosError struct {
+ Code string
+ Message string
+ }
+ if xml.Unmarshal(raw, &eosError) == nil &&
+ (eosError.Code == "RequestExpired" || (eosError.Code == "AccessDenied" && eosError.Message == "Request has expired")) {
+ return errUploadURLExpired
+ }
+ }
+ return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(raw))
+}
+
func (d *Yun139) getDiskQuotaDetail(ctx context.Context) (*DiskQuotaDetail, error) {
data := map[string]interface{}{
"userDomainId": d.UserDomainID,