diff --git a/drivers/139/driver.go b/drivers/139/driver.go index a647fed94a..7af8c7d238 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -802,32 +802,31 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } - // 先上传前100个分片 - err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, ss, p) - if err != nil { - return err - } - - // 如果还有剩余分片,分批获取上传地址并上传 - for i := 100; i < len(partInfos); i += 100 { - end := min(i+100, len(partInfos)) - batchPartInfos := partInfos[i:end] - moredata := base.Json{ + fetchUploadURLs := func(parts []PartInfo) ([]PersonalPartInfo, error) { + payload := base.Json{ "fileId": resp.Data.FileId, "uploadId": resp.Data.UploadId, - "partInfos": batchPartInfos, + "partInfos": parts, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } - var moreresp PersonalUploadUrlResp - _, err = d.newPost(getUploadUrlPath, moredata, &moreresp) + var urls PersonalUploadUrlResp + _, err := d.newPost(getUploadUrlPath, payload, &urls) + return urls.Data.PartInfos, err + } + + if err := d.uploadPersonalParts(ctx, firstPartInfos, resp.Data.PartInfos, fetchUploadURLs, ss, p); err != nil { + return err + } + for i := len(firstPartInfos); i < len(partInfos); i += 100 { + batch := partInfos[i:min(i+100, len(partInfos))] + urls, err := fetchUploadURLs(batch) if err != nil { return err } - err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, ss, p) - if err != nil { + if err := d.uploadPersonalParts(ctx, batch, urls, fetchUploadURLs, ss, p); err != nil { return err } } diff --git a/drivers/139/upload_test.go b/drivers/139/upload_test.go new file mode 100644 index 0000000000..bef0613271 --- /dev/null +++ b/drivers/139/upload_test.go @@ -0,0 +1,116 @@ +package _139 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" +) + +type testUploadSections struct { + *strings.Reader +} + +func (s testUploadSections) GetSectionReader(offset, length int64) (io.ReadSeeker, error) { + return io.NewSectionReader(s.Reader, offset, length), nil +} + +func (testUploadSections) FreeSectionReader(io.ReadSeeker) {} + +func (testUploadSections) DiscardSection(int64, int64) error { return nil } + +func TestUploadPersonalPartsRenewsExpiredBatch(t *testing.T) { + var mu sync.Mutex + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + mu.Lock() + requests = append(requests, fmt.Sprintf("%s %s %s", r.Method, r.URL.Path, body)) + mu.Unlock() + if r.URL.Path == "/expired" { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, "AccessDeniedRequest has expired") + return + } + if r.URL.Path == "/stale-third" { + t.Error("the third part must use the renewed URL") + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + useUploadTestClient(t, server.Client()) + + parts := []PartInfo{ + {PartNumber: 101, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 0}}, + {PartNumber: 102, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 2}}, + {PartNumber: 103, PartSize: 1, ParallelHashCtx: ParallelHashCtx{PartOffset: 4}}, + } + urls := []PersonalPartInfo{ + {PartNumber: 102, UploadUrl: server.URL + "/expired"}, + {PartNumber: 103, UploadUrl: server.URL + "/stale-third"}, + {PartNumber: 101, UploadUrl: server.URL + "/first"}, + } + refreshes := 0 + fetchURLs := func(pending []PartInfo) ([]PersonalPartInfo, error) { + refreshes++ + if !reflect.DeepEqual(pending, parts[1:]) { + t.Fatalf("renewal requested the wrong parts: %+v", pending) + } + return []PersonalPartInfo{ + {PartNumber: 103, UploadUrl: server.URL + "/fresh-third"}, + {PartNumber: 102, UploadUrl: server.URL + "/fresh-second"}, + }, nil + } + progress := driver.NewProgress(5, func(float64) {}) + sections := testUploadSections{strings.NewReader("abcde")} + if err := (&Yun139{}).uploadPersonalParts(context.Background(), parts, urls, fetchURLs, sections, progress); err != nil { + t.Fatal(err) + } + want := []string{"PUT /first ab", "PUT /expired cd", "PUT /fresh-second cd", "PUT /fresh-third e"} + mu.Lock() + defer mu.Unlock() + if !reflect.DeepEqual(requests, want) || refreshes != 1 || progress.Done != 5 { + t.Fatalf("requests = %v, refreshes = %d, uploaded bytes = %d", requests, refreshes, progress.Done) + } +} + +func TestPutPersonalPartDistinguishesExpiredAndDenied(t *testing.T) { + for _, tt := range []struct { + name, body string + expired bool + }{ + {"expired", "RequestExpired", true}, + {"denied", "AccessDeniedAccess denied", false}, + } { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, tt.body) + })) + defer server.Close() + useUploadTestClient(t, server.Client()) + err := putPersonalPart(context.Background(), server.URL, 1, strings.NewReader("x")) + if err == nil || (err == errUploadURLExpired) != tt.expired { + t.Fatalf("got %v, want expired = %t", err, tt.expired) + } + }) + } +} + +func useUploadTestClient(t *testing.T, client *http.Client) { + t.Helper() + previous := base.HttpClient + base.HttpClient = client + t.Cleanup(func() { base.HttpClient = previous }) +} diff --git a/drivers/139/util.go b/drivers/139/util.go index a53f99e951..7eff4389e7 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1078,70 +1078,102 @@ func (d *Yun139) getGroupCloudHost() string { return d.GroupCloudHost } -func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, ss streamPkg.StreamSectionReader, p *driver.Progress) error { - // 确保数组以 PartNumber 从小到大排序 - sort.Slice(uploadPartInfos, func(i, j int) bool { - return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber - }) +var errUploadURLExpired = errors.New("139 upload URL expired") - for _, uploadPartInfo := range uploadPartInfos { - index := uploadPartInfo.PartNumber - 1 - if index < 0 || index >= len(partInfos) { - return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos)) +func matchUploadURLs(parts []PartInfo, urls []PersonalPartInfo) error { + if len(parts) != len(urls) { + return fmt.Errorf("139 returned %d upload URLs for %d parts", len(urls), len(parts)) + } + sort.Slice(urls, func(i, j int) bool { + return urls[i].PartNumber < urls[j].PartNumber + }) + for i, part := range parts { + if int64(urls[i].PartNumber) != part.PartNumber || urls[i].UploadUrl == "" { + return fmt.Errorf("139 did not return a valid upload URL for part %d", part.PartNumber) } - partSize := partInfos[index].PartSize - offset := partInfos[index].ParallelHashCtx.PartOffset - log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos)) + } + return nil +} - rd, getErr := ss.GetSectionReader(offset, partSize) - if getErr != nil { - return getErr +func (d *Yun139) uploadPersonalParts(ctx context.Context, parts []PartInfo, urls []PersonalPartInfo, fetchURLs func([]PartInfo) ([]PersonalPartInfo, error), ss streamPkg.StreamSectionReader, p *driver.Progress) error { + if err := matchUploadURLs(parts, urls); err != nil { + return err + } + for i, part := range parts { + rd, err := ss.GetSectionReader(part.ParallelHashCtx.PartOffset, part.PartSize) + if err != nil { + return err } - // Save progress before this part so retries don't double-count bytes - partDoneStart := p.Done - err := retry.Do( - func() error { - // Reset progress to the start of this part on each attempt - p.Done = partDoneStart - if _, seekErr := rd.Seek(0, io.SeekStart); seekErr != nil { - return seekErr - } - req, reqErr := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, io.TeeReader(rd, p)) - if reqErr != nil { - return reqErr - } - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", fmt.Sprint(partSize)) - req.Header.Set("Origin", "https://yun.139.com") - req.Header.Set("Referer", "https://yun.139.com/") - req.ContentLength = partSize - - res, doErr := base.HttpClient.Do(req) - if doErr != nil { - return doErr + start := p.Done + upload := func(uploadURL string) error { + return retry.Do(func() error { + p.Done = start + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return retry.Unrecoverable(err) } - defer res.Body.Close() - log.Debugf("[139] uploaded: %+v", res) - if res.StatusCode != http.StatusOK { - body, _ := io.ReadAll(res.Body) - return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body)) + err := putPersonalPart(ctx, uploadURL, part.PartSize, io.TeeReader(rd, p)) + if errors.Is(err, errUploadURLExpired) { + return retry.Unrecoverable(err) } - return nil - }, - retry.Context(ctx), - retry.Attempts(3), - retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second), - ) + return err + }, retry.Context(ctx), retry.Attempts(3), retry.LastErrorOnly(true), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + } + + err = upload(urls[i].UploadUrl) + if errors.Is(err, errUploadURLExpired) && ctx.Err() == nil { + p.Done = start + // Renew this part and the rest of the batch under the same upload ID. + var refreshed []PersonalPartInfo + refreshed, err = fetchURLs(parts[i:]) + if err == nil { + err = matchUploadURLs(parts[i:], refreshed) + } + if err == nil { + copy(urls[i:], refreshed) + err = upload(urls[i].UploadUrl) + } + } ss.FreeSectionReader(rd) if err != nil { - return err + return fmt.Errorf("upload 139 part %d: %w", part.PartNumber, err) } } return nil } +func putPersonalPart(ctx context.Context, uploadURL string, size int64, body io.Reader) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = size + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode == http.StatusOK { + return nil + } + raw, _ := io.ReadAll(res.Body) + if res.StatusCode == http.StatusForbidden { + var eosError struct { + Code string + Message string + } + if xml.Unmarshal(raw, &eosError) == nil && + (eosError.Code == "RequestExpired" || (eosError.Code == "AccessDenied" && eosError.Message == "Request has expired")) { + return errUploadURLExpired + } + } + return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(raw)) +} + func (d *Yun139) getDiskQuotaDetail(ctx context.Context) (*DiskQuotaDetail, error) { data := map[string]interface{}{ "userDomainId": d.UserDomainID,