Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 15 additions & 16 deletions drivers/139/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -802,32 +802,31 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return err
}

// 先上传前100个分片
err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, ss, p)
if err != nil {
return err
}

// 如果还有剩余分片,分批获取上传地址并上传
for i := 100; i < len(partInfos); i += 100 {
end := min(i+100, len(partInfos))
batchPartInfos := partInfos[i:end]
moredata := base.Json{
fetchUploadURLs := func(parts []PartInfo) ([]PersonalPartInfo, error) {
payload := base.Json{
"fileId": resp.Data.FileId,
"uploadId": resp.Data.UploadId,
"partInfos": batchPartInfos,
"partInfos": parts,
"commonAccountInfo": base.Json{
"account": d.getAccount(),
"accountType": 1,
},
}
var moreresp PersonalUploadUrlResp
_, err = d.newPost(getUploadUrlPath, moredata, &moreresp)
var urls PersonalUploadUrlResp
_, err := d.newPost(getUploadUrlPath, payload, &urls)
return urls.Data.PartInfos, err
}

if err := d.uploadPersonalParts(ctx, firstPartInfos, resp.Data.PartInfos, fetchUploadURLs, ss, p); err != nil {
return err
}
for i := len(firstPartInfos); i < len(partInfos); i += 100 {
batch := partInfos[i:min(i+100, len(partInfos))]
urls, err := fetchUploadURLs(batch)
if err != nil {
return err
}
err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, ss, p)
if err != nil {
if err := d.uploadPersonalParts(ctx, batch, urls, fetchUploadURLs, ss, p); err != nil {
return err
}
}
Expand Down
116 changes: 116 additions & 0 deletions drivers/139/upload_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package _139

import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"

"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
)

type testUploadSections struct {
*strings.Reader
}

func (s testUploadSections) GetSectionReader(offset, length int64) (io.ReadSeeker, error) {
return io.NewSectionReader(s.Reader, offset, length), nil
}

func (testUploadSections) FreeSectionReader(io.ReadSeeker) {}

func (testUploadSections) DiscardSection(int64, int64) error { return nil }

func TestUploadPersonalPartsRenewsExpiredBatch(t *testing.T) {
var mu sync.Mutex
var requests []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Error(err)
}
mu.Lock()
requests = append(requests, fmt.Sprintf("%s %s %s", r.Method, r.URL.Path, body))
mu.Unlock()
if r.URL.Path == "/expired" {
w.WriteHeader(http.StatusForbidden)
fmt.Fprint(w, "<Error><Code>AccessDenied</Code><Message>Request has expired</Message></Error>")
return
}
if r.URL.Path == "/stale-third" {
t.Error("the third part must use the renewed URL")
w.WriteHeader(http.StatusBadRequest)
}
}))
defer server.Close()
useUploadTestClient(t, server.Client())

parts := []PartInfo{
{PartNumber: 101, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 0}},
{PartNumber: 102, PartSize: 2, ParallelHashCtx: ParallelHashCtx{PartOffset: 2}},
{PartNumber: 103, PartSize: 1, ParallelHashCtx: ParallelHashCtx{PartOffset: 4}},
}
urls := []PersonalPartInfo{
{PartNumber: 102, UploadUrl: server.URL + "/expired"},
{PartNumber: 103, UploadUrl: server.URL + "/stale-third"},
{PartNumber: 101, UploadUrl: server.URL + "/first"},
}
refreshes := 0
fetchURLs := func(pending []PartInfo) ([]PersonalPartInfo, error) {
refreshes++
if !reflect.DeepEqual(pending, parts[1:]) {
t.Fatalf("renewal requested the wrong parts: %+v", pending)
}
return []PersonalPartInfo{
{PartNumber: 103, UploadUrl: server.URL + "/fresh-third"},
{PartNumber: 102, UploadUrl: server.URL + "/fresh-second"},
}, nil
}
progress := driver.NewProgress(5, func(float64) {})
sections := testUploadSections{strings.NewReader("abcde")}
if err := (&Yun139{}).uploadPersonalParts(context.Background(), parts, urls, fetchURLs, sections, progress); err != nil {
t.Fatal(err)
}
want := []string{"PUT /first ab", "PUT /expired cd", "PUT /fresh-second cd", "PUT /fresh-third e"}
mu.Lock()
defer mu.Unlock()
if !reflect.DeepEqual(requests, want) || refreshes != 1 || progress.Done != 5 {
t.Fatalf("requests = %v, refreshes = %d, uploaded bytes = %d", requests, refreshes, progress.Done)
}
}

func TestPutPersonalPartDistinguishesExpiredAndDenied(t *testing.T) {
for _, tt := range []struct {
name, body string
expired bool
}{
{"expired", "<Error><Code>RequestExpired</Code></Error>", true},
{"denied", "<Error><Code>AccessDenied</Code><Message>Access denied</Message></Error>", false},
} {
t.Run(tt.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprint(w, tt.body)
}))
defer server.Close()
useUploadTestClient(t, server.Client())
err := putPersonalPart(context.Background(), server.URL, 1, strings.NewReader("x"))
if err == nil || (err == errUploadURLExpired) != tt.expired {
t.Fatalf("got %v, want expired = %t", err, tt.expired)
}
})
}
}

func useUploadTestClient(t *testing.T, client *http.Client) {
t.Helper()
previous := base.HttpClient
base.HttpClient = client
t.Cleanup(func() { base.HttpClient = previous })
}
132 changes: 82 additions & 50 deletions drivers/139/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -1078,70 +1078,102 @@ func (d *Yun139) getGroupCloudHost() string {
return d.GroupCloudHost
}

func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, ss streamPkg.StreamSectionReader, p *driver.Progress) error {
// 确保数组以 PartNumber 从小到大排序
sort.Slice(uploadPartInfos, func(i, j int) bool {
return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber
})
var errUploadURLExpired = errors.New("139 upload URL expired")

for _, uploadPartInfo := range uploadPartInfos {
index := uploadPartInfo.PartNumber - 1
if index < 0 || index >= len(partInfos) {
return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos))
func matchUploadURLs(parts []PartInfo, urls []PersonalPartInfo) error {
if len(parts) != len(urls) {
return fmt.Errorf("139 returned %d upload URLs for %d parts", len(urls), len(parts))
}
sort.Slice(urls, func(i, j int) bool {
return urls[i].PartNumber < urls[j].PartNumber
})
for i, part := range parts {
if int64(urls[i].PartNumber) != part.PartNumber || urls[i].UploadUrl == "" {
return fmt.Errorf("139 did not return a valid upload URL for part %d", part.PartNumber)
}
partSize := partInfos[index].PartSize
offset := partInfos[index].ParallelHashCtx.PartOffset
log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos))
}
return nil
}

rd, getErr := ss.GetSectionReader(offset, partSize)
if getErr != nil {
return getErr
func (d *Yun139) uploadPersonalParts(ctx context.Context, parts []PartInfo, urls []PersonalPartInfo, fetchURLs func([]PartInfo) ([]PersonalPartInfo, error), ss streamPkg.StreamSectionReader, p *driver.Progress) error {
if err := matchUploadURLs(parts, urls); err != nil {
return err
}
for i, part := range parts {
rd, err := ss.GetSectionReader(part.ParallelHashCtx.PartOffset, part.PartSize)
if err != nil {
return err
}

// Save progress before this part so retries don't double-count bytes
partDoneStart := p.Done
err := retry.Do(
func() error {
// Reset progress to the start of this part on each attempt
p.Done = partDoneStart
if _, seekErr := rd.Seek(0, io.SeekStart); seekErr != nil {
return seekErr
}
req, reqErr := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, io.TeeReader(rd, p))
if reqErr != nil {
return reqErr
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprint(partSize))
req.Header.Set("Origin", "https://yun.139.com")
req.Header.Set("Referer", "https://yun.139.com/")
req.ContentLength = partSize

res, doErr := base.HttpClient.Do(req)
if doErr != nil {
return doErr
start := p.Done
upload := func(uploadURL string) error {
return retry.Do(func() error {
p.Done = start
if _, err := rd.Seek(0, io.SeekStart); err != nil {
return retry.Unrecoverable(err)
}
defer res.Body.Close()
log.Debugf("[139] uploaded: %+v", res)
if res.StatusCode != http.StatusOK {
body, _ := io.ReadAll(res.Body)
return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body))
err := putPersonalPart(ctx, uploadURL, part.PartSize, io.TeeReader(rd, p))
if errors.Is(err, errUploadURLExpired) {
return retry.Unrecoverable(err)
}
return nil
},
retry.Context(ctx),
retry.Attempts(3),
retry.DelayType(retry.BackOffDelay),
retry.Delay(time.Second),
)
return err
}, retry.Context(ctx), retry.Attempts(3), retry.LastErrorOnly(true), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second))
}

err = upload(urls[i].UploadUrl)
if errors.Is(err, errUploadURLExpired) && ctx.Err() == nil {
p.Done = start
// Renew this part and the rest of the batch under the same upload ID.
var refreshed []PersonalPartInfo
refreshed, err = fetchURLs(parts[i:])
if err == nil {
err = matchUploadURLs(parts[i:], refreshed)
}
if err == nil {
copy(urls[i:], refreshed)
err = upload(urls[i].UploadUrl)
}
}
ss.FreeSectionReader(rd)
if err != nil {
return err
return fmt.Errorf("upload 139 part %d: %w", part.PartNumber, err)
}
}
return nil
}

func putPersonalPart(ctx context.Context, uploadURL string, size int64, body io.Reader) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, body)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Origin", "https://yun.139.com")
req.Header.Set("Referer", "https://yun.139.com/")
req.ContentLength = size

res, err := base.HttpClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode == http.StatusOK {
return nil
}
raw, _ := io.ReadAll(res.Body)
if res.StatusCode == http.StatusForbidden {
var eosError struct {
Code string
Message string
}
if xml.Unmarshal(raw, &eosError) == nil &&
(eosError.Code == "RequestExpired" || (eosError.Code == "AccessDenied" && eosError.Message == "Request has expired")) {
return errUploadURLExpired
}
}
return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(raw))
}

func (d *Yun139) getDiskQuotaDetail(ctx context.Context) (*DiskQuotaDetail, error) {
data := map[string]interface{}{
"userDomainId": d.UserDomainID,
Expand Down
Loading