Migrate bundle state filer read off workspace-files to /workspace/export

acceptance/bundle/state/basic/out.state.txt

@@ -1,8 +1,16 @@
 {
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/state/default/state/STATE_FILENAME"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/state/default/state/STATE_FILENAME"
+  }
 }
 {
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/state/default/state/deployment.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/state/default/state/deployment.json"
+  }
 }

acceptance/bundle/state/basic/script

@@ -1,4 +1,4 @@
 trace $CLI bundle deploy
 trace $CLI bundle deploy # We do 2 deploys because only 2nd deploy will pull state from remote after 1st created it
-jq 'select(.path | test("/api/2.0/workspace-files/Workspace/Users/.*/.bundle/state/default/state/"))' out.requests.txt > out.state.txt
+jq 'select(.path == "/api/2.0/workspace/export" and (.q.path | test("/.bundle/state/default/state/")) and (.q.path | test("deploy.lock") | not))' out.requests.txt > out.state.txt
 rm out.requests.txt

acceptance/bundle/state/force_pull_commands/output.txt

@@ -15,7 +15,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 >>> [CLI] bundle summary --force-pull
 {
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+  }
 }
 
 === bundle open without --force-pull: no remote state read
@@ -29,5 +33,9 @@ Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID]
 Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID]
 {
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME"
+  }
 }

acceptance/bundle/state/force_pull_commands/script

@@ -11,19 +11,19 @@ export BROWSER="echo_browser.py"
 title "bundle summary without --force-pull: no remote state read\n"
 trace $CLI bundle summary > /dev/null
 touch out.requests.txt
-print_requests.py --get //workspace-files/
+print_requests.py --get //workspace/export
 
 title "bundle summary --force-pull: remote state read\n"
 trace $CLI bundle summary --force-pull > /dev/null
 touch out.requests.txt
-print_requests.py --get //workspace-files/
+print_requests.py --get //workspace/export
 
 title "bundle open without --force-pull: no remote state read\n"
 trace $CLI bundle open foo > /dev/null
 touch out.requests.txt
-print_requests.py --get //workspace-files/
+print_requests.py --get //workspace/export
 
 title "bundle open --force-pull: remote state read\n"
 trace $CLI bundle open foo --force-pull > /dev/null
 touch out.requests.txt
-print_requests.py --get //workspace-files/
+print_requests.py --get //workspace/export

acceptance/bundle/user_agent/output.txt

@@ -48,7 +48,7 @@ MISS	deploy.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF
 MISS	deploy.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	deploy.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	destroy.direct	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	destroy.direct	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	destroy.direct	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	destroy.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	destroy.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 OK  	destroy.direct	/api/2.1/unity-catalog/schemas/mycatalog.myschema	engine/direct
@@ -61,7 +61,7 @@ OK  	destroy.direct	/api/2.0/workspace-files/import-file/Workspace/Users/[USERNA
 OK  	destroy.direct	/api/2.0/workspace/delete	engine/direct
 MISS	destroy.direct	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	destroy.terraform	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	destroy.terraform	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	destroy.terraform	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	destroy.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	destroy.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 OK  	destroy.terraform	/api/2.0/workspace/export	engine/terraform
@@ -93,30 +93,30 @@ MISS	plan.terraform	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks
 MISS	plan.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	plan.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	plan2.direct	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	plan2.direct	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	plan2.direct	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	plan2.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	plan2.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
-OK  	plan2.direct	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json	engine/direct
+OK  	plan2.direct	/api/2.0/workspace/export	engine/direct
 OK  	plan2.direct	/api/2.0/workspace/get-status	engine/direct
 OK  	plan2.direct	/api/2.1/unity-catalog/schemas/mycatalog.myschema	engine/direct
 MISS	plan2.direct	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	plan2.terraform	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	plan2.terraform	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	plan2.terraform	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	plan2.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	plan2.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat'
-OK  	plan2.terraform	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json	engine/terraform
+OK  	plan2.terraform	/api/2.0/workspace/export	engine/terraform
 OK  	plan2.terraform	/api/2.0/workspace/get-status	engine/terraform
 MISS	plan2.terraform	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	plan2.terraform	/api/2.1/unity-catalog/schemas/mycatalog.myschema	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat'
 MISS	plan2.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	plan2.terraform	/.well-known/databricks-config	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5'
 MISS	run.direct	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	run.direct	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	run.direct	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.direct	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.direct	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	run.terraform	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
-MISS	run.terraform	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
+MISS	run.terraform	/api/2.0/workspace/export	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.terraform	/api/2.0/workspace/get-status	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	run.terraform	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'

acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  }
 }
 {
   "headers": {

acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  }
 }
 {
   "headers": {

acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  }
 }
 {
   "headers": {
@@ -49,7 +53,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+  }
 }
 {
   "headers": {

acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  }
 }
 {
   "headers": {
@@ -49,7 +53,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+  }
 }
 {
   "headers": {

acceptance/bundle/user_agent/simple/out.requests.run.direct.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json"
+  }
 }
 {
   "headers": {

acceptance/bundle/user_agent/simple/out.requests.run.terraform.json

@@ -14,7 +14,11 @@
     ]
   },
   "method": "GET",
-  "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  "path": "/api/2.0/workspace/export",
+  "q": {
+    "direct_download": "true",
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+  }
 }
 {
   "headers": {

bundle/deploy/filer.go

@@ -1,19 +1,14 @@
 package deploy
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"io/fs"
-	"net/http"
-	"net/url"
-	"strings"
 
 	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/auth"
 	"github.com/databricks/cli/libs/filer"
-	"github.com/databricks/databricks-sdk-go/client"
+	"github.com/databricks/databricks-sdk-go"
 )
 
 // FilerFactory is a function that returns a filer.Filer.
@@ -22,8 +17,8 @@ type FilerFactory func(ctx context.Context, b *bundle.Bundle) (filer.Filer, erro
 type stateFiler struct {
 	filer filer.Filer
 
-	apiClient *client.DatabricksClient
-	root      filer.WorkspaceRootPath
+	workspaceClient *databricks.WorkspaceClient
+	root            filer.WorkspaceRootPath
 }
 
 func (s stateFiler) Delete(ctx context.Context, path string, mode ...filer.DeleteMode) error {
@@ -49,14 +44,8 @@ func (s stateFiler) Read(ctx context.Context, path string) (io.ReadCloser, error
 		return nil, fmt.Errorf("not a file: %s", absPath)
 	}
 
-	var buf bytes.Buffer
-	urlPath := "/api/2.0/workspace-files/" + url.PathEscape(strings.TrimLeft(absPath, "/"))
-	err = s.apiClient.Do(ctx, http.MethodGet, urlPath, auth.WorkspaceIDHeaders(s.apiClient.Config), nil, nil, &buf)
-	if err != nil {
-		return nil, err
-	}
-
-	return io.NopCloser(&buf), nil
+	// Stream via the SDK's /workspace/export (direct_download=true); no 10 MB cap.
+	return s.workspaceClient.Workspace.Download(ctx, absPath)
 }
 
 func (s stateFiler) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) {
@@ -72,24 +61,19 @@ func (s stateFiler) Write(ctx context.Context, path string, reader io.Reader, mo
 }
 
 // StateFiler returns a filer.Filer that can be used to read/write state files.
-// We use a custom workspace filer which uses workspace-files API to read state files.
-// This API has a higher than 10 MB limits and allows to export large state files.
-// We don't use the same API for read because it doesn't correct get the file content for notebooks and returns
-// "File Not Found" error instead.
+// Reads use the streaming /workspace/export API, which is officially supported,
+// scoped, and streams state files well beyond the 10 MB JSON export limit.
 func StateFiler(ctx context.Context, b *bundle.Bundle) (filer.Filer, error) {
-	f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
-	if err != nil {
-		return nil, err
-	}
+	w := b.WorkspaceClient(ctx)
 
-	apiClient, err := client.New(b.WorkspaceClient(ctx).Config)
+	f, err := filer.NewWorkspaceFilesClient(w, b.Config.Workspace.StatePath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create API client: %w", err)
+		return nil, err
 	}
 
 	return stateFiler{
-		filer:     f,
-		root:      filer.NewWorkspaceRootPath(b.Config.Workspace.StatePath),
-		apiClient: apiClient,
+		filer:           f,
+		root:            filer.NewWorkspaceRootPath(b.Config.Workspace.StatePath),
+		workspaceClient: w,
 	}, nil
 }