internal: Refactor immutable folder upload as an internal resource in direct engine

acceptance/bundle/deploy/immutable-no-artifacts/.gitignore

@@ -0,0 +1,2 @@
+*.txt
+*.json

acceptance/bundle/deploy/immutable-no-artifacts/output.txt

@@ -8,21 +8,122 @@ Workspace:
 
 Validation OK!
 
+>>> [CLI] bundle plan -o json
+
+>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json
+[
+  {
+    "notebook_task": {
+      "base_parameters": {
+        "path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path"
+      },
+      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
+    },
+    "task_key": "notebook_task"
+  },
+  {
+    "environment_key": "env",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
+    },
+    "task_key": "spark_python_task"
+  }
+]
+
+>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json
+{
+  "action": "create",
+  "new_state": {
+    "value": {
+      "remote_root": "/Workspace/Users/[UUID]/.snapshots",
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip"
+    }
+  }
+}
+
 >>> [CLI] bundle deploy
-Uploading immutable bundle snapshot...
+Created internal_immutable_snapshots.immutable
 Created jobs.my_job
 Files: 0 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.tasks job.json
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.tasks job.json
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.tasks job.json
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path"
 
+>>> title Check snapshot plan after deploy: 
+
+=== Check snapshot plan after deploy: 
+>>> [CLI] bundle plan -o json
+
+>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json
+{
+  "action": "skip",
+  "remote_state": {
+    "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+    "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+  },
+  "changes": {
+    "acl": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "new": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "remote": null
+    },
+    "bundle_id": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "[UUID]",
+      "new": "[UUID]",
+      "remote": ""
+    },
+    "remote_root": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "/Workspace/Users/[UUID]/.snapshots",
+      "new": "/Workspace/Users/[UUID]/.snapshots",
+      "remote": ""
+    },
+    "zip_path": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
+      "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
+      "remote": ""
+    }
+  }
+}
+
+>>> [CLI] bundle deploy
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+Snapshot did not change as expected
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.jobs.my_job

acceptance/bundle/deploy/immutable-no-artifacts/script

@@ -6,13 +6,33 @@ cleanup() {
 trap cleanup EXIT
 
 trace $CLI bundle validate
-trace $CLI bundle deploy
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json
+trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json
 
+trace $CLI bundle deploy $(readplanarg plan.json)
 
 # Get a job and check that task paths point into the snapshot
 JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id')
-trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file'
-trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path'
-trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path'
+$CLI jobs get $JOB_ID > job.json
+trace jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file'
+trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path'
+trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path'
+PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
 
+trace title "Check snapshot plan after deploy: "
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json
+
+trace $CLI bundle deploy $(readplanarg plan.json)
+rm plan.json
+rm job.json
+
+PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
+if [ "$PYTHON_FILE_1" == "$PYTHON_FILE_2" ]; then
+    echo "Snapshot did not change as expected"
+else
+    echo "Snapshot changed"
+    exit 1
+fi
 trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/immutable-no-artifacts/test.toml

@@ -1,5 +1,6 @@
 Cloud = false # Temporary disable cloud tests until the API is fully available
 RecordRequests = true
+EnvMatrix.READPLAN = ["", "1"]
 
 # immutable_folder only works with the direct engine.
 EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
@@ -17,3 +18,9 @@ Ignore = [
 [[Repls]]
 Old = '[0-9a-f]{64}'
 New = '[SNAPSHOT_HASH]'
+
+# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan plan.json".
+# Normalize so both variants produce identical output.
+[[Repls]]
+Old = ' --plan plan.json'
+New = ''

acceptance/bundle/deploy/immutable-permissions-change/output.txt

@@ -1,10 +1,10 @@
 
 === Deploy without permissions
 >>> [CLI] bundle deploy
-Uploading immutable bundle snapshot...
+Created internal_immutable_snapshots.immutable
 Created jobs.my_job
 Files: 0 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 First snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py
 
 === Add permissions and redeploy
@@ -21,11 +21,11 @@ Consider using a adding a top-level permissions section such as the following:
 See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration.
   in databricks.yml:22:3
 
-Uploading immutable bundle snapshot...
+Recreated internal_immutable_snapshots.immutable
 Updated jobs.my_job
 Created jobs.my_job.permissions
 Files: 0 uploaded, 0 deleted
-Resources: 1 created, 1 changed, 0 deleted, 0 unchanged
+Resources: 2 created, 1 changed, 1 deleted, 0 unchanged
 Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py
 
 === Verify snapshot path changed after permissions change

acceptance/bundle/deploy/immutable/output.txt

@@ -10,10 +10,12 @@ Validation OK!
 
 >>> [CLI] bundle plan -o json
 Building python_artifact...
+
+>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json
 [
   {
     "notebook_task": {
-      "notebook_path": "${workspace.snapshot_path}/files/src/notebook"
+      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
     },
     "task_key": "notebook_task"
   },
@@ -28,30 +30,156 @@ Building python_artifact...
   {
     "environment_key": "env",
     "spark_python_task": {
-      "python_file": "${workspace.snapshot_path}/files/src/main.py"
+      "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
     },
     "task_key": "spark_python_task"
   }
 ]
 
+>>> jq .plan["resources.jobs.my_job"].new_state.value.environments plan.json
+[
+  {
+    "environment_key": "env",
+    "spec": {
+      "dependencies": [
+        "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
+      ],
+      "environment_version": "4"
+    }
+  }
+]
+
+>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json
+{
+  "action": "create",
+  "new_state": {
+    "value": {
+      "remote_root": "/Workspace/Users/[UUID]/.snapshots",
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip"
+    }
+  }
+}
+
 >>> [CLI] bundle deploy
 Building python_artifact...
-Uploading immutable bundle snapshot...
+Created internal_immutable_snapshots.immutable
 Created jobs.my_job
 Files: 0 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.tasks job.json
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.tasks job.json
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
 
->>> [CLI] jobs get [NUMID]
+>>> jq .settings.environments[0].spec.dependencies job.json
 [
   "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
 ]
 
+>>> title Check snapshot plan after deploy
+
+=== Check snapshot plan after deploy
+>>> [CLI] bundle plan -o json
+Building python_artifact...
+
+>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json
+{
+  "action": "recreate",
+  "new_state": {
+    "value": {
+      "remote_root": "/Workspace/Users/[UUID]/.snapshots",
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip"
+    }
+  },
+  "remote_state": {
+    "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+    "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+  },
+  "changes": {
+    "acl": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "new": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "remote": null
+    },
+    "bundle_id": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "[UUID]",
+      "new": "[UUID]",
+      "remote": ""
+    },
+    "full_path": {
+      "action": "recreate",
+      "reason": "immutable",
+      "old": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "new": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "remote": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+    },
+    "relative_path": {
+      "action": "recreate",
+      "reason": "immutable",
+      "old": "[UUID]/[SNAPSHOT_HASH]",
+      "new": "[UUID]/[SNAPSHOT_HASH]",
+      "remote": "[UUID]/[SNAPSHOT_HASH]"
+    },
+    "remote_root": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "/Workspace/Users/[UUID]/.snapshots",
+      "new": "/Workspace/Users/[UUID]/.snapshots",
+      "remote": ""
+    },
+    "zip_path": {
+      "action": "recreate",
+      "reason": "immutable",
+      "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
+      "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
+      "remote": ""
+    }
+  }
+}
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Recreated internal_immutable_snapshots.immutable
+Updated jobs.my_job
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 1 deleted, 0 unchanged
+
+=== Check that job uses a new snapshot: Snapshot changed as expected
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.jobs.my_job