Add a new workspace.sync_mode flag

.nextchanges/bundles/workspace-sync-mode.md

@@ -0,0 +1 @@
+* Add the `workspace.sync_mode` setting with the values `overwrite` (default), `in-place`, and `immutable`. It replaces `presets.source_linked_deployment` (use `in-place`) and `experimental.immutable_folder` (use `immutable`), which are now deprecated and emit a warning.

acceptance/bundle/deploy/immutable-dev-mode/databricks.yml.tmpl

@@ -1,8 +1,8 @@
 bundle:
   name: test-bundle-immutable-dev-mode-$UNIQUE_NAME
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 resources:
   jobs:

acceptance/bundle/deploy/immutable-no-artifacts/databricks.yml.tmpl

@@ -1,8 +1,8 @@
 bundle:
   name: test-bundle-immutable-no-artifacts-$UNIQUE_NAME
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 resources:
   jobs:

acceptance/bundle/deploy/immutable-permissions-change/databricks.yml.tmpl

@@ -1,8 +1,8 @@
 bundle:
   name: test-bundle-immutable-perms-$UNIQUE_NAME
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 resources:
   jobs:

acceptance/bundle/deploy/immutable/databricks.yml.tmpl

@@ -1,8 +1,8 @@
 bundle:
   name: test-bundle-immutable-$UNIQUE_NAME
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 artifacts:
   python_artifact:

acceptance/bundle/dms/immutable-folder/databricks.yml.tmpl

@@ -1,7 +1,8 @@
 bundle:
   name: dms-immutable-folder-$UNIQUE_NAME
+workspace:
+  sync_mode: immutable
 experimental:
-  immutable_folder: true
   deployment_history: true
 
 resources:

acceptance/bundle/resources/apps/immutable/databricks.yml.tmpl

@@ -1,8 +1,8 @@
 bundle:
   name: test-bundle-immutable-app-$UNIQUE_NAME
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 resources:
   apps:

acceptance/bundle/validate/immutable_folder_permissions/databricks.yml

@@ -1,8 +1,8 @@
 bundle:
   name: my-bundle
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 permissions:
   - level: CAN_VIEW

acceptance/bundle/validate/immutable_workspace_paths/databricks.yml

@@ -1,8 +1,8 @@
 bundle:
   name: my-bundle
 
-experimental:
-  immutable_folder: true
+workspace:
+  sync_mode: immutable
 
 resources:
   jobs:

acceptance/bundle/validate/immutable_workspace_paths/output.txt

@@ -12,7 +12,8 @@
     "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
     "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources",
     "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default",
-    "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state"
+    "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state",
+    "sync_mode": "immutable"
   },
   "tasks": [
     {

acceptance/bundle/validate/sync_mode/databricks.yml

@@ -0,0 +1,50 @@
+bundle:
+  name: test-bundle
+
+variables:
+  mode:
+    default: immutable
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: my_task
+          existing_cluster_id: "0101-120000-aaaaaaaa"
+          spark_python_task:
+            python_file: ./src/main.py
+
+targets:
+  default:
+    default: true
+  overwrite:
+    workspace:
+      sync_mode: overwrite
+  in-place:
+    workspace:
+      sync_mode: in-place
+  immutable:
+    workspace:
+      sync_mode: immutable
+  source_linked_enabled:
+    presets:
+      source_linked_deployment: true
+  source_linked_disabled:
+    presets:
+      source_linked_deployment: false
+  source_linked_with_sync_mode:
+    presets:
+      source_linked_deployment: true
+    workspace:
+      sync_mode: immutable
+  invalid:
+    workspace:
+      sync_mode: inplace
+  variable:
+    workspace:
+      sync_mode: ${var.mode}
+  immutable_file_path:
+    workspace:
+      sync_mode: immutable
+      file_path: /Workspace/Shared/files

acceptance/bundle/validate/sync_mode/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/sync_mode/output.txt

@@ -0,0 +1,122 @@
+
+=== target default
+>>> [CLI] bundle validate -t default -o json
+{
+  "sync_mode": null,
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/src/main.py"
+}
+
+=== target overwrite
+>>> [CLI] bundle validate -t overwrite -o json
+{
+  "sync_mode": "overwrite",
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/overwrite/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/overwrite/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/overwrite/files/src/main.py"
+}
+
+=== target in-place
+>>> [CLI] bundle validate -t in-place -o json
+Warning: source-linked deployment is available only in the Databricks Workspace
+  at workspace.sync_mode
+  in databricks.yml:26:18
+
+{
+  "sync_mode": "overwrite",
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/in-place/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/in-place/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/in-place/files/src/main.py"
+}
+
+=== target immutable
+>>> [CLI] bundle validate -t immutable -o json
+{
+  "sync_mode": "immutable",
+  "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
+  "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts",
+  "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py"
+}
+
+=== target source_linked_enabled
+>>> [CLI] bundle validate -t source_linked_enabled -o json
+Warning: presets.source_linked_deployment is deprecated, please set workspace.sync_mode to "in-place" instead
+  at presets.source_linked_deployment
+  in databricks.yml:32:33
+
+Warning: source-linked deployment is available only in the Databricks Workspace
+  at presets.source_linked_deployment
+  in databricks.yml:32:33
+
+{
+  "sync_mode": "overwrite",
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_enabled/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_enabled/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_enabled/files/src/main.py"
+}
+
+=== target source_linked_disabled
+>>> [CLI] bundle validate -t source_linked_disabled -o json
+Warning: presets.source_linked_deployment is deprecated, please set workspace.sync_mode to "overwrite" instead
+  at presets.source_linked_deployment
+  in databricks.yml:35:33
+
+{
+  "sync_mode": null,
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_disabled/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_disabled/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/source_linked_disabled/files/src/main.py"
+}
+
+=== target source_linked_with_sync_mode
+>>> [CLI] bundle validate -t source_linked_with_sync_mode -o json
+Warning: presets.source_linked_deployment is deprecated and ignored because workspace.sync_mode is set, please remove it
+  at presets.source_linked_deployment
+  in databricks.yml:38:33
+
+{
+  "sync_mode": "immutable",
+  "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
+  "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts",
+  "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py"
+}
+
+=== target invalid
+>>> [CLI] bundle validate -t invalid
+Error: invalid value "inplace" for workspace.sync_mode (expected one of [overwrite in-place immutable])
+  at workspace.sync_mode
+  in databricks.yml:43:18
+
+Name: test-bundle
+Target: invalid
+Workspace:
+  User: [USERNAME]
+
+Found 1 error
+
+=== target variable
+>>> [CLI] bundle validate -t variable
+Error: invalid value "${var.mode}" for workspace.sync_mode (expected one of [overwrite in-place immutable])
+  at workspace.sync_mode
+  in databricks.yml:46:18
+
+Name: test-bundle
+Target: variable
+Workspace:
+  User: [USERNAME]
+
+Found 1 error
+
+=== target immutable_file_path
+>>> [CLI] bundle validate -t immutable_file_path
+Error: workspace.file_path cannot be configured when workspace.sync_mode is immutable
+  in databricks.yml:50:18
+
+Name: test-bundle
+Target: immutable_file_path
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/immutable_file_path
+
+Found 1 error

acceptance/bundle/validate/sync_mode/script

@@ -0,0 +1,9 @@
+for target in default overwrite in-place immutable source_linked_enabled source_linked_disabled source_linked_with_sync_mode; do
+    title "target $target"
+    trace $CLI bundle validate -t $target -o json | jq '{sync_mode: .workspace.sync_mode, file_path: .workspace.file_path, artifact_path: .workspace.artifact_path, python_file: .resources.jobs.my_job.tasks[0].spark_python_task.python_file}'
+done
+
+for target in invalid variable immutable_file_path; do
+    title "target $target"
+    musterr trace $CLI bundle validate -t $target
+done

acceptance/bundle/validate/sync_mode/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/validate/sync_mode/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks"]

acceptance/bundle/validate/sync_mode_immutable_folder/databricks.yml

@@ -0,0 +1,28 @@
+bundle:
+  name: test-bundle
+
+experimental:
+  immutable_folder: true
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: my_task
+          existing_cluster_id: "0101-120000-aaaaaaaa"
+          spark_python_task:
+            python_file: ./src/main.py
+
+targets:
+  default:
+    default: true
+  source_linked_disabled:
+    presets:
+      source_linked_deployment: false
+  source_linked_enabled:
+    presets:
+      source_linked_deployment: true
+  with_sync_mode:
+    workspace:
+      sync_mode: overwrite

acceptance/bundle/validate/sync_mode_immutable_folder/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/sync_mode_immutable_folder/output.txt

@@ -0,0 +1,60 @@
+
+=== target default
+>>> [CLI] bundle validate -t default -o json
+Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead
+  at experimental.immutable_folder
+  in databricks.yml:5:21
+
+{
+  "sync_mode": null,
+  "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
+  "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts",
+  "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py"
+}
+
+=== target source_linked_disabled
+>>> [CLI] bundle validate -t source_linked_disabled -o json
+Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead
+  at experimental.immutable_folder
+  in databricks.yml:5:21
+
+Warning: presets.source_linked_deployment is deprecated and ignored because experimental.immutable_folder is set, please remove it
+  at presets.source_linked_deployment
+  in databricks.yml:22:33
+
+{
+  "sync_mode": null,
+  "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
+  "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts",
+  "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py"
+}
+
+=== target source_linked_enabled
+>>> [CLI] bundle validate -t source_linked_enabled -o json
+Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead
+  at experimental.immutable_folder
+  in databricks.yml:5:21
+
+Warning: presets.source_linked_deployment is deprecated and ignored because experimental.immutable_folder is set, please remove it
+  at presets.source_linked_deployment
+  in databricks.yml:25:33
+
+{
+  "sync_mode": null,
+  "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files",
+  "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts",
+  "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py"
+}
+
+=== target with_sync_mode
+>>> [CLI] bundle validate -t with_sync_mode -o json
+Warning: experimental.immutable_folder is deprecated and ignored because workspace.sync_mode is set, please remove it
+  at experimental.immutable_folder
+  in databricks.yml:5:21
+
+{
+  "sync_mode": "overwrite",
+  "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/with_sync_mode/files",
+  "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/with_sync_mode/artifacts",
+  "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/with_sync_mode/files/src/main.py"
+}

acceptance/bundle/validate/sync_mode_immutable_folder/script

@@ -0,0 +1,4 @@
+for target in default source_linked_disabled source_linked_enabled with_sync_mode; do
+    title "target $target"
+    trace $CLI bundle validate -t $target -o json | jq '{sync_mode: .workspace.sync_mode, file_path: .workspace.file_path, artifact_path: .workspace.artifact_path, python_file: .resources.jobs.my_job.tasks[0].spark_python_task.python_file}'
+done

acceptance/bundle/validate/sync_mode_immutable_folder/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/validate/sync_mode_immutable_folder/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks"]