From 72d6ddd401325dc3244a93f64c2c7bf4e28e6b17 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 08:18:13 +0200 Subject: [PATCH 1/8] Fix terraform migration failing on refs to undeployed resources A resource that is in the terraform state but references a resource that was never deployed (new in config) made the migration abort. Record such fields as unset so the first direct plan applies them. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 36 ++++++++++++++++++++++++++++++ bundle/migrate/build_state_test.go | 27 ++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 0f0776152eb..5c669d2ec59 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "maps" + "reflect" "slices" "strings" @@ -195,6 +196,22 @@ func BuildStateFromTF( return warningsSeen, fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) } + // The referenced resource was never deployed (new in config), so the deployed + // resource cannot have the field set either. Record it as unset: the first direct + // plan sees the difference and applies it once the referenced resource exists. + if referencesUndeployed(pending.refTemplate, tfIDs) { + current, err := structaccess.Get(sv.Value, fieldPath) + if err != nil { + return warningsSeen, fmt.Errorf("%s: reading field %q: %w", node, pending.fieldPathStr, err) + } + zero := reflect.Zero(reflect.TypeOf(current)).Interface() + if err := structaccess.Set(sv.Value, fieldPath, zero); err != nil { + return warningsSeen, fmt.Errorf("%s: cannot unset field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + continue + } + // ResolveFieldRef returns the fully resolved value for this field, // using either Method A (TF state lookup) or Method B (template evaluation). value, warned, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate, warnPrefix) @@ -250,6 +267,25 @@ func BuildStateFromTF( return warningsSeen, nil } +// referencesUndeployed reports whether refTemplate references a resource that is not in the terraform state. +func referencesUndeployed(refTemplate string, tfIDs map[string]string) bool { + ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + if !ok { + return false + } + for _, targetPath := range ref.References() { + path, err := dyn.NewPathFromString(targetPath) + if err != nil { + continue + } + targetNode, _ := config.GetNodeAndType(path) + if _, ok := tfIDs[targetNode.String()]; !ok { + return true + } + } + return false +} + // reconcileIDFields aligns each id-composing field (provided_id_fields, updatable_id_fields) // in the migrated state with the deployed terraform state. The state is otherwise seeded from // config; for id fields, recording the current config would snapshot a pending change as diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index a495436008c..513e50a5b63 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -303,3 +303,30 @@ resources: }) } } + +func TestBuildStateFromTFRefToUndeployedResourceIsUnset(t *testing.T) { + state := runBuildStateFromTF(t, ` +resources: + alerts: + new_alert: + display_name: "new" + jobs: + my_job: + name: "hello" + tasks: + - task_key: t + alert_task: + alert_id: "${resources.alerts.new_alert.id}" +`, + migrate.TFStateAttrs{ + "databricks_job": {"my_job": json.RawMessage(`{"id": "1", "name": "hello", "task": [{"task_key": "t"}]}`)}, + }, + map[string]string{"resources.jobs.my_job": "1"}, + ) + + entry, ok := state["resources.jobs.my_job"] + require.True(t, ok) + assert.NotContains(t, string(entry.State), "new_alert") + assert.Contains(t, string(entry.State), `"alert_id": ""`) + assert.NotContains(t, state, "resources.alerts.new_alert") +} From 319b4948e9a096d2b7848088114606bb81c5df70 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 08:18:34 +0200 Subject: [PATCH 2/8] Add changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/7157.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7157.md diff --git a/.nextchanges/bundles/7157.md b/.nextchanges/bundles/7157.md new file mode 100644 index 00000000000..60bb25670ec --- /dev/null +++ b/.nextchanges/bundles/7157.md @@ -0,0 +1 @@ +* Fix migration from Terraform failing when a deployed resource references a resource that was never deployed ([#7157](https://github.com/databricks/cli/pull/7157)) From cec7e586e97a35d00499fe9b2bc4e5d4e61811a7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 08:23:52 +0200 Subject: [PATCH 3/8] Add acceptance test for migration with a ref to a new resource Co-authored-by: Isaac --- .../auto/new-resource-ref/databricks.yml | 6 + .../auto/new-resource-ref/out.test.toml | 2 + .../migrate/auto/new-resource-ref/output.txt | 31 +++++ .../migrate/auto/new-resource-ref/script | 27 +++++ .../new-resource-ref/terraform-requests.json | 19 +++ .../auto/new-resource-ref/terraform.tfstate | 110 ++++++++++++++++++ .../migrate/auto/new-resource-ref/test.toml | 9 ++ 7 files changed, 204 insertions(+) create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/out.test.toml create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/output.txt create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/script create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/terraform-requests.json create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/terraform.tfstate create mode 100644 acceptance/bundle/migrate/auto/new-resource-ref/test.toml diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml new file mode 100644 index 00000000000..5fa3205df9d --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml @@ -0,0 +1,6 @@ +bundle: {name: test-bundle} + +resources: + jobs: + test_job: + name: "Test Auto-Migrate Job" diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/out.test.toml b/acceptance/bundle/migrate/auto/new-resource-ref/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt new file mode 100644 index 00000000000..9fe1e356896 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt @@ -0,0 +1,31 @@ + +=== Replay a prior terraform deployment of a job only +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Add a new alert and a job task that references it +=== Deploy auto-migrates to direct and creates the alert and the task +>>> [CLI] bundle deploy +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Migrated 1 resource to direct deployment engine. +Created alerts.new_alert +Updated jobs.test_job +Files: 1 uploaded, 0 deleted +Resources: 1 created, 1 changed, 0 deleted, 0 unchanged + +=== The job update carries the new alert id +>>> print_requests.py //api/2.2/jobs/reset +json.body.new_settings.tasks[0].alert_task.alert_id = "[NEW_ALERT_ID]"; + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.alerts.new_alert + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/script b/acceptance/bundle/migrate/auto/new-resource-ref/script new file mode 100644 index 00000000000..6b95079272d --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/script @@ -0,0 +1,27 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Replay a prior terraform deployment of a job only" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +rm -f "$OUT_REQUESTS" + +# The job gains an alert_task that references a new alert, neither of which terraform ever deployed. +# Neither the job's terraform state nor the alert has the referenced id, so the migration must not +# try to resolve it. +title "Add a new alert and a job task that references it" +ALERT=' alerts: {new_alert: {display_name: "New alert", query_text: "select 2", warehouse_id: "'$TEST_DEFAULT_WAREHOUSE_ID'", evaluation: {comparison_operator: EQUAL, source: {name: "1"}, threshold: {value: {double_value: 1.3}}}, schedule: {pause_status: UNPAUSED, quartz_cron_schedule: "44 19 */1 * * ?", timezone_id: "Europe/Amsterdam"}}}' +update_file.py databricks.yml ' jobs:' "$ALERT"$'\n jobs:' +update_file.py databricks.yml 'name: "Test Auto-Migrate Job"' 'name: "Test Auto-Migrate Job"'$'\n tasks: [{task_key: check, alert_task: {alert_id: "${resources.alerts.new_alert.id}"}}]' + +title "Deploy auto-migrates to direct and creates the alert and the task" +trace $CLI bundle deploy +replace_ids.py + +title "The job update carries the new alert id" +trace print_requests.py //api/2.2/jobs/reset | gron.py | grep alert_id +rm -f "$OUT_REQUESTS" + +title "Redeploy is a no-op" +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/terraform-requests.json b/acceptance/bundle/migrate/auto/new-resource-ref/terraform-requests.json new file mode 100644 index 00000000000..6482593b189 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/terraform-requests.json @@ -0,0 +1,19 @@ +[ + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Test Auto-Migrate Job", + "queue": { + "enabled": true + } + } + } +] diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/terraform.tfstate b/acceptance/bundle/migrate/auto/new-resource-ref/terraform.tfstate new file mode 100644 index 00000000000..c04f332a595 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/terraform.tfstate @@ -0,0 +1,110 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "c39488aa-8f36-06fd-a7cb-3f0c95b4c506", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "test_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790612527183335000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Test Auto-Migrate Job", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790612527183335000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/test.toml b/acceptance/bundle/migrate/auto/new-resource-ref/test.toml new file mode 100644 index 00000000000..34dd8bbcbda --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref/test.toml @@ -0,0 +1,9 @@ +# Local-only: the replay fixtures are captured against the fake server, so they reproduce a +# terraform state only there, not on a real workspace (replaying them on cloud fails auth and +# hits backend validation the mock does not enforce). +Cloud = false +RecordRequests = true + +Ignore = [ + ".databricks", +] From 6de85e447273389ce0bff9f37e046ea550be64a4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 08:24:25 +0200 Subject: [PATCH 4/8] Fix changelog fragment format Co-authored-by: Isaac --- .nextchanges/bundles/7157.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/7157.md b/.nextchanges/bundles/7157.md index 60bb25670ec..1e9fb9bc064 100644 --- a/.nextchanges/bundles/7157.md +++ b/.nextchanges/bundles/7157.md @@ -1 +1 @@ -* Fix migration from Terraform failing when a deployed resource references a resource that was never deployed ([#7157](https://github.com/databricks/cli/pull/7157)) +* Fix migration from Terraform failing when a deployed resource references a resource that was never deployed. ([#7157](https://github.com/databricks/cli/pull/7157)) From c42fff8bda8a9efcf4ec4826cb7c5c222dc820d6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 08:35:04 +0200 Subject: [PATCH 5/8] acc: make new-resource-ref fixture readable with commented-out config Co-authored-by: Isaac --- .../auto/new-resource-ref/databricks.yml | 28 ++++++++++++++++++- .../migrate/auto/new-resource-ref/script | 6 ++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml index 5fa3205df9d..3220ce5f298 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml +++ b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml @@ -1,6 +1,32 @@ -bundle: {name: test-bundle} +bundle: + name: test-bundle resources: + # alerts: + # new_alert: + # display_name: "New alert" + # query_text: "select 2" + # warehouse_id: "${var.warehouse_id}" + # evaluation: + # comparison_operator: EQUAL + # source: + # name: "1" + # threshold: + # value: + # double_value: 1.3 + # schedule: + # pause_status: UNPAUSED + # quartz_cron_schedule: "44 19 */1 * * ?" + # timezone_id: Europe/Amsterdam + jobs: test_job: name: "Test Auto-Migrate Job" + # tasks: + # - task_key: check + # alert_task: + # alert_id: "${resources.alerts.new_alert.id}" + +variables: + warehouse_id: + description: Set via BUNDLE_VAR_warehouse_id diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/script b/acceptance/bundle/migrate/auto/new-resource-ref/script index 6b95079272d..332a9f9c727 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/script +++ b/acceptance/bundle/migrate/auto/new-resource-ref/script @@ -1,4 +1,5 @@ export DATABRICKS_BUNDLE_ENGINE= +export BUNDLE_VAR_warehouse_id=$TEST_DEFAULT_WAREHOUSE_ID title "Replay a prior terraform deployment of a job only" trace replay_tfstate.py terraform-requests.json terraform.tfstate @@ -8,9 +9,8 @@ rm -f "$OUT_REQUESTS" # Neither the job's terraform state nor the alert has the referenced id, so the migration must not # try to resolve it. title "Add a new alert and a job task that references it" -ALERT=' alerts: {new_alert: {display_name: "New alert", query_text: "select 2", warehouse_id: "'$TEST_DEFAULT_WAREHOUSE_ID'", evaluation: {comparison_operator: EQUAL, source: {name: "1"}, threshold: {value: {double_value: 1.3}}}, schedule: {pause_status: UNPAUSED, quartz_cron_schedule: "44 19 */1 * * ?", timezone_id: "Europe/Amsterdam"}}}' -update_file.py databricks.yml ' jobs:' "$ALERT"$'\n jobs:' -update_file.py databricks.yml 'name: "Test Auto-Migrate Job"' 'name: "Test Auto-Migrate Job"'$'\n tasks: [{task_key: check, alert_task: {alert_id: "${resources.alerts.new_alert.id}"}}]' +# Uncomment the alert and the task in databricks.yml. +update_file.py databricks.yml '# ' '' title "Deploy auto-migrates to direct and creates the alert and the task" trace $CLI bundle deploy From 66700cf7e9f284808ac66291ec5876eae5d60af3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 09:57:25 +0200 Subject: [PATCH 6/8] Fix panic on a ref to an undeployed resource in a number field A ref in a number field is already unset after conversion, so there is nothing to zero. Also skip refs that do not target a resource (for example variables) when deciding whether a target is undeployed. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 13 ++++++++++--- bundle/migrate/build_state_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 5c669d2ec59..62d68905dc4 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -204,9 +204,12 @@ func BuildStateFromTF( if err != nil { return warningsSeen, fmt.Errorf("%s: reading field %q: %w", node, pending.fieldPathStr, err) } - zero := reflect.Zero(reflect.TypeOf(current)).Interface() - if err := structaccess.Set(sv.Value, fieldPath, zero); err != nil { - return warningsSeen, fmt.Errorf("%s: cannot unset field %q: %w", node, pending.fieldPathStr, err) + // A nil value means the field is already unset (e.g. an unresolved ref in a number). + if current != nil { + zero := reflect.Zero(reflect.TypeOf(current)).Interface() + if err := structaccess.Set(sv.Value, fieldPath, zero); err != nil { + return warningsSeen, fmt.Errorf("%s: cannot unset field %q: %w", node, pending.fieldPathStr, err) + } } delete(sv.Refs, pending.fieldPathStr) continue @@ -279,6 +282,10 @@ func referencesUndeployed(refTemplate string, tfIDs map[string]string) bool { continue } targetNode, _ := config.GetNodeAndType(path) + // Not a resource reference (e.g. ${var.x}). + if len(targetNode) == 0 { + continue + } if _, ok := tfIDs[targetNode.String()]; !ok { return true } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 513e50a5b63..108c0591968 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -330,3 +330,32 @@ resources: assert.Contains(t, string(entry.State), `"alert_id": ""`) assert.NotContains(t, state, "resources.alerts.new_alert") } + +func TestBuildStateFromTFRefToUndeployedResourceInNumberField(t *testing.T) { + state := runBuildStateFromTF(t, ` +resources: + jobs: + new_job: + name: "new" + my_job: + name: "hello" + timeout_seconds: "${resources.jobs.new_job.timeout_seconds}" +`, + migrate.TFStateAttrs{"databricks_job": {"my_job": json.RawMessage(`{"id": "1", "name": "hello"}`)}}, + map[string]string{"resources.jobs.my_job": "1"}, + ) + assert.Contains(t, state, "resources.jobs.my_job") +} + +func TestBuildStateFromTFNonResourceRefIsNotUndeployed(t *testing.T) { + state := runBuildStateFromTF(t, ` +resources: + jobs: + my_job: + name: "${var.x}" +`, + migrate.TFStateAttrs{"databricks_job": {"my_job": json.RawMessage(`{"id": "1", "name": "hello"}`)}}, + map[string]string{"resources.jobs.my_job": "1"}, + ) + assert.Contains(t, string(state["resources.jobs.my_job"].State), `"name": "hello"`) +} From cbf7be8f1eec81841989a17553abfac669731fc3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 10:00:15 +0200 Subject: [PATCH 7/8] acc: cover a number field reference in new-resource-ref Co-authored-by: Isaac --- .../bundle/migrate/auto/new-resource-ref/databricks.yml | 3 +++ acceptance/bundle/migrate/auto/new-resource-ref/output.txt | 3 ++- acceptance/bundle/migrate/auto/new-resource-ref/script | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml index 3220ce5f298..b78d8f2a865 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml +++ b/acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml @@ -9,6 +9,8 @@ resources: # warehouse_id: "${var.warehouse_id}" # evaluation: # comparison_operator: EQUAL + # notification: + # retrigger_seconds: 300 # source: # name: "1" # threshold: @@ -22,6 +24,7 @@ resources: jobs: test_job: name: "Test Auto-Migrate Job" + # timeout_seconds: ${resources.alerts.new_alert.evaluation.notification.retrigger_seconds} # tasks: # - task_key: check # alert_task: diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt index 9fe1e356896..e1662a62522 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt +++ b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt @@ -13,9 +13,10 @@ Updated jobs.test_job Files: 1 uploaded, 0 deleted Resources: 1 created, 1 changed, 0 deleted, 0 unchanged -=== The job update carries the new alert id +=== The job update carries the new alert id and timeout >>> print_requests.py //api/2.2/jobs/reset json.body.new_settings.tasks[0].alert_task.alert_id = "[NEW_ALERT_ID]"; +json.body.new_settings.timeout_seconds = 300; === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/script b/acceptance/bundle/migrate/auto/new-resource-ref/script index 332a9f9c727..d4f7a0691e4 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/script +++ b/acceptance/bundle/migrate/auto/new-resource-ref/script @@ -9,15 +9,15 @@ rm -f "$OUT_REQUESTS" # Neither the job's terraform state nor the alert has the referenced id, so the migration must not # try to resolve it. title "Add a new alert and a job task that references it" -# Uncomment the alert and the task in databricks.yml. +# Uncomment the alert, the task and timeout_seconds (a number field with a reference) in databricks.yml. update_file.py databricks.yml '# ' '' title "Deploy auto-migrates to direct and creates the alert and the task" trace $CLI bundle deploy replace_ids.py -title "The job update carries the new alert id" -trace print_requests.py //api/2.2/jobs/reset | gron.py | grep alert_id +title "The job update carries the new alert id and timeout" +trace print_requests.py //api/2.2/jobs/reset | gron.py | grep -E "alert_id|timeout_seconds" rm -f "$OUT_REQUESTS" title "Redeploy is a no-op" From 394ed4cd9ec68c51bea8e9f367a6f9e1a171c7a4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 10:07:35 +0200 Subject: [PATCH 8/8] acc: record the JSON plan in the new resource ref test Co-authored-by: Isaac --- .../migrate/auto/new-resource-ref/output.txt | 145 +++++++++++++++++- .../migrate/auto/new-resource-ref/script | 6 +- 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt index e1662a62522..e3c122f31c1 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt +++ b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt @@ -3,7 +3,150 @@ >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Add a new alert and a job task that references it -=== Deploy auto-migrates to direct and creates the alert and the task +=== Plan shows the new alert and the job update (the terraform state is migrated in memory) +>>> [CLI] bundle plan -o json +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "plan": { + "resources.alerts.new_alert": { + "action": "create", + "new_state": { + "value": { + "display_name": "New alert", + "evaluation": { + "comparison_operator": "EQUAL", + "notification": { + "retrigger_seconds": 300 + }, + "source": { + "name": "1" + }, + "threshold": { + "value": { + "double_value": 1.3 + } + } + }, + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", + "query_text": "select 2", + "schedule": { + "pause_status": "UNPAUSED", + "quartz_cron_schedule": "44 19 */1 * * ?", + "timezone_id": "Europe/Amsterdam" + }, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + }, + "resources.jobs.test_job": { + "depends_on": [ + { + "node": "resources.alerts.new_alert", + "label": "${resources.alerts.new_alert.evaluation.notification.retrigger_seconds}" + }, + { + "node": "resources.alerts.new_alert", + "label": "${resources.alerts.new_alert.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Test Auto-Migrate Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "alert_task": { + "alert_id": "${resources.alerts.new_alert.id}" + }, + "task_key": "check" + } + ], + "timeout_seconds": 300 + }, + "vars": { + "tasks[0].alert_task.alert_id": "${resources.alerts.new_alert.id}" + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [TEST_JOB_ID], + "max_concurrent_runs": 1, + "name": "Test Auto-Migrate Job", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks": { + "action": "update", + "old": [ + { + "alert_task": { + "alert_id": "" + }, + "task_key": "check" + } + ], + "new": [ + { + "alert_task": { + "alert_id": "${resources.alerts.new_alert.id}" + }, + "task_key": "check" + } + ] + }, + "tasks[task_key='check'].alert_task.alert_id": { + "action": "update", + "old": "", + "new": "${resources.alerts.new_alert.id}" + }, + "timeout_seconds": { + "action": "update", + "new": 300, + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== Deploy migrates to direct, creates the alert and updates the job >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/script b/acceptance/bundle/migrate/auto/new-resource-ref/script index d4f7a0691e4..d2e16d8ef9c 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/script +++ b/acceptance/bundle/migrate/auto/new-resource-ref/script @@ -12,7 +12,11 @@ title "Add a new alert and a job task that references it" # Uncomment the alert, the task and timeout_seconds (a number field with a reference) in databricks.yml. update_file.py databricks.yml '# ' '' -title "Deploy auto-migrates to direct and creates the alert and the task" +title "Plan shows the new alert and the job update (the terraform state is migrated in memory)" +trace $CLI bundle plan -o json +rm -f "$OUT_REQUESTS" + +title "Deploy migrates to direct, creates the alert and updates the job" trace $CLI bundle deploy replace_ids.py