Fix terraform migration failing on refs to undeployed resources

.nextchanges/bundles/7157.md

@@ -0,0 +1 @@
+* Fix migration from Terraform failing when a deployed resource references a resource that was never deployed. ([#7157](https://github.com/databricks/cli/pull/7157))

acceptance/bundle/migrate/auto/new-resource-ref/databricks.yml

@@ -0,0 +1,35 @@
+bundle:
+  name: test-bundle
+
+resources:
+  # alerts:
+  #   new_alert:
+  #     display_name: "New alert"
+  #     query_text: "select 2"
+  #     warehouse_id: "${var.warehouse_id}"
+  #     evaluation:
+  #       comparison_operator: EQUAL
+  #       notification:
+  #         retrigger_seconds: 300
+  #       source:
+  #         name: "1"
+  #       threshold:
+  #         value:
+  #           double_value: 1.3
+  #     schedule:
+  #       pause_status: UNPAUSED
+  #       quartz_cron_schedule: "44 19 */1 * * ?"
+  #       timezone_id: Europe/Amsterdam
+
+  jobs:
+    test_job:
+      name: "Test Auto-Migrate Job"
+      # timeout_seconds: ${resources.alerts.new_alert.evaluation.notification.retrigger_seconds}
+      # tasks:
+      #   - task_key: check
+      #     alert_task:
+      #       alert_id: "${resources.alerts.new_alert.id}"
+
+variables:
+  warehouse_id:
+    description: Set via BUNDLE_VAR_warehouse_id

acceptance/bundle/migrate/auto/new-resource-ref/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/auto/new-resource-ref/output.txt

@@ -0,0 +1,175 @@
+
+=== Replay a prior terraform deployment of a job only
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+=== Add a new alert and a job task that references it
+=== Plan shows the new alert and the job update (the terraform state is migrated in memory)
+>>> [CLI] bundle plan -o json
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+{
+  "plan_version": [PLAN_VERSION],
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 3,
+  "plan": {
+    "resources.alerts.new_alert": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "display_name": "New alert",
+          "evaluation": {
+            "comparison_operator": "EQUAL",
+            "notification": {
+              "retrigger_seconds": 300
+            },
+            "source": {
+              "name": "1"
+            },
+            "threshold": {
+              "value": {
+                "double_value": 1.3
+              }
+            }
+          },
+          "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources",
+          "query_text": "select 2",
+          "schedule": {
+            "pause_status": "UNPAUSED",
+            "quartz_cron_schedule": "44 19 */1 * * ?",
+            "timezone_id": "Europe/Amsterdam"
+          },
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      }
+    },
+    "resources.jobs.test_job": {
+      "depends_on": [
+        {
+          "node": "resources.alerts.new_alert",
+          "label": "${resources.alerts.new_alert.evaluation.notification.retrigger_seconds}"
+        },
+        {
+          "node": "resources.alerts.new_alert",
+          "label": "${resources.alerts.new_alert.id}"
+        }
+      ],
+      "action": "update",
+      "new_state": {
+        "value": {
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+          },
+          "edit_mode": "UI_LOCKED",
+          "format": "MULTI_TASK",
+          "max_concurrent_runs": 1,
+          "name": "Test Auto-Migrate Job",
+          "queue": {
+            "enabled": true
+          },
+          "tasks": [
+            {
+              "alert_task": {
+                "alert_id": "${resources.alerts.new_alert.id}"
+              },
+              "task_key": "check"
+            }
+          ],
+          "timeout_seconds": 300
+        },
+        "vars": {
+          "tasks[0].alert_task.alert_id": "${resources.alerts.new_alert.id}"
+        }
+      },
+      "remote_state": {
+        "created_time": [UNIX_TIME_MILLIS],
+        "creator_user_name": "[USERNAME]",
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "email_notifications": {},
+        "format": "MULTI_TASK",
+        "job_id": [TEST_JOB_ID],
+        "max_concurrent_runs": 1,
+        "name": "Test Auto-Migrate Job",
+        "queue": {
+          "enabled": true
+        },
+        "run_as_user_name": "[USERNAME]",
+        "timeout_seconds": 0,
+        "webhook_notifications": {}
+      },
+      "changes": {
+        "email_notifications": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": {}
+        },
+        "tasks": {
+          "action": "update",
+          "old": [
+            {
+              "alert_task": {
+                "alert_id": ""
+              },
+              "task_key": "check"
+            }
+          ],
+          "new": [
+            {
+              "alert_task": {
+                "alert_id": "${resources.alerts.new_alert.id}"
+              },
+              "task_key": "check"
+            }
+          ]
+        },
+        "tasks[task_key='check'].alert_task.alert_id": {
+          "action": "update",
+          "old": "",
+          "new": "${resources.alerts.new_alert.id}"
+        },
+        "timeout_seconds": {
+          "action": "update",
+          "new": 300,
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": {}
+        }
+      }
+    }
+  }
+}
+
+=== Deploy migrates to direct, creates the alert and updates the job
+>>> [CLI] bundle deploy
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Migrated 1 resource to direct deployment engine.
+Created alerts.new_alert
+Updated jobs.test_job
+Files: 1 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 0 deleted, 0 unchanged
+
+=== The job update carries the new alert id and timeout
+>>> print_requests.py //api/2.2/jobs/reset
+json.body.new_settings.tasks[0].alert_task.alert_id = "[NEW_ALERT_ID]";
+json.body.new_settings.timeout_seconds = 300;
+
+=== Redeploy is a no-op
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.alerts.new_alert
+  delete resources.jobs.test_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Destroy: 2 deleted

acceptance/bundle/migrate/auto/new-resource-ref/script

@@ -0,0 +1,31 @@
+export DATABRICKS_BUNDLE_ENGINE=
+export BUNDLE_VAR_warehouse_id=$TEST_DEFAULT_WAREHOUSE_ID
+
+title "Replay a prior terraform deployment of a job only"
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+rm -f "$OUT_REQUESTS"
+
+# The job gains an alert_task that references a new alert, neither of which terraform ever deployed.
+# Neither the job's terraform state nor the alert has the referenced id, so the migration must not
+# try to resolve it.
+title "Add a new alert and a job task that references it"
+# Uncomment the alert, the task and timeout_seconds (a number field with a reference) in databricks.yml.
+update_file.py databricks.yml '# ' ''
+
+title "Plan shows the new alert and the job update (the terraform state is migrated in memory)"
+trace $CLI bundle plan -o json
+rm -f "$OUT_REQUESTS"
+
+title "Deploy migrates to direct, creates the alert and updates the job"
+trace $CLI bundle deploy
+replace_ids.py
+
+title "The job update carries the new alert id and timeout"
+trace print_requests.py //api/2.2/jobs/reset | gron.py | grep -E "alert_id|timeout_seconds"
+rm -f "$OUT_REQUESTS"
+
+title "Redeploy is a no-op"
+trace $CLI bundle plan
+
+trace $CLI bundle destroy --auto-approve
+rm -f "$OUT_REQUESTS"

acceptance/bundle/migrate/auto/new-resource-ref/terraform-requests.json

@@ -0,0 +1,19 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "Test Auto-Migrate Job",
+      "queue": {
+        "enabled": true
+      }
+    }
+  }
+]

acceptance/bundle/migrate/auto/new-resource-ref/terraform.tfstate

@@ -0,0 +1,110 @@
+{
+  "version": 4,
+  "terraform_version": "1.5.5",
+  "serial": 1,
+  "lineage": "c39488aa-8f36-06fd-a7cb-3f0c95b4c506",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "databricks_job",
+      "name": "test_job",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 2,
+          "attributes": {
+            "always_running": false,
+            "budget_policy_id": null,
+            "continuous": [],
+            "control_run_state": false,
+            "dbt_task": [],
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "description": null,
+            "edit_mode": "UI_LOCKED",
+            "email_notifications": [
+              {
+                "no_alert_for_skipped_runs": false,
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ],
+            "environment": [],
+            "existing_cluster_id": null,
+            "format": "MULTI_TASK",
+            "git_source": [],
+            "health": [],
+            "id": "8790612527183335000",
+            "job_cluster": [],
+            "library": [],
+            "max_concurrent_runs": 1,
+            "max_retries": 0,
+            "min_retry_interval_millis": 0,
+            "name": "Test Auto-Migrate Job",
+            "new_cluster": [],
+            "notebook_task": [],
+            "notification_settings": [],
+            "parameter": [],
+            "parent_path": null,
+            "performance_target": null,
+            "pipeline_task": [],
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "python_wheel_task": [],
+            "queue": [
+              {
+                "enabled": true
+              }
+            ],
+            "retry_on_timeout": false,
+            "run_as": [
+              {
+                "group_name": "",
+                "service_principal_name": "",
+                "user_name": "tester@databricks.com"
+              }
+            ],
+            "run_job_task": [],
+            "schedule": [],
+            "spark_jar_task": [],
+            "spark_python_task": [],
+            "spark_submit_task": [],
+            "tags": null,
+            "task": [],
+            "timeout_seconds": 0,
+            "timeouts": null,
+            "trigger": [],
+            "triggers": [],
+            "url": "[DATABRICKS_URL]/#job/8790612527183335000",
+            "usage_policy_id": null,
+            "webhook_notifications": [
+              {
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ]
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
+        }
+      ]
+    }
+  ],
+  "check_results": null
+}

acceptance/bundle/migrate/auto/new-resource-ref/test.toml

@@ -0,0 +1,9 @@
+# Local-only: the replay fixtures are captured against the fake server, so they reproduce a
+# terraform state only there, not on a real workspace (replaying them on cloud fails auth and
+# hits backend validation the mock does not enforce).
+Cloud = false
+RecordRequests = true
+
+Ignore = [
+    ".databricks",
+]