Fix another pipeline recreate test

pietern · pietern · commit 983571e61504 · 2025-12-04T11:45:47.000+01:00
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/databricks.yml b/acceptance/bundle/resource_deps/pipelines_recreate/databricks.yml
@@ -2,7 +2,7 @@ resources:
   pipelines:
     foo:
       name: pipeline foo
-      catalog: mycatalog
+      storage: dbfs:/my-storage
   jobs:
     bar:
       name: job bar
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.create.requests.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.create.requests.json
@@ -2,14 +2,14 @@
   "method": "POST",
   "path": "/api/2.0/pipelines",
   "body": {
-    "catalog": "mycatalog",
     "channel": "CURRENT",
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
     },
     "edition": "ADVANCED",
-    "name": "pipeline foo"
+    "name": "pipeline foo",
+    "storage": "dbfs:/my-storage"
   }
 }
 {
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json
@@ -32,14 +32,14 @@
       "action": "create",
       "new_state": {
         "value": {
-          "catalog": "mycatalog",
           "channel": "CURRENT",
           "deployment": {
             "kind": "BUNDLE",
             "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
           },
           "edition": "ADVANCED",
-          "name": "pipeline foo"
+          "name": "pipeline foo",
+          "storage": "dbfs:/my-storage"
         }
       }
     }
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json
@@ -57,15 +57,15 @@
         "pipeline_id": "[UUID]",
         "run_as_user_name": "[USERNAME]",
         "spec": {
-          "catalog": "mynewcatalog",
           "channel": "CURRENT",
           "deployment": {
             "kind": "BUNDLE",
             "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
           },
           "edition": "ADVANCED",
           "id": "[UUID]",
-          "name": "pipeline foo"
+          "name": "pipeline foo",
+          "storage": "dbfs:/my-new-storage"
         },
         "state": "IDLE"
       }
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json
@@ -0,0 +1,10 @@
+{
+  "plan": {
+    "resources.jobs.bar": {
+      "action": "skip"
+    },
+    "resources.pipelines.foo": {
+      "action": "skip"
+    }
+  }
+}
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json
@@ -78,22 +78,22 @@
       "action": "recreate",
       "new_state": {
         "value": {
-          "catalog": "mynewcatalog",
           "channel": "CURRENT",
           "deployment": {
             "kind": "BUNDLE",
             "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
           },
           "edition": "ADVANCED",
-          "name": "pipeline foo"
+          "name": "pipeline foo",
+          "storage": "dbfs:/my-new-storage"
         }
       },
       "changes": {
         "local": {
-          "catalog": {
+          "storage": {
             "action": "recreate",
-            "old": "mycatalog",
-            "new": "mynewcatalog"
+            "old": "dbfs:/my-storage",
+            "new": "dbfs:/my-new-storage"
           }
         }
       }
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.terraform.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.terraform.json
@@ -1,10 +1,10 @@
 {
   "plan": {
     "resources.jobs.bar": {
-      "action": "skip"
+      "action": "update"
     },
     "resources.pipelines.foo": {
-      "action": "update"
+      "action": "recreate"
     }
   }
 }
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.requests.txt b/acceptance/bundle/resource_deps/pipelines_recreate/out.requests.txt
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.direct.json
@@ -6,14 +6,14 @@
   "method": "POST",
   "path": "/api/2.0/pipelines",
   "body": {
-    "catalog": "mynewcatalog",
     "channel": "CURRENT",
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
     },
     "edition": "ADVANCED",
-    "name": "pipeline foo"
+    "name": "pipeline foo",
+    "storage": "dbfs:/my-new-storage"
   }
 }
 {
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.terraform.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.terraform.json
@@ -1,15 +1,44 @@
 {
-  "method": "PUT",
-  "path": "/api/2.0/pipelines/[FOO_ID]",
+  "method": "DELETE",
+  "path": "/api/2.0/pipelines/[FOO_ID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/pipelines",
   "body": {
-    "catalog": "mynewcatalog",
     "channel": "CURRENT",
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
     },
     "edition": "ADVANCED",
-    "id": "[FOO_ID]",
-    "name": "pipeline foo"
+    "name": "pipeline foo",
+    "storage": "dbfs:/my-new-storage"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/reset",
+  "body": {
+    "job_id": [BAR_ID],
+    "new_settings": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "description": "depends on foo id [UUID]",
+      "edit_mode": "UI_LOCKED",
+      "email_notifications": {},
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "job bar",
+      "queue": {
+        "enabled": true
+      },
+      "run_as": {
+        "user_name": "[USERNAME]"
+      },
+      "webhook_notifications": {}
+    }
   }
 }
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt
@@ -16,16 +16,23 @@ Deployment complete!
 >>> [CLI] bundle plan
 Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 
-=== Update catalog, triggering recreate for pipeline; this means updating downstream deps
->>> update_file.py databricks.yml mycatalog mynewcatalog
+=== Update storage, triggering recreate for pipeline; this means updating downstream deps
+>>> update_file.py databricks.yml storage: dbfs:/my-storage storage: dbfs:/my-new-storage
 
 >>> [CLI] bundle plan
-update pipelines.foo
+update jobs.bar
+recreate pipelines.foo
 
-Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged
+Plan: 1 to add, 1 to change, 1 to delete, 0 unchanged
 
 >>> [CLI] bundle deploy --auto-approve
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+
+This action will result in the deletion or recreation of the following Lakeflow Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will
+restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
+properties such as the 'catalog' or 'storage' are changed:
+  recreate resources.pipelines.foo
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -34,26 +41,98 @@ Deployment complete!
 
 === Fetch resource IDs and verify remote state
 >>> musterr [CLI] pipelines get [FOO_ID]
+Error: The specified pipeline [FOO_ID] was not found.
+
+>>> [CLI] pipelines get [UUID]
 {
   "creator_user_name":"[USERNAME]",
-  "last_modified":[UNIX_TIME_MILLIS],
+  "last_modified":[UNIX_TIME_MILLIS][0],
   "name":"pipeline foo",
-  "pipeline_id":"[FOO_ID]",
+  "pipeline_id":"[UUID]",
   "run_as_user_name":"[USERNAME]",
   "spec": {
-    "catalog":"mynewcatalog",
     "channel":"CURRENT",
     "deployment": {
       "kind":"BUNDLE",
       "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
     },
     "edition":"ADVANCED",
-    "id":"[FOO_ID]",
-    "name":"pipeline foo"
+    "id":"[UUID]",
+    "name":"pipeline foo",
+    "storage":"dbfs:/my-new-storage"
   },
   "state":"IDLE"
 }
 
-Unexpected success
+>>> [CLI] jobs get [BAR_ID]
+{
+  "created_time": [UNIX_TIME_MILLIS][1],
+  "creator_user_name": "[USERNAME]",
+  "job_id": [BAR_ID],
+  "run_as_user_name": "[USERNAME]",
+  "settings": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+    },
+    "description": "depends on foo id [UUID]",
+    "edit_mode": "UI_LOCKED",
+    "email_notifications": {},
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "job bar",
+    "queue": {
+      "enabled": true
+    },
+    "timeout_seconds": 0,
+    "webhook_notifications": {}
+  }
+}
+
+=== Follow up plan & deploy do nothing
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs //pipelines
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.bar
+  delete resources.pipelines.foo
+
+This action will result in the deletion of the following Lakeflow Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them:
+  delete resources.pipelines.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py --sort //jobs //pipelines
+{
+  "method": "DELETE",
+  "path": "/api/2.0/pipelines/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [BAR_ID]
+  }
+}
+
+>>> musterr [CLI] pipelines get [FOO_ID]
+Error: The specified pipeline [FOO_ID] was not found.
+
+>>> musterr [CLI] pipelines get [UUID]
+Error: The specified pipeline [UUID] was not found.
 
-Exit code: 1
+>>> musterr [CLI] jobs get [BAR_ID]
+Error: Not Found
diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script
@@ -13,8 +13,8 @@ echo "$bar_id:BAR_ID" >> ACC_REPLS
 
 trace $CLI bundle plan  # empty
 
-title "Update catalog, triggering recreate for pipeline; this means updating downstream deps"
-trace update_file.py databricks.yml mycatalog mynewcatalog
+title "Update storage, triggering recreate for pipeline; this means updating downstream deps"
+trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage"
 trace $CLI bundle plan
 $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy --auto-approve

Original file line number	Diff line number	Diff line change
`@@ -2,14 +2,14 @@`
`2`	`2`	`"method": "POST",`
`3`	`3`	`"path": "/api/2.0/pipelines",`
`4`	`4`	`"body": {`
`5`		`- "catalog": "mycatalog",`
`6`	`5`	`"channel": "CURRENT",`
`7`	`6`	`"deployment": {`
`8`	`7`	`"kind": "BUNDLE",`
`9`	`8`	`"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"`
`10`	`9`	`},`
`11`	`10`	`"edition": "ADVANCED",`
`12`		`- "name": "pipeline foo"`
	`11`	`+ "name": "pipeline foo",`
	`12`	`+ "storage": "dbfs:/my-storage"`
`13`	`13`	`}`
`14`	`14`	`}`
`15`	`15`	`{`
Original file line number	Diff line number	Diff line change
`@@ -32,14 +32,14 @@`
`32`	`32`	`"action": "create",`
`33`	`33`	`"new_state": {`
`34`	`34`	`"value": {`
`35`		`- "catalog": "mycatalog",`
`36`	`35`	`"channel": "CURRENT",`
`37`	`36`	`"deployment": {`
`38`	`37`	`"kind": "BUNDLE",`
`39`	`38`	`"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"`
`40`	`39`	`},`
`41`	`40`	`"edition": "ADVANCED",`
`42`		`- "name": "pipeline foo"`
	`41`	`+ "name": "pipeline foo",`
	`42`	`+ "storage": "dbfs:/my-storage"`
`43`	`43`	`}`
`44`	`44`	`}`
`45`	`45`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,10 @@`
`1`	`1`	`{`
`2`	`2`	`"plan": {`
`3`	`3`	`"resources.jobs.bar": {`
`4`		`- "action": "skip"`
	`4`	`+ "action": "update"`
`5`	`5`	`},`
`6`	`6`	`"resources.pipelines.foo": {`
`7`		`- "action": "update"`
	`7`	`+ "action": "recreate"`
`8`	`8`	`}`
`9`	`9`	`}`
`10`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,14 +6,14 @@`
`6`	`6`	`"method": "POST",`
`7`	`7`	`"path": "/api/2.0/pipelines",`
`8`	`8`	`"body": {`
`9`		`- "catalog": "mynewcatalog",`
`10`	`9`	`"channel": "CURRENT",`
`11`	`10`	`"deployment": {`
`12`	`11`	`"kind": "BUNDLE",`
`13`	`12`	`"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"`
`14`	`13`	`},`
`15`	`14`	`"edition": "ADVANCED",`
`16`		`- "name": "pipeline foo"`
	`15`	`+ "name": "pipeline foo",`
	`16`	`+ "storage": "dbfs:/my-new-storage"`
`17`	`17`	`}`
`18`	`18`	`}`
`19`	`19`	`{`