fix(configurator): clean stale runners and unblock additive IAM drafts

landing-zone-configurator/app/apps/api/db/076_runner_retention.sql

@@ -0,0 +1,54 @@
+-- Only stopped CF execution apps are candidates. Job/history/State rows are retained.
+CREATE TABLE lzc.runner_cleanup_receipts (
+ app_id uuid PRIMARY KEY,
+ runner_id uuid NOT NULL,
+ kind text NOT NULL CHECK(kind IN ('plan','warm')),
+ removed_at timestamptz NOT NULL DEFAULT clock_timestamp()
+);
+ALTER TABLE lzc.runner_cleanup_receipts ENABLE ROW LEVEL SECURITY;
+ALTER TABLE lzc.runner_cleanup_receipts FORCE ROW LEVEL SECURITY;
+CREATE POLICY runner_cleanup_migration ON lzc.runner_cleanup_receipts TO configurator_migration USING(true) WITH CHECK(true);
+
+CREATE INDEX plan_runner_retention_app ON lzc.plan_runs(runner_app_id) WHERE runner_app_id IS NOT NULL;
+CREATE INDEX application_runner_retention_app ON lzc.application_dispatches(runner_app_id) WHERE runner_app_id IS NOT NULL;
+CREATE INDEX pool_runner_retention_app ON lzc.runner_pool_slots(app_id) WHERE app_id IS NOT NULL;
+CREATE INDEX recovery_runner_retention_app ON lzc.application_state_recovery_tasks(runner_app_id) WHERE runner_app_id IS NOT NULL;
+
+-- A missing association is distinct from a durably finished association. Only
+-- new origin-labelled apps may use the orphan path; legacy names alone are not ownership.
+CREATE FUNCTION lzc_auth.runner_cleanup_disposition(p_runner uuid,p_app uuid,p_kind text)
+RETURNS text LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$
+DECLARE known boolean:=false;
+BEGIN
+ IF p_kind NOT IN ('plan','warm') THEN RETURN 'protected'; END IF;
+ IF EXISTS(SELECT 1 FROM lzc.application_jobs j WHERE j.id=p_runner AND NOT EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE d.job_id=j.id AND d.runner_app_id=p_app))
+  OR EXISTS(SELECT 1 FROM lzc.application_state_recovery_tasks WHERE id=p_runner OR runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.runner_pool_slots WHERE (id=p_runner OR app_id=p_app) AND
+   (app_id IS DISTINCT FROM p_app OR (p_kind='warm' AND id<>p_runner) OR expires_at>now() OR status IN ('warming','ready','reserved','assigned')))
+  OR EXISTS(SELECT 1 FROM lzc.plan_runs WHERE (id=p_runner OR runner_app_id=p_app) AND
+   (runner_app_id IS DISTINCT FROM p_app OR (p_kind='plan' AND id<>p_runner) OR expires_at>now() OR finished_at IS NULL OR finished_at>now()-interval '30 seconds' OR status NOT IN ('succeeded','failed','cancelled')))
+  OR EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE (job_id=p_runner OR runner_app_id=p_app) AND
+   (runner_app_id IS DISTINCT FROM p_app OR (p_kind='plan' AND job_id<>p_runner) OR finished_at IS NULL OR finished_at>now()-interval '30 seconds'
+    OR NOT (status IN ('succeeded','failed') OR (status='reconciliation_required' AND EXISTS(SELECT 1 FROM lzc.application_recovery_replans r WHERE r.job_id=d.job_id)))
+    OR EXISTS(SELECT 1 FROM lzc.application_runner_tickets t WHERE t.job_id=d.job_id AND t.expires_at>now())
+    OR EXISTS(SELECT 1 FROM lzc.application_state_recoveries r WHERE r.job_id=d.job_id AND r.phase IN ('requested','assessing','prepared','committing','reconciliation-required'))))
+ THEN RETURN 'protected'; END IF;
+ -- Consumed warm slots without a completed associated job are never mere orphans.
+ IF EXISTS(SELECT 1 FROM lzc.runner_pool_slots s WHERE s.app_id=p_app AND s.status='consumed'
+  AND NOT EXISTS(SELECT 1 FROM lzc.plan_runs p WHERE p.id=s.job_id AND p.runner_app_id=p_app)
+  AND NOT EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE d.job_id=s.job_id AND d.runner_app_id=p_app))
+ THEN RETURN 'protected'; END IF;
+ SELECT EXISTS(SELECT 1 FROM lzc.plan_runs WHERE runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.application_dispatches WHERE runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.runner_pool_slots WHERE app_id=p_app AND id=p_runner AND p_kind='warm') INTO known;
+ RETURN CASE WHEN known THEN 'completed' ELSE 'orphan' END;
+END $$;
+REVOKE ALL ON FUNCTION lzc_auth.runner_cleanup_disposition(uuid,uuid,text) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION lzc_auth.runner_cleanup_disposition(uuid,uuid,text) TO configurator_app;
+
+CREATE FUNCTION lzc_auth.record_runner_cleanup(p_runner uuid,p_app uuid,p_kind text) RETURNS void
+LANGUAGE sql SECURITY DEFINER SET search_path=pg_catalog AS $$
+ INSERT INTO lzc.runner_cleanup_receipts(app_id,runner_id,kind) VALUES(p_app,p_runner,p_kind) ON CONFLICT(app_id) DO NOTHING;
+$$;
+REVOKE ALL ON FUNCTION lzc_auth.record_runner_cleanup(uuid,uuid,text) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION lzc_auth.record_runner_cleanup(uuid,uuid,text) TO configurator_app;

landing-zone-configurator/app/apps/api/src/assistant/knowledge-topics.generated.ts

@@ -407,7 +407,7 @@ export const assistantKnowledgeTopics = {
         {
           path: "landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts",
           sha256:
-            "c52a6b232e658d0d4224d4595baca70c8974284d67448addbb98390846af0e6e",
+            "f34bfcb1b5d8e0f2149d4779223e663e4bedab6326f955d6416cafd77d585401",
         },
         {
           path: "landing-zone-configurator/app/apps/api/src/assistant/editor.ts",
@@ -1393,9 +1393,9 @@ export const assistantKnowledgeTopics = {
       ],
       source:
         "landing-zone-configurator/docs/assistant-knowledge/project-templates.md",
-      text: '# Landing-Zone Projects and Application Templates\n\n## Purpose\n\nTemplates/IAM.\n\n## Accelerator Contract\n\nlanding_zones: corporate/region/area, project_code/env/owner_email, network/roles/services.\nPublic:corporate=false. Drafts: projectTemplates/policies, no instance/publication fields.\naccessProfile: granular/owner; separate central initial owner/automation.\nAutomatic=null. Corporate Accelerator reads attached regional SNA default_prefix_length\nbefore Plan; supplies explicit numeric prefix to Provider0.114, preserves DNS.\nProvider0.114 rejects null prefix WITH DNS (even []): Invalid IPv4 configuration.\nAccelerator resolves it; no provider-upgrade fix. Explicit overrides win; Public provider sizing.\n\n## Dependencies\n\nGovernance, verified identity, same-region Corporate SNA, separate credentials.\nPublic!=Corporate spoke. Namespace services require Platform Kubernetes.\n\n## Dialogue\n\nNew: name/kind/region/settings; reuse keys, delegate names only.\nCentral=Corporate; Internet-only=Public; local network=true. Merge, retain Audit/Git,\nOmit unrequested services; ask missing choices. "100 projects"=capacity, not examples/quota;\nReserve platform/firewall space; no free-capacity promise. /26: network_prefix_length=26;\nAutomatic=null; no mandatory /24/numeric choice/disabled DNS. Preserve other settings.\nNew: granular recommended; owner customer choice.\nIDP, Configurator roles and deployed project IAM are separate.\nImmutable accessProfile, never order input. Granular schemaVersion=1,mode=granular:\ninitialOwnerEmail=user/SA, not requester; permissions/automationPermissions use official IDs,\ne.g. iaas.server.get, never compute.*. Default144: VM,Object Storage,Observability,Secrets Manager;\nVM-only narrower. All project rights selectable, including IAM/admin.\nTrusted catalogue IDs only. ADD Kubernetes/SKE=ske rights by union into requested lists\n(default both), preserving owner/rights; no unrelated IAM. SKE IAM!=Kubernetes RBAC.\nIAM grants can escalate; excluded from defaults.\nOwner={schemaVersion:1,mode:owner}: requester/automation have all project rights;\nIAM/project powers; no immutable central audit/routing claim.\nComplete atomic profile; preserve other fields. No legacy custom_roles/role_assignments/\nautomation_role or role-context combination. Ask missing central owner. Absent profile preserves\nlegacy owner; inherited grants stay additive.\n\n## Examples\n\nSNA /28: Automatic /28+DNS.\n\n## Limits\n\nOld versions/State bound; changes need reviewed source/publication/migration. Never delete State.\nRegion/area/DNS match; empty roles/ACLs prove no security. Effective rights need live acceptance, not mock Plans. Not owner-less.\n\n## Credentials\n\nSeparate access/runtime grants; no secrets/profile IDs in chat/exports.\n\n## Sources\n\nIaC.\n',
+      text: '# Landing-Zone Projects and Application Templates\n\n## Purpose\n\nTemplates/IAM.\n\n## Accelerator Contract\n\nlanding_zones: corporate/region/area, project_code/env/owner_email, network/roles/services.\nPublic:corporate=false. Drafts: projectTemplates/policies, no instance/publication fields.\naccessProfile: granular/owner; separate central initial owner/automation.\nAutomatic=null. Corporate Accelerator reads attached regional SNA default_prefix_length\nbefore Plan; supplies explicit numeric prefix to Provider0.114, preserves DNS.\nProvider0.114 rejects null prefix WITH DNS (even []): Invalid IPv4 configuration.\nAccelerator resolves it; no provider-upgrade fix. Explicit overrides win; Public provider sizing.\n\n## Dependencies\n\nGovernance, verified identity, same-region Corporate SNA, separate credentials.\nPublic!=Corporate spoke. Namespace services require Platform Kubernetes.\n\n## Dialogue\n\nNew: name/kind/region/settings; reuse keys; delegate names only.\nCentral=Corporate; Internet-only=Public; local network=true. Merge, retain Audit/Git,\nOmit unrequested services; ask missing choices. "100 projects"=capacity, not examples/quota;\nReserve platform/firewall space; no free-capacity promise. /26: network_prefix_length=26;\nAutomatic=null; no mandatory /24/numeric choice/disabled DNS. Preserve other settings.\nNew: granular recommended; owner customer choice.\nIDP, Configurator roles and deployed project IAM are separate.\nImmutable accessProfile, never order input. Granular schemaVersion=1,mode=granular:\ninitialOwnerEmail=user/SA, not requester; permissions/automationPermissions: official IDs,\ne.g. iaas.server.get; no compute.*. Default144: VM,Object Storage,Observability,Secrets Manager;\nVM-only narrower. All project rights selectable, including IAM/admin.\nCatalogue IDs only. ADD Kubernetes/SKE=ske union into requested lists\n(default both), preserving owner/rights; no unrelated IAM. SKE IAM!=Kubernetes RBAC.\nIAM grants can escalate; excluded from defaults.\nOwner={schemaVersion:1,mode:owner}: requester/automation have all project rights;\nIAM/project powers; no immutable central audit/routing claim.\nAtomic profile; preserve other fields. No legacy custom_roles/role_assignments/\nautomation_role/role-context combination. ADD preserves even empty owner; no owner question.\nPublication needs valid owner; new profile asks missing owner. Absent profile preserves\nlegacy owner; inherited grants stay additive.\n\n## Examples\n\nSNA /28: Automatic /28+DNS.\n\n## Limits\n\nVersions/State bound; review source/publication/migration. Never delete State.\nMatch region/area/DNS. Empty roles/ACLs prove no security. Rights need live acceptance, not mock Plans. Owner required.\n\n## Credentials\n\nSeparate access/runtime grants; no secrets/profile IDs in chat/exports.\n\n## Sources\n\nIaC.\n',
       sha256:
-        "18d2ea300edc1bf8a2fadace7fe615e82f616b0e9274328ff6b9ebcdc8f9a4fa",
+        "3c2ecdc335042a1b3ff2b53ce1814b1c7e2fbd3fe316193e4feb5222a4aa697b",
     },
     {
       id: "sandbox-projects",
@@ -1580,7 +1580,7 @@ export const assistantKnowledgeTopics = {
         {
           path: "landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts",
           sha256:
-            "c52a6b232e658d0d4224d4595baca70c8974284d67448addbb98390846af0e6e",
+            "f34bfcb1b5d8e0f2149d4779223e663e4bedab6326f955d6416cafd77d585401",
         },
         {
           path: "landing-zone-configurator/app/apps/api/src/assistant/knowledge.ts",

landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts

@@ -1430,7 +1430,7 @@ export class AssistantModelServing {
           {
             role: "system",
             content:
-              "Application project IAM: distinguish information questions from desired changes. If the CURRENT USER supplies literal arrays for both permissions and automationPermissions, choose applicationAccess custom-granular with only key, initialOwnerEmail and listOperation: the server uses those exact field-labelled lists, so do not echo them and never substitute a preset or generic templateSetup. Set listOperation=replace only for a complete desired set; choose append for Ergänze/add/append requests, preserving each recipient and central owner; choose clarify when the two recipients request different operations or the operation is unclear. Advice remains applicationAccess=null. An accessProfile-only change on an existing template MUST use applicationAccess, not generic templateSetup/editorInputs. Full project Owner rights explicitly requested for requester and automation select choice=owner (schemaVersion 1, mode owner), with no granular fields; do not substitute the workload default. A wish that future orderers receive Kubernetes/other service rights is an explicit template-change request, even without the words add/proposal: use topic=project-templates, mode=draft, adviceTask=other, applicationAccess.choice=add-service-permissions. Granting a future requester the ability to order/use a Kubernetes cluster means adding SKE project IAM to the existing template, NOT creating a cluster, platform Kubernetes or namespace building block. If only Public/Corporate is specified, use key=__select_by_kind__, templateKind and exact templateEvidence from the current message. The server asks which when several match. If the request concerns what the orderer receives, permissionTarget=requester; do not also change Application-service-account permissions unless requested. Preserve the existing central owner and all other settings. Actual advice questions about what roles mean use mode=advice with applicationAccess=null. Never treat a desired permission change as a request for the general access overview. If pendingPermissionSelection is supplied and the current user chooses that template, continue its original USER service request using that controlled serviceEvidenceSource and key, preserving the original services and recipient. The current selection is not a new advice request. Otherwise never reactivate old permission tasks.",
+              "Application project IAM: distinguish information questions from desired changes. If the CURRENT USER supplies literal arrays for both permissions and automationPermissions, choose applicationAccess custom-granular with only key, initialOwnerEmail and listOperation: the server uses those exact field-labelled lists, so do not echo them and never substitute a preset or generic templateSetup. Set listOperation=replace only for a complete desired set; choose append for Ergänze/add/append requests, preserving each recipient and central owner; choose clarify when the two recipients request different operations or the operation is unclear. Advice remains applicationAccess=null. An accessProfile-only change on an existing template MUST use applicationAccess, not generic templateSetup/editorInputs. Full project Owner rights explicitly requested for requester and automation select choice=owner (schemaVersion 1, mode owner), with no granular fields; do not substitute the workload default. A wish that future orderers receive Kubernetes/other service rights is an explicit template-change request, even without the words add/proposal: use topic=project-templates, mode=draft, adviceTask=other, applicationAccess.choice=add-service-permissions. Granting a future requester the ability to order/use a Kubernetes cluster means adding SKE project IAM to the existing template, NOT creating a cluster, platform Kubernetes or namespace building block. If only Public/Corporate is specified, use key=__select_by_kind__, templateKind and exact templateEvidence from the current message. The server asks which when several match. If the request concerns what the orderer receives, permissionTarget=requester; do not also change Application-service-account permissions unless requested. Preserve the existing central owner and all other settings, including an unfinished empty owner field: adding service permissions to a draft must still produce a proposal, not a new owner/preset question. Publication separately requires a valid initial owner. Actual advice questions about what roles mean use mode=advice with applicationAccess=null. Never treat a desired permission change as a request for the general access overview. If pendingPermissionSelection is supplied and the current user chooses that template, continue its original USER service request using that controlled serviceEvidenceSource and key, preserving the original services and recipient. The current selection is not a new advice request. Otherwise never reactivate old permission tasks.",
           },
           {
             role: "system",
@@ -1892,6 +1892,7 @@ export class AssistantModelServing {
         }
       }
       if (
+        selected.choice !== "add-service-permissions" &&
         profile.mode === "granular" &&
         !z.email().safeParse(profile.initialOwnerEmail).success
       ) {
@@ -1926,7 +1927,7 @@ export class AssistantModelServing {
           settings,
           objectValue(objectValue(template.parameterPolicy).fields)
             .role_assignments,
-          true,
+          selected.choice !== "add-service-permissions",
         );
         if (unknownApplicationPermissions(profile, livePermissionNames).length)
           throw new Error("assistant_editor_value_unconfirmed");

landing-zone-configurator/app/apps/api/src/main.ts

@@ -27,6 +27,7 @@ import { CloudFoundryPlanRunner } from "./plans/cloud-foundry.js";
 import { ArtifactCrypto } from "./plans/crypto.js";
 import { PlatformExecution } from "./plans/execution.js";
 import { LocalPlanRunner } from "./plans/local.js";
+import { RunnerCleanup } from "./plans/runner-cleanup.js";
 import { RunnerPool, runnerPoolSize } from "./plans/runner-pool.js";
 import { Plans } from "./plans/service.js";
 import { databaseConfig } from "./storage/database.js";
@@ -48,6 +49,7 @@ let stackit: StackitServices | undefined;
 let applicationRunner: CloudFoundryPlanRunner | undefined;
 let applicationRunnerPackageId: string | undefined;
 let runnerPool: RunnerPool | undefined;
+let runnerCleanup: RunnerCleanup | undefined;
 const warmRunnerCount = runnerPoolSize(process.env.LZC_RUNNER_POOL_SIZE);
 const executionEnabled = process.env.LZC_EXECUTION_ENABLED === "true";
 const applicationExecutionEnabled =
@@ -130,6 +132,11 @@ if (process.env.LZC_AUTH_ENABLED === "true") {
       )
     : undefined;
   if (process.env.LZC_PLANS_ENABLED === "true") {
+    runnerCleanup = new RunnerCleanup(
+      pool,
+      new CloudFoundryPlanRunner(runnerConfiguration()),
+      origin,
+    );
     runnerPool = new RunnerPool(
       pool,
       artifactCrypto,
@@ -346,10 +353,37 @@ const poolMaintenance = runnerPool
   ? setInterval(() => void maintainRunners(), 5000)
   : undefined;
 poolMaintenance?.unref();
+let cleanupMaintenance: Promise<void> | undefined;
+const cleanupTimer = runnerCleanup
+  ? setInterval(() => {
+      if (cleanupMaintenance) return;
+      cleanupMaintenance = runnerCleanup!
+        .maintain()
+        .then((summary) => {
+          if (summary)
+            app.log.info(
+              { event: "runner_retention", ...summary },
+              "Runner retention checked",
+            );
+        })
+        .catch(() =>
+          app.log.warn(
+            { event: "runner_retention_failed" },
+            "Runner retention failed",
+          ),
+        )
+        .finally(() => {
+          cleanupMaintenance = undefined;
+        });
+    }, 60000)
+  : undefined;
+cleanupTimer?.unref();
 app.addHook("onClose", async () => {
   clearInterval(maintenance);
   clearInterval(poolMaintenance);
+  clearInterval(cleanupTimer);
   await runnerMaintenance;
+  await cleanupMaintenance;
   await pool?.end();
 });
 const port = Number(process.env.PORT ?? "3000");