fix(runner): retire already reconciled recovery runners

landing-zone-configurator/app/apps/api/db/077_reconciled_runner_retirement.sql

@@ -0,0 +1,56 @@
+-- Reconciled interruptions have an immutable clearance too. Their original
+-- warm app may still be desired STARTED with a CRASHED process; do not leak it.
+-- Unknown recovery, active recovery and unexpired execution authority remain protected.
+CREATE OR REPLACE FUNCTION lzc_auth.retire_completed_application_runners(p_space uuid,p_origin text)
+RETURNS integer LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$
+DECLARE retired integer;
+BEGIN
+ UPDATE lzc.runner_pool_slots slot SET status='retiring',assignment=NULL,token_hash=NULL
+ FROM lzc.application_dispatches dispatched
+ WHERE slot.space_id=p_space AND slot.origin=p_origin AND slot.status='consumed'
+  AND NOT slot.is_probe AND slot.job_id=dispatched.job_id
+  AND slot.app_id=dispatched.runner_app_id AND slot.source_droplet_id=dispatched.runner_package_id
+  AND (dispatched.status IN ('succeeded','failed') OR (
+   dispatched.status='reconciliation_required'
+   AND lzc_auth.application_recovery_replan_valid(dispatched.job_id)
+   AND NOT EXISTS(SELECT 1 FROM lzc.application_runner_tickets t WHERE t.job_id=dispatched.job_id AND t.expires_at>now())
+   AND NOT EXISTS(SELECT 1 FROM lzc.application_state_recoveries r WHERE r.job_id=dispatched.job_id AND r.phase IN ('requested','assessing','prepared','committing','reconciliation-required'))
+  ))
+  AND dispatched.finished_at IS NOT NULL;
+ GET DIAGNOSTICS retired=ROW_COUNT;
+ RETURN retired;
+END $$;
+REVOKE ALL ON FUNCTION lzc_auth.retire_completed_application_runners(uuid,text) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION lzc_auth.retire_completed_application_runners(uuid,text) TO configurator_app;
+
+-- Reuse the same established clearance validator for the stopped-app sweep.
+CREATE OR REPLACE FUNCTION lzc_auth.runner_cleanup_disposition(p_runner uuid,p_app uuid,p_kind text)
+RETURNS text LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$
+DECLARE known boolean:=false;
+BEGIN
+ IF p_kind NOT IN ('plan','warm') THEN RETURN 'protected'; END IF;
+ IF EXISTS(SELECT 1 FROM lzc.application_jobs j WHERE j.id=p_runner AND NOT EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE d.job_id=j.id AND d.runner_app_id=p_app))
+  OR EXISTS(SELECT 1 FROM lzc.application_state_recovery_tasks WHERE id=p_runner OR runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.runner_pool_slots WHERE (id=p_runner OR app_id=p_app) AND
+   (app_id IS DISTINCT FROM p_app OR (p_kind='warm' AND id<>p_runner) OR expires_at>now() OR status IN ('warming','ready','reserved','assigned')))
+  OR EXISTS(SELECT 1 FROM lzc.plan_runs WHERE (id=p_runner OR runner_app_id=p_app) AND
+   (runner_app_id IS DISTINCT FROM p_app OR (p_kind='plan' AND id<>p_runner) OR expires_at>now() OR finished_at IS NULL OR finished_at>now()-interval '30 seconds' OR status NOT IN ('succeeded','failed','cancelled')))
+  OR EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE (job_id=p_runner OR runner_app_id=p_app) AND
+   (runner_app_id IS DISTINCT FROM p_app OR (p_kind='plan' AND job_id<>p_runner) OR finished_at IS NULL OR finished_at>now()-interval '30 seconds'
+    OR NOT (status IN ('succeeded','failed') OR (status='reconciliation_required' AND lzc_auth.application_recovery_replan_valid(d.job_id)))
+    OR EXISTS(SELECT 1 FROM lzc.application_runner_tickets t WHERE t.job_id=d.job_id AND t.expires_at>now())
+    OR EXISTS(SELECT 1 FROM lzc.application_state_recoveries r WHERE r.job_id=d.job_id AND r.phase IN ('requested','assessing','prepared','committing','reconciliation-required'))))
+ THEN RETURN 'protected'; END IF;
+ -- Consumed warm slots without a completed associated job are never mere orphans.
+ IF EXISTS(SELECT 1 FROM lzc.runner_pool_slots s WHERE s.app_id=p_app AND s.status='consumed'
+  AND NOT EXISTS(SELECT 1 FROM lzc.plan_runs p WHERE p.id=s.job_id AND p.runner_app_id=p_app)
+  AND NOT EXISTS(SELECT 1 FROM lzc.application_dispatches d WHERE d.job_id=s.job_id AND d.runner_app_id=p_app))
+ THEN RETURN 'protected'; END IF;
+ SELECT EXISTS(SELECT 1 FROM lzc.plan_runs WHERE runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.application_dispatches WHERE runner_app_id=p_app)
+  OR EXISTS(SELECT 1 FROM lzc.runner_pool_slots WHERE app_id=p_app AND id=p_runner AND p_kind='warm') INTO known;
+ RETURN CASE WHEN known THEN 'completed' ELSE 'orphan' END;
+END $$;
+REVOKE ALL ON FUNCTION lzc_auth.runner_cleanup_disposition(uuid,uuid,text) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION lzc_auth.runner_cleanup_disposition(uuid,uuid,text) TO configurator_app;
+

landing-zone-configurator/app/integration/identity.test.ts

@@ -8355,6 +8355,56 @@ describe("real PostgreSQL session and tenant boundaries", () => {
             expect(await approved(owner)).toBe(true);
             expect(await approved(engineer)).toBe(true);
             expect(await approved(bob)).toBe(false);
+            // Seed expired execution authority in an isolated fixture transaction.
+            // Rollback retains the real recovery fixture for the assertions below.
+            const retiredSlot = randomUUID(),
+              retiredSpace = randomUUID();
+            const retiredOrigin = "https://retention.example";
+            await admin.query(
+              "BEGIN; SET LOCAL session_replication_role=replica",
+            );
+            try {
+              await admin.query(
+                "UPDATE lzc.application_runner_tickets SET issued_at=now()-interval '2 hours',expires_at=now()-interval '1 hour' WHERE job_id=$1",
+                [partialApply.jobId],
+              );
+              await admin.query(
+                "UPDATE lzc.application_job_grants SET expires_at=now()-interval '1 hour' WHERE job_id=$1",
+                [partialApply.jobId],
+              );
+              await admin.query(
+                "INSERT INTO lzc.runner_pool_slots(id,pool_key,origin,space_id,source_droplet_id,app_id,status,job_id,expires_at) SELECT $1,'reconciled-retention',$2,$3,runner_package_id,runner_app_id,'consumed',job_id,now()-interval '1 hour' FROM lzc.application_dispatches WHERE job_id=$4",
+                [retiredSlot, retiredOrigin, retiredSpace, partialApply.jobId],
+              );
+              const retire = async () =>
+                (
+                  await admin.query(
+                    "SELECT lzc_auth.retire_completed_application_runners($1,$2) AS retired",
+                    [retiredSpace, retiredOrigin],
+                  )
+                ).rows[0].retired;
+              expect(await retire()).toBe(1);
+              await admin.query(
+                "UPDATE lzc.runner_pool_slots SET status='consumed' WHERE id=$1",
+                [retiredSlot],
+              );
+              await admin.query(
+                "UPDATE lzc.application_runner_tickets SET expires_at=now()+interval '1 hour' WHERE job_id=$1",
+                [partialApply.jobId],
+              );
+              expect(await retire()).toBe(0);
+              await admin.query(
+                "UPDATE lzc.application_runner_tickets SET issued_at=now()-interval '2 hours',expires_at=now()-interval '1 hour' WHERE job_id=$1",
+                [partialApply.jobId],
+              );
+              await admin.query(
+                "DELETE FROM lzc.application_recovery_replans WHERE job_id=$1",
+                [partialApply.jobId],
+              );
+              expect(await retire()).toBe(0);
+            } finally {
+              await admin.query("ROLLBACK");
+            }
             await expect(
               clear(reviewer, { sha: "b".repeat(64) }),
             ).rejects.toMatchObject({

landing-zone-configurator/docs/plan-execution.de.md

@@ -991,3 +991,8 @@ von diesem Lauf nicht gelöscht. Fehler erscheinen ohne Zugangsdaten als
 
 Die Log-Zusammenfassung `runner_retention` enthält nur Anzahlen geprüfter, gelöschter,
 geschützter und nicht zuordenbarer Apps. So werden ungeklärte Altbestände sichtbar.
+
+Auch abgeglichene Warm-Runner-Abbrüche werden im normalen Pool ausgemustert, selbst
+wenn CF nach Prozessende weiter STARTED anfordert. Erforderlich sind der bestehende
+gültige Recovery-Neuplan-Nachweis, abgelaufene Runner-Tickets und keine aktive geführte
+Recovery. Ein durch späteren separaten Recovery-State ungültiger Nachweis schützt die App erneut.

landing-zone-configurator/docs/plan-execution.md

@@ -475,3 +475,8 @@ receipt. The template and recovery apps are excluded. Failures log the non-secre
 
 The `runner_retention` log summary contains only scanned, deleted, protected and
 unassociated counts, allowing operators to detect unresolved legacy accumulation.
+
+Reconciled warm-runner interruptions also retire through the normal pool, even if CF
+still desires STARTED after process exit. Retirement requires the existing valid
+recovery-replan clearance, expired runner tickets and no active guided recovery.
+A clearance invalidated by a later separate recovery State protects the app again.