From bb5a57f7cb8db6b40e732af3f459124fea3a565c Mon Sep 17 00:00:00 2001 From: caigee-cmd <210803857+caigee-cmd@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:19:44 +0800 Subject: [PATCH] =?UTF-8?q?docs(runtime):=20=E4=BF=AE=E6=AD=A3=20liveness?= =?UTF-8?q?=20watchdog=20=E6=B3=A8=E9=87=8A=EF=BC=8C=E6=8F=8F=E8=BF=B0=20r?= =?UTF-8?q?estartStuckWorker=20=E7=9A=84=E5=AE=9E=E9=99=85=E8=A1=8C?= =?UTF-8?q?=E4=B8=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/runtime/manager.go | 5 +++-- internal/runtime/manager_maintenance.go | 9 +++++---- internal/runtime/probe_worker_liveness_test.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/runtime/manager.go b/internal/runtime/manager.go index cb5bb69..5f0ee99 100644 --- a/internal/runtime/manager.go +++ b/internal/runtime/manager.go @@ -131,8 +131,9 @@ type Manager struct { // healthFailures counts consecutive liveness probe failures per worker // account. The maintenance loop's workerLiveness watchdog increments it on // each failed /health probe and clears it on success; reaching - // workerLivenessThreshold triggers a stopAccount so watchAccount + - // beginAccountRecovery reboot the stuck worker. Guarded by mu. + // workerLivenessThreshold triggers restartStuckWorker, which removes the + // process from the table, stops it, then calls beginAccountRecovery to + // reboot the stuck worker. Guarded by mu. healthFailures map[string]int } diff --git a/internal/runtime/manager_maintenance.go b/internal/runtime/manager_maintenance.go index 00f650f..2dc9e48 100644 --- a/internal/runtime/manager_maintenance.go +++ b/internal/runtime/manager_maintenance.go @@ -500,8 +500,9 @@ const workerLivenessThreshold = 3 // that goes "alive but stuck" after boot — /health no longer responds, or its // goroutine is wedged — would stay routed to forever without this loop. Each // 30s tick issues one /health round-trip per worker account; on the -// threshold-th consecutive failure it calls stopAccount so the existing -// watchAccount + beginAccountRecovery path reboots the worker with backoff. +// threshold-th consecutive failure it calls restartStuckWorker, which removes +// the process from the table, stops it, then calls beginAccountRecovery so the +// worker is rebooted with backoff. // // In-process accounts are skipped by probeHealthOnly and never enter the // failure counter, so WorkBuddy / Trae / Devin are unaffected. @@ -517,8 +518,8 @@ func (manager *Manager) probeWorkerLiveness(ctx context.Context) { manager.mu.Unlock() continue } - // Skip accounts already in recovery: stopAccount removed them from - // processes and beginAccountRecovery owns the reboot. Touching them + // Skip accounts already in recovery: they are no longer in processes + // and beginAccountRecovery owns the reboot. Touching them // here would double-restart or race the recovering guard. manager.mu.Lock() if manager.recovering[item.ID] { diff --git a/internal/runtime/probe_worker_liveness_test.go b/internal/runtime/probe_worker_liveness_test.go index 8569493..ec2f8da 100644 --- a/internal/runtime/probe_worker_liveness_test.go +++ b/internal/runtime/probe_worker_liveness_test.go @@ -86,7 +86,7 @@ func TestProbeWorkerLivenessRestartsStuckWorker(t *testing.T) { t.Fatal("starter was not invoked") } - // Threshold consecutive failures must trigger stopAccount. The account's + // Threshold consecutive failures must trigger restartStuckWorker. The account's // /health URL is not serving, so every probe fails on transport. for i := 1; i <= 3; i++ { manager.probeWorkerLiveness(ctx)