feat(coder): v2.6 Phase 3 — lifecycle hardening (idle evict, crash recovery, worktree reaper)

Idle TTL eviction per (chat,agent) + LRU cap (never a busy backend); pure lifecycle-decisions.ts (TDD). Crash recovery lifts openchamber's health-monitor + busy-aware-restart + stale-grace state machine into opencode-server.ts (+ port reclaim) and warm-acp.ts; opencode crash -> fresh sessions, ACP -> re-session/new. F.1 turn-guard + U.6 usage preserved (their tests pass). Orphan worktree reaper (1h grace, superset-style dirty/unpushed preflight, Paseo soft-delete) + close hooks + diff re-baseline after apply_pending. 35 new tests + DB-opt-in reconnect test; 215 coder tests pass; tsc + build clean. Completes v2.6. Follow-ups out of scope: apps/server close-hook caller, 3.7 DiffPanel staging hint, live smokes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/coder/src/services/backends/opencode-server.ts

@@ -21,9 +21,9 @@
  *   - promptAsync is fire-and-forget (204); the turn completes via a
  *     'session.idle' event for that opencode session id.
  */
-import { spawn, type ChildProcess } from 'node:child_process';
+import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { createServer } from 'node:net';
+import { createServer, connect as netConnect } from 'node:net';
 import type { FastifyBaseLogger } from 'fastify';
 import {
   createOpencodeClient,
@@ -39,6 +39,7 @@ import type { Sql } from '../../db.js';
 import type { AcpToolSnapshot } from '../acp-tool-snapshot.js';
 import { armAbortGuard, noteTurnActivity, consumeTerminal } from './turn-guard.js';
 import { stepEndedToUsage, type StepUsage } from './opencode-usage.js';
+import { decideRestart, DEFAULT_HEALTH_FAILURE_THRESHOLD } from './lifecycle-decisions.js';
 import type {
   AgentBackend,
   AgentEvent,
@@ -104,6 +105,11 @@ export class OpenCodeServerBackend implements AgentBackend {
   private port: number | null = null;
   private up = false;
   private serverStarting: Promise<void> | null = null;
+  // Phase 3 busy-aware health monitor (openchamber lift): consecutive failed
+  // probes + the start of an unhealthy-while-busy window feed `decideRestart`.
+  private consecutiveHealthFailures = 0;
+  private unhealthyBusySince = 0;
+  private restarting: Promise<void> | null = null;
 
   /** opencode session id → demux state. Maintained by ensureSession; read by the SSE loop. */
   private readonly byOpencodeId = new Map<string, SessionState>();
@@ -119,11 +125,30 @@ export class OpenCodeServerBackend implements AgentBackend {
     return this.up ? 'up' : 'down';
   }
 
-  // ─── Server lifecycle (1.2: spawn once + client + ready) ─────────────────────
+  /** Phase 3: busy iff ANY pooled opencode session has an in-flight turn. The
+   *  pool reads this to skip idle/LRU eviction and the health-monitor to defer a
+   *  restart (never tear down a session mid-stream). */
+  isBusy(): boolean {
+    for (const st of this.byOpencodeId.values()) {
+      if (st.activeTurn) return true;
+    }
+    return false;
+  }
 
-  /** Lazy: start the single server on first use. Idempotent — one server per backend. */
+  // ─── Server lifecycle (1.2: spawn once + client + ready; Phase 3 crash-restart) ──
+
+  /**
+   * Lazy: start the single server on first use; re-spawn after a crash. Idempotent
+   * within one live server — `serverStarting` caches the in-flight start, and is
+   * reset to null by the crash handler so the NEXT ensureServer re-spawns a fresh
+   * server (Phase 3 crash recovery). A dead-but-not-yet-reaped child (exit handler
+   * raced) is also treated as needing a restart.
+   */
   private ensureServer(): Promise<void> {
-    if (!this.serverStarting) this.serverStarting = this.startServer();
+    const childDead = this.child != null && (this.child.exitCode !== null || this.child.signalCode !== null);
+    if (!this.serverStarting || (!this.up && childDead)) {
+      this.serverStarting = this.startServer();
+    }
     return this.serverStarting;
   }
 
@@ -143,11 +168,15 @@ export class OpenCodeServerBackend implements AgentBackend {
     this.port = port;
 
     // Child lifetime is the backend's (the pool's), NOT a request's. We never tie
-    // it to a per-turn abort signal. On unexpected exit we mark down + log; crash
-    // recovery is Phase 3.
+    // it to a per-turn abort signal. Phase 3: on unexpected exit we recover —
+    // settle any in-flight turns as failed, mark their agent_sessions rows crashed,
+    // and reset `serverStarting` so the next ensureServer re-spawns. opencode keeps
+    // sessions on disk, but a fresh server's in-memory state is gone, so the next
+    // turn's ensureSession (rows now 'crashed') creates fresh opencode sessions.
     child.on('exit', (code, signal) => {
-      this.up = false;
-      this.log.warn({ code, signal, port }, 'opencode-server: child exited (recovery is Phase 3)');
+      // Only react to THIS child's exit (a restart may have swapped in a new one).
+      if (this.child !== child) return;
+      this.handleServerCrash(code, signal, port);
     });
 
     await waitForReady(child, READY_TIMEOUT_MS);
@@ -157,6 +186,136 @@ export class OpenCodeServerBackend implements AgentBackend {
     this.log.info({ port }, 'opencode-server: ready');
   }
 
+  /**
+   * Crash handler (Phase 3, lift of openchamber's restart-on-exit path). The
+   * server died with N live opencode sessions; we can't restart it here (the next
+   * turn does, lazily — avoids a restart storm if the binary is broken). We:
+   *   1. fail every in-flight turn so its dispatcher unblocks + publishes an error,
+   *   2. mark each session's agent_sessions row 'crashed' so ensureSession won't
+   *      resume a now-dead native session id (it creates fresh),
+   *   3. tear down the SSE loops + demux state (stale against the dead server),
+   *   4. reclaim the port + reset state so the next ensureServer re-spawns.
+   */
+  private handleServerCrash(code: number | null, signal: NodeJS.Signals | null, port: number): void {
+    this.up = false;
+    const states = [...this.byOpencodeId.values()];
+    this.log.warn(
+      { code, signal, port, liveSessions: states.length },
+      'opencode-server: child exited — recovering (fail in-flight, mark crashed, re-spawn next turn)',
+    );
+
+    const crashedIds: string[] = [];
+    for (const st of states) {
+      st.sseAbort?.abort();
+      if (st.activeTurn) {
+        st.activeTurn.settle({ ok: false, error: 'opencode server crashed mid-turn' });
+        st.activeTurn = null;
+      }
+      if (st.watchdog) {
+        clearTimeout(st.watchdog);
+        st.watchdog = null;
+      }
+      crashedIds.push(st.agentSessionId);
+    }
+    // Drop the demux map: every session id is stale against a fresh server.
+    this.byOpencodeId.clear();
+    this.client = null;
+    this.serverStarting = null; // force a re-spawn on the next ensureServer
+
+    if (crashedIds.length > 0) {
+      this.sql`
+        UPDATE agent_sessions SET status = 'crashed'
+        WHERE agent_session_id = ANY(${crashedIds}) AND status <> 'closed'
+      `.catch((err) => {
+        this.log.warn({ err: errMsg(err) }, 'opencode-server: failed to mark crashed sessions (non-fatal)');
+      });
+    }
+
+    // Reclaim the port so a re-spawn on a fixed/leaked port isn't blocked. Best
+    // effort; the next start uses a fresh ephemeral port anyway.
+    reclaimPort(port);
+  }
+
+  /**
+   * Phase 3 proactive health monitor (openchamber `runHealthCheckCycle` lift,
+   * busy-aware). Probes the server's /global/health; on a sustained failure of a
+   * NON-busy server, force a restart so the next turn isn't blocked by a wedged
+   * (hung-but-not-exited) process. Busy servers are deferred via the stale-grace in
+   * `decideRestart` — never tear down live work. Driven by the pool's periodic
+   * sweep (best-effort; a crash-exit is already handled by `handleServerCrash` +
+   * lazy `ensureServer` re-spawn, so this only catches the hung case). No-op when
+   * the server was never started or a restart is already in flight.
+   */
+  async tickHealth(now: number = Date.now()): Promise<void> {
+    if (!this.child || this.restarting) return;
+    const childExited = this.child.exitCode !== null || this.child.signalCode !== null;
+    // An exited child is recovered lazily by ensureServer; don't double-restart it.
+    if (childExited) return;
+
+    const healthy = await this.probeHealth();
+    if (healthy) {
+      this.consecutiveHealthFailures = 0;
+      this.unhealthyBusySince = 0;
+      return;
+    }
+    this.consecutiveHealthFailures += 1;
+    const busy = this.isBusy();
+    const decision = decideRestart({
+      processExited: false,
+      consecutiveFailures: this.consecutiveHealthFailures,
+      busy,
+      unhealthyBusySince: this.unhealthyBusySince,
+      now,
+      failureThreshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
+    });
+    // Stamp the start of an unhealthy-while-busy window so the stale-grace can fire.
+    if (busy && this.unhealthyBusySince === 0) this.unhealthyBusySince = now;
+    if (decision.action === 'restart') {
+      this.log.warn(
+        { failures: this.consecutiveHealthFailures, busy, reason: decision.reason },
+        'opencode-server: health monitor forcing restart',
+      );
+      this.consecutiveHealthFailures = 0;
+      this.unhealthyBusySince = 0;
+      await this.restartServer();
+    }
+  }
+
+  private async probeHealth(): Promise<boolean> {
+    if (!this.client) return false;
+    try {
+      const res = await this.client.global.health();
+      return !res.error;
+    } catch {
+      return false;
+    }
+  }
+
+  /** Force-kill the current server + reclaim its port; the next ensureServer
+   *  re-spawns (lazy). Mirrors handleServerCrash's state reset but is initiated by
+   *  the health monitor rather than the OS. */
+  private async restartServer(): Promise<void> {
+    if (this.restarting) return this.restarting;
+    this.restarting = (async () => {
+      const child = this.child;
+      const port = this.port;
+      this.up = false;
+      // Fail in-flight turns + mark sessions crashed via the same path as a crash.
+      if (child) {
+        this.handleServerCrash(null, null, port ?? 0);
+        if (!child.killed) child.kill('SIGTERM');
+      }
+      if (port) {
+        reclaimPort(port);
+        await waitForPortRelease(port, 3_000);
+      }
+      this.child = null;
+    })().finally(() => {
+      this.restarting = null;
+    });
+    return this.restarting;
+  }
+
   // ─── SSE read loop + demux + translate (1.3) + dedup (1.4) ───────────────────
 
   /** Per-session SSE subscription, scoped to the session's worktree directory.
@@ -756,6 +915,67 @@ function mapToolStatus(s: ToolState['status'] | undefined): ToolCallStatus | nul
   }
 }
 
+/**
+ * Reclaim a loopback port a dead opencode child may still hold (lift of
+ * openchamber `killProcessOnPort`). Best-effort, POSIX-only (`lsof`/`kill`); a
+ * failure is harmless because the next spawn allocates a fresh ephemeral port.
+ * Never kills this process. Synchronous + short-timeout so the crash handler
+ * doesn't block.
+ */
+function reclaimPort(port: number | null): void {
+  if (!port || process.platform === 'win32') return;
+  try {
+    const res = spawnSync('lsof', ['-ti', `:${port}`], { encoding: 'utf8', timeout: 3_000, windowsHide: true });
+    const out = res.stdout || '';
+    const myPid = process.pid;
+    for (const pidStr of out.split(/\s+/)) {
+      const pid = parseInt(pidStr.trim(), 10);
+      if (pid && pid !== myPid) {
+        try {
+          spawnSync('kill', ['-9', String(pid)], { stdio: 'ignore', timeout: 2_000 });
+        } catch {
+          // ignore — best effort
+        }
+      }
+    }
+  } catch {
+    // lsof absent or failed — the fresh-ephemeral-port spawn doesn't need this.
+  }
+}
+
+/**
+ * Resolve true once nothing is listening on `port` (lift of openchamber
+ * `waitForPortRelease`). Used before re-spawning on a fixed port; with ephemeral
+ * ports it's a fast no-op. Probes 127.0.0.1; resolves false at the deadline.
+ */
+function waitForPortRelease(port: number, timeoutMs: number): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  return new Promise((resolve) => {
+    const attempt = () => {
+      const socket = netConnect({ port, host: '127.0.0.1' });
+      let settled = false;
+      const finish = (released: boolean) => {
+        if (settled) return;
+        settled = true;
+        socket.removeAllListeners();
+        socket.destroy();
+        if (released || Date.now() >= deadline) {
+          resolve(released);
+          return;
+        }
+        setTimeout(attempt, 150);
+      };
+      socket.once('connect', () => finish(false));
+      socket.once('error', (err: NodeJS.ErrnoException) => {
+        if (err && (err.code === 'ECONNREFUSED' || err.code === 'EHOSTUNREACH')) finish(true);
+        else finish(false);
+      });
+      socket.setTimeout(500, () => finish(true));
+    };
+    attempt();
+  });
+}
+
 /** Bind-probe an ephemeral port on loopback. */
 function freePort(): Promise<number> {
   return new Promise((resolve, reject) => {