fix(tunnel): break the expired-leaf renewal deadlock

`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.

agent/test/dial.test.ts

@@ -6,7 +6,9 @@ import type { AgentConfig } from '../src/config/agentConfig.js'
 import { generateIdentity } from '../src/keys/identity.js'
 import { openKeystore } from '../src/keys/keystore.js'
 import {
+  CertExpiredBeyondGraceError,
   CertExpiredError,
+  DEFAULT_EXPIRED_RENEW_GRACE_MS,
   NotEnrolledError,
   buildTlsOptions,
   dialRelay,
@@ -65,6 +67,82 @@ describe('buildTlsOptions (T12, INV14/INV4)', () => {
   })
 })
 
+/**
+ * Expired-leaf RENEWAL grace. The tunnel dial stays fail-closed forever; only the /renew recovery
+ * path may opt in, and only inside a bounded window — otherwise an expired leaf deadlocks the agent
+ * (mTLS renew needs a cert, and the only cert has expired) and the host is bricked until a re-pair.
+ */
+describe('buildTlsOptions expired-renewal grace (deadlock recovery)', () => {
+  const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
+  const DAY = 86_400_000
+
+  it('stays fail-closed by DEFAULT — no grace opt-in ⇒ CertExpiredError', () => {
+    const { dir, ks } = enrolledKs()
+    expect(() => buildTlsOptions(ks, { certParser: expiredBy(DAY) })).toThrow(CertExpiredError)
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('expired INSIDE the grace window → returns options so /renew can authenticate', () => {
+    const { dir, ks } = enrolledKs()
+    const tls = buildTlsOptions(ks, {
+      certParser: expiredBy(8 * DAY),
+      expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
+    })
+    expect(tls.cert).toBe('CERTPEM')
+    expect(tls.rejectUnauthorized).toBe(true)
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('expired BEYOND the grace window → CertExpiredBeyondGraceError (re-pair required)', () => {
+    const { dir, ks } = enrolledKs()
+    expect(() =>
+      buildTlsOptions(ks, {
+        certParser: expiredBy(31 * DAY),
+        expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
+      }),
+    ).toThrow(CertExpiredBeyondGraceError)
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('grace of 0 is not a bypass — an expired leaf still throws', () => {
+    const { dir, ks } = enrolledKs()
+    expect(() => buildTlsOptions(ks, { certParser: expiredBy(1000), expiredGraceMs: 0 })).toThrow(
+      CertExpiredError,
+    )
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('a still-valid cert is unaffected by the grace option', () => {
+    const { dir, ks } = enrolledKs()
+    const tls = buildTlsOptions(ks, {
+      certParser: future,
+      expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
+    })
+    expect(tls.cert).toBe('CERTPEM')
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('the TUNNEL dial never gets grace — an expired leaf still refuses to dial', () => {
+    const { dir, ks } = enrolledKs()
+    let constructed = 0
+    class NeverCtor implements RawTlsWs {
+      constructor() {
+        constructed += 1
+      }
+      send(): void {}
+      close(): void {}
+      on(): void {}
+      once(): void {}
+    }
+    // `dialRelay` has no grace seam at all: it fails fast (synchronously) before any socket exists.
+    expect(() =>
+      dialRelay(CFG, ks, { Ctor: NeverCtor as unknown as TlsWsConstructor, certParser: past }),
+    ).toThrow(CertExpiredError)
+    expect(constructed).toBe(0)
+    rmSync(dir, { recursive: true, force: true })
+  })
+})
+
 describe('dialRelay (T12)', () => {
   it('constructs the wss client with the TLS options and resolves on open', async () => {
     const { dir, ks } = enrolledKs()

agent/test/nativeRenew.test.ts

@@ -27,6 +27,10 @@ import {
   type MtlsRequest,
 } from '../src/certs/nativeRenew.js'
 import { FakeTimer } from './fixtures/fakes.js'
+import {
+  CertExpiredBeyondGraceError,
+  DEFAULT_EXPIRED_RENEW_GRACE_MS,
+} from '../src/transport/dial.js'
 
 const CFG: AgentConfig = {
   relayUrl: 'wss://relay/agent',
@@ -109,11 +113,21 @@ describe('createMtlsFetch (A5)', () => {
 describe('wireAutoRenew (A5)', () => {
   function fakeRotator(): {
     rotator: CertRotator
-    fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void }
+    fire: {
+      rotated?: () => void
+      revoked?: () => void
+      error?: (e: unknown) => void
+      exhausted?: (e: CertExpiredBeyondGraceError) => void
+    }
     start: ReturnType<typeof vi.fn>
     stop: ReturnType<typeof vi.fn>
   } {
-    const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {}
+    const fire: {
+      rotated?: () => void
+      revoked?: () => void
+      error?: (e: unknown) => void
+      exhausted?: (e: CertExpiredBeyondGraceError) => void
+    } = {}
     const start = vi.fn()
     const stop = vi.fn()
     const rotator: CertRotator = {
@@ -128,6 +142,9 @@ describe('wireAutoRenew (A5)', () => {
       onError: (cb) => {
         fire.error = cb
       },
+      onExhausted: (cb) => {
+        fire.exhausted = cb
+      },
     }
     return { rotator, fire, start, stop }
   }
@@ -277,3 +294,72 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => {
     rmSync(dir, { recursive: true, force: true })
   })
 })
+
+/**
+ * Expired-leaf recovery. `createMtlsFetch` is the ONE place that decides whether a lapsed leaf may
+ * still be presented; if it keeps refusing (the pre-fix behaviour) the renewal can never leave the
+ * host and the tunnel stays dead until a manual re-pair.
+ */
+describe('createMtlsFetch expired-leaf recovery', () => {
+  const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
+  const DAY = 86_400_000
+
+  it('still presents a leaf that expired INSIDE the grace window (renewal can go out)', async () => {
+    const { dir, ks } = enrolledKs()
+    let presented = ''
+    const request: MtlsRequest = async (_url, tls) => {
+      presented = tls.cert
+      return { status: 201, body: '{}' }
+    }
+    const f = createMtlsFetch(ks, { request, certParser: expiredBy(8 * DAY) })
+    const res = await f('https://recover.example.com/renew', { method: 'POST', body: '{}' })
+    expect(res.status).toBe(201)
+    expect(presented).toBe('LEAFCERT')
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('refuses once the leaf is BEYOND the grace window (terminal — re-pair required)', async () => {
+    const { dir, ks } = enrolledKs()
+    const request: MtlsRequest = async () => ({ status: 201, body: '{}' })
+    const f = createMtlsFetch(ks, {
+      request,
+      certParser: expiredBy(DEFAULT_EXPIRED_RENEW_GRACE_MS + DAY),
+    })
+    await expect(f('https://recover.example.com/renew', { method: 'POST' })).rejects.toThrow(
+      CertExpiredBeyondGraceError,
+    )
+    rmSync(dir, { recursive: true, force: true })
+  })
+})
+
+describe('wireAutoRenew exhausted routing', () => {
+  it('logs an actionable re-pair alarm and leaves the supervisor running', () => {
+    const fire: { exhausted?: (e: CertExpiredBeyondGraceError) => void } = {}
+    const rotator: CertRotator = {
+      start: vi.fn(),
+      stop: vi.fn(),
+      onRotated: () => {},
+      onRevoked: () => {},
+      onError: () => {},
+      onExhausted: (cb) => {
+        fire.exhausted = cb
+      },
+    }
+    const restartChild = vi.fn()
+    const stop = vi.fn()
+    const lines: string[] = []
+    wireAutoRenew(rotator, { restartChild, stop }, createLogger('info', (l) => lines.push(l)), {
+      subdomain: 'h7fd8',
+      hostId: 'h-1',
+    })
+    fire.exhausted?.(new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000))
+
+    const alarm = lines.find((l) => /re-pair/i.test(l))
+    expect(alarm).toBeDefined()
+    expect(alarm).toContain('h7fd8')
+    // The supervisor keeps running: a later `pair` writes fresh cert files that the restarting
+    // frpc child picks up. Tearing down here would make recovery need a manual restart too.
+    expect(stop).not.toHaveBeenCalled()
+    expect(restartChild).not.toHaveBeenCalled()
+  })
+})

agent/test/rotation.test.ts

@@ -8,9 +8,11 @@ import { openKeystore } from '../src/keys/keystore.js'
 import {
   computeRenewDelayMs,
   createCertRotator,
+  recoveryRenewalUrlFor,
   renewCert,
   renewalUrlFor,
 } from '../src/certs/rotation.js'
+import { CertExpiredBeyondGraceError } from '../src/transport/dial.js'
 import { createBackoff } from '../src/transport/backoff.js'
 import { FakeTimer } from './fixtures/fakes.js'
 
@@ -159,3 +161,121 @@ describe('createCertRotator (T13)', () => {
     rmSync(dir, { recursive: true, force: true })
   })
 })
+
+/**
+ * Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf
+ * it renews, so a lapsed leaf can never renew itself. Recovery needs BOTH a different endpoint (the
+ * strict nginx vhost rejects an expired client cert with a bare 400 before any location runs) and a
+ * terminal signal when even the grace window is spent, so the agent stops retrying forever.
+ */
+describe('expired-leaf recovery routing', () => {
+  it('derives the recovery endpoint from an `enroll.` host', () => {
+    expect(recoveryRenewalUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
+      'https://recover.terminal.example.com/renew',
+    )
+  })
+
+  it('honours an explicit recoverUrl over the derivation', () => {
+    expect(
+      recoveryRenewalUrlFor({
+        ...CFG,
+        enrollUrl: 'https://enroll.terminal.example.com/enroll',
+        recoverUrl: 'https://elsewhere.example.com/renew',
+      }),
+    ).toBe('https://elsewhere.example.com/renew')
+  })
+
+  it('is null when the enroll host carries no `enroll.` label (nothing to derive)', () => {
+    expect(recoveryRenewalUrlFor(CFG)).toBeNull()
+  })
+
+  it('posts to the NORMAL endpoint while the leaf is still valid', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    const urls: string[] = []
+    const rotator = createCertRotator(
+      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
+      ks.loadIdentity()!,
+      ks,
+      {
+        timer,
+        renewBeforeMs: 1000,
+        fetchImpl: (async (u: string) => {
+          urls.push(u)
+          return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+        }) as unknown as typeof fetch,
+        now: () => new Date(0),
+        parseCert: () => new Date(2000), // still valid at now=0
+      },
+    )
+    rotator.start()
+    timer.advance(1000)
+    await flush()
+    expect(urls).toEqual(['https://enroll.terminal.example.com/renew'])
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('posts to the RECOVERY endpoint once the leaf has already expired', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    const urls: string[] = []
+    const rotator = createCertRotator(
+      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
+      ks.loadIdentity()!,
+      ks,
+      {
+        timer,
+        renewBeforeMs: 1000,
+        fetchImpl: (async (u: string) => {
+          urls.push(u)
+          return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+        }) as unknown as typeof fetch,
+        now: () => new Date(10_000),
+        parseCert: () => new Date(2000), // expired 8s ago ⇒ delay clamps to 0
+      },
+    )
+    rotator.start()
+    timer.advance(0)
+    await flush()
+    expect(urls).toEqual(['https://recover.terminal.example.com/renew'])
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('fires onExhausted and STOPS retrying when the grace window is spent', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    let attempts = 0
+    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
+      timer,
+      renewBeforeMs: 1000,
+      retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
+      fetchImpl: (async () => {
+        attempts += 1
+        throw new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000)
+      }) as unknown as typeof fetch,
+      now: () => new Date(0),
+      parseCert: () => new Date(2000),
+    })
+    const errors: unknown[] = []
+    let exhausted: CertExpiredBeyondGraceError | null = null
+    rotator.onError((e) => errors.push(e))
+    rotator.onExhausted((e) => {
+      exhausted = e
+    })
+    rotator.start()
+    timer.advance(1000)
+    await flush()
+
+    expect(attempts).toBe(1)
+    expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError)
+    // Terminal: no retry is armed, because only a re-pair can help — retrying forever is the log
+    // spam that buried the real signal in production (6380 identical warnings).
+    expect(errors).toHaveLength(0)
+    timer.advance(60_000)
+    await flush()
+    expect(attempts).toBe(1)
+    rmSync(dir, { recursive: true, force: true })
+  })
+})