fix(tunnel): break the expired-leaf renewal deadlock

`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.

agent/test/rotation.test.ts

@@ -8,9 +8,11 @@ import { openKeystore } from '../src/keys/keystore.js'
 import {
   computeRenewDelayMs,
   createCertRotator,
+  recoveryRenewalUrlFor,
   renewCert,
   renewalUrlFor,
 } from '../src/certs/rotation.js'
+import { CertExpiredBeyondGraceError } from '../src/transport/dial.js'
 import { createBackoff } from '../src/transport/backoff.js'
 import { FakeTimer } from './fixtures/fakes.js'
 
@@ -159,3 +161,121 @@ describe('createCertRotator (T13)', () => {
     rmSync(dir, { recursive: true, force: true })
   })
 })
+
+/**
+ * Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf
+ * it renews, so a lapsed leaf can never renew itself. Recovery needs BOTH a different endpoint (the
+ * strict nginx vhost rejects an expired client cert with a bare 400 before any location runs) and a
+ * terminal signal when even the grace window is spent, so the agent stops retrying forever.
+ */
+describe('expired-leaf recovery routing', () => {
+  it('derives the recovery endpoint from an `enroll.` host', () => {
+    expect(recoveryRenewalUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
+      'https://recover.terminal.example.com/renew',
+    )
+  })
+
+  it('honours an explicit recoverUrl over the derivation', () => {
+    expect(
+      recoveryRenewalUrlFor({
+        ...CFG,
+        enrollUrl: 'https://enroll.terminal.example.com/enroll',
+        recoverUrl: 'https://elsewhere.example.com/renew',
+      }),
+    ).toBe('https://elsewhere.example.com/renew')
+  })
+
+  it('is null when the enroll host carries no `enroll.` label (nothing to derive)', () => {
+    expect(recoveryRenewalUrlFor(CFG)).toBeNull()
+  })
+
+  it('posts to the NORMAL endpoint while the leaf is still valid', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    const urls: string[] = []
+    const rotator = createCertRotator(
+      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
+      ks.loadIdentity()!,
+      ks,
+      {
+        timer,
+        renewBeforeMs: 1000,
+        fetchImpl: (async (u: string) => {
+          urls.push(u)
+          return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+        }) as unknown as typeof fetch,
+        now: () => new Date(0),
+        parseCert: () => new Date(2000), // still valid at now=0
+      },
+    )
+    rotator.start()
+    timer.advance(1000)
+    await flush()
+    expect(urls).toEqual(['https://enroll.terminal.example.com/renew'])
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('posts to the RECOVERY endpoint once the leaf has already expired', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    const urls: string[] = []
+    const rotator = createCertRotator(
+      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
+      ks.loadIdentity()!,
+      ks,
+      {
+        timer,
+        renewBeforeMs: 1000,
+        fetchImpl: (async (u: string) => {
+          urls.push(u)
+          return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+        }) as unknown as typeof fetch,
+        now: () => new Date(10_000),
+        parseCert: () => new Date(2000), // expired 8s ago ⇒ delay clamps to 0
+      },
+    )
+    rotator.start()
+    timer.advance(0)
+    await flush()
+    expect(urls).toEqual(['https://recover.terminal.example.com/renew'])
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('fires onExhausted and STOPS retrying when the grace window is spent', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    let attempts = 0
+    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
+      timer,
+      renewBeforeMs: 1000,
+      retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
+      fetchImpl: (async () => {
+        attempts += 1
+        throw new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000)
+      }) as unknown as typeof fetch,
+      now: () => new Date(0),
+      parseCert: () => new Date(2000),
+    })
+    const errors: unknown[] = []
+    let exhausted: CertExpiredBeyondGraceError | null = null
+    rotator.onError((e) => errors.push(e))
+    rotator.onExhausted((e) => {
+      exhausted = e
+    })
+    rotator.start()
+    timer.advance(1000)
+    await flush()
+
+    expect(attempts).toBe(1)
+    expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError)
+    // Terminal: no retry is armed, because only a re-pair can help — retrying forever is the log
+    // spam that buried the real signal in production (6380 identical warnings).
+    expect(errors).toHaveLength(0)
+    timer.advance(60_000)
+    await flush()
+    expect(attempts).toBe(1)
+    rmSync(dir, { recursive: true, force: true })
+  })
+})