Merge fix-cert-renew-deadlock: break the expired-leaf renewal deadlock

The tunnel had been down 8 days because /renew is mTLS-authenticated by the very leaf it renews, so a lapsed leaf could never renew itself. Recovery now runs over plain HTTPS with the expired cert in the body (CSR self-signature still proves key possession), bounded by a 30-day grace; past that the agent stops retrying and names the re-pair. Verified live end-to-end: an 8-day-expired leaf self-healed and the tunnel returned HTTP 200 with no human action.
2026-07-29 10:09:35 +02:00
parent b1bc50ccd1 7064a39bf1
commit befe677759
9 changed files with 761 additions and 23 deletions
--- a/agent/src/certs/nativeRenew.ts
+++ b/agent/src/certs/nativeRenew.ts
@@ -24,7 +24,11 @@ import type { TimerLike } from '../transport/seams.js'
 import { createBackoff } from '../transport/backoff.js'
 import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js'
 import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js'
-import { createCertRotator, type CertRotator } from './rotation.js'
+import {
+  createCertRotator,
+  type CertExpiredBeyondGraceError,
+  type CertRotator,
+} from './rotation.js'

 /** Non-secret message from an unknown thrown value (never serializes cert/key material). */
 function errorMessage(err: unknown): string {
@@ -135,6 +139,8 @@ export function createMtlsFetch(
    // SYSTEM roots — its host (the LE-fronted control-plane) is publicly trusted; pinning the private
    // enroll caChain here fails with "unable to get local issuer certificate". So drop `ca` (absent →
    // node uses the default roots); rejectUnauthorized stays true.
+    // Deliberately still fail-closed on an EXPIRED leaf: nginx would refuse to forward it anyway, so
+    // a lapsed leaf is routed to the plain `/recover` endpoint by the rotator instead of through here.
    const full = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) })
    const tls: TlsClientOptions = { cert: full.cert, key: full.key, rejectUnauthorized: full.rejectUnauthorized }
    const reqInit: MtlsRequestInit = {
@@ -195,6 +201,16 @@ export function wireAutoRenew(
      error: errorMessage(err),
    })
  })
+  // Terminal: the grace window is spent, so every further attempt is guaranteed to fail. Say so once,
+  // at error level, naming the fix — and deliberately do NOT stop the supervisor: `pair` writes fresh
+  // cert files that the restart-on-exit frpc child picks up without a manual service restart.
+  rotator.onExhausted((err) => {
+    logger.log('error', 'frp-client cert expired beyond recovery grace — run `web-terminal-agent pair <CODE>` to re-pair this host', {
+      ...meta,
+      expiredForMs: err.expiredForMs,
+      graceMs: err.graceMs,
+    })
+  })
  rotator.start()
  return { stop: () => rotator.stop() }
 }
@@ -205,6 +221,10 @@ export function wireAutoRenew(
 export interface NativeAutoRenewOpts {
  readonly mtlsRequest?: MtlsRequest
  readonly certParser?: CertParser
+  /** Window in which an already-expired leaf may still be recovered via `/recover`. */
+  readonly expiredGraceMs?: number
+  /** Plain (NON-mTLS) fetch for the `/recover` call; unset ⇒ global fetch. */
+  readonly recoverFetchImpl?: typeof fetch
  readonly timer?: TimerLike
  readonly renewBeforeMs?: number
  readonly retryBaseMs?: number
@@ -235,6 +255,8 @@ export function startNativeAutoRenew(
  })
  const rotator = createCertRotator(cfg, id, ks, {
    fetchImpl,
+    ...(opts.expiredGraceMs !== undefined ? { expiredGraceMs: opts.expiredGraceMs } : {}),
+    ...(opts.recoverFetchImpl ? { recoverFetchImpl: opts.recoverFetchImpl } : {}),
    renewBeforeMs: opts.renewBeforeMs ?? DEFAULT_CERT_RENEW_WINDOW_MS,
    ...(opts.timer ? { timer: opts.timer } : {}),
    ...(opts.now ? { now: opts.now } : {}),
--- a/agent/src/certs/rotation.ts
+++ b/agent/src/certs/rotation.ts
@@ -19,6 +19,30 @@ import { certResponseToPem } from './pem.js'

 export const DEFAULT_RENEW_BEFORE_MS = 5 * 60_000 // renew 5 min before expiry

+/**
+ * How long after `notAfter` a lapsed leaf may still be swapped for a fresh one (30 days).
+ *
+ * `/renew` is mTLS-authenticated by the very leaf it renews, so a lapsed leaf cannot renew itself —
+ * a deadlock that bricked a host for 8 days in production (the laptop slept through its renewal
+ * window, then the agent logged `client certificate has expired` 6380 times and never recovered).
+ * Inside this window the agent switches to the `/recover` endpoint instead; past it, only a re-pair
+ * can help and the rotator says so once and stops.
+ */
+export const DEFAULT_EXPIRED_RENEW_GRACE_MS = 30 * 24 * 60 * 60 * 1000
+
+/** The leaf lapsed longer ago than the recovery grace allows ⇒ operator must re-pair this host. */
+export class CertExpiredBeyondGraceError extends Error {
+  constructor(
+    /** How long ago the leaf expired (ms) — non-secret, safe to log. */
+    readonly expiredForMs: number,
+    /** The grace window that was exceeded (ms). */
+    readonly graceMs: number,
+  ) {
+    super('client certificate expired beyond the recovery grace window; re-pair required')
+    this.name = 'CertExpiredBeyondGraceError'
+  }
+}
+
 export interface CertRotator {
  start(): void
  stop(): void
@@ -26,6 +50,11 @@ export interface CertRotator {
  onRevoked(cb: () => void): void
  /** A renewal attempt failed (network/HTTP, NOT a 403 revoke). The rotator retries with backoff. */
  onError(cb: (err: unknown) => void): void
+  /**
+   * TERMINAL: the leaf expired past the renewal grace window, so no future attempt can succeed. The
+   * rotator has stopped; recovery requires an operator re-pair.
+   */
+  onExhausted(cb: (err: CertExpiredBeyondGraceError) => void): void
 }

 export type RenewOutcome = 'rotated' | 'revoked'
@@ -35,6 +64,23 @@ export function renewalUrlFor(cfg: AgentConfig): string {
  return cfg.enrollUrl.replace(/\/enroll$/, '/renew')
 }

+/**
+ * Recovery route for a leaf that has ALREADY EXPIRED — a sibling PATH on the same enroll host.
+ *
+ * It cannot be `/renew`, because nginx will not forward an expired client certificate at all: under
+ * `ssl_verify_client optional` it answers a bare `400 Bad Request`, and `optional_no_ca` does not
+ * help either — nginx only tolerates CHAIN errors there (`ngx_ssl_verify_error_optional` covers
+ * self-signed / unknown-issuer / unverifiable-leaf, NOT `X509_V_ERR_CERT_HAS_EXPIRED`). So recovery
+ * drops mTLS entirely: it is a plain HTTPS POST carrying the expired cert in the BODY. Nothing is
+ * lost by that — the accompanying CSR is self-signed by the same private key, and the control-plane
+ * signer already enforces CSR proof-of-possession plus `CSR key == registered key`, so possession is
+ * proven exactly as the TLS handshake used to prove it.
+ */
+export function recoveryUrlFor(cfg: AgentConfig): string {
+  if (cfg.recoverUrl != null && cfg.recoverUrl.length > 0) return cfg.recoverUrl
+  return cfg.enrollUrl.replace(/\/enroll$/, '/recover')
+}
+
 /** Ms until (validTo − renewBeforeMs), clamped to ≥ 0. */
 export function computeRenewDelayMs(
  certPem: string,
@@ -56,9 +102,10 @@ export async function renewCert(
  id: AgentIdentity,
  ks: Keystore,
  fetchImpl: typeof fetch,
+  opts: { url?: string } = {},
 ): Promise<RenewOutcome> {
  const csr = buildCsr(id, cfg.subdomain ?? 'web-terminal-agent')
-  const res = await fetchImpl(renewalUrlFor(cfg), {
+  const res = await fetchImpl(opts.url ?? renewalUrlFor(cfg), {
    method: 'POST',
    headers: { 'content-type': 'application/json' },
    body: JSON.stringify({ csr }),
@@ -73,6 +120,34 @@ export async function renewCert(
  return 'rotated'
 }

+/**
+ * One recovery round-trip for an EXPIRED leaf: plain HTTPS (no client cert — see `recoveryUrlFor`)
+ * POSTing the expired cert alongside a fresh CSR over the SAME key. Same outcome contract as
+ * `renewCert`: 'rotated' installs the new leaf, 403 ⇒ 'revoked', anything else throws to the retry.
+ */
+export async function recoverCert(
+  cfg: AgentConfig,
+  id: AgentIdentity,
+  ks: Keystore,
+  fetchImpl: typeof fetch,
+  url: string = recoveryUrlFor(cfg),
+): Promise<RenewOutcome> {
+  const certs = ks.loadCert()
+  if (certs === null) throw new Error('cannot recover without the expired leaf on disk')
+  const csr = buildCsr(id, cfg.subdomain ?? 'web-terminal-agent')
+  const res = await fetchImpl(url, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({ cert: certs.certPem, csr }),
+  })
+  if (res.status === 403) return 'revoked'
+  if (!res.ok) throw new Error(`cert recovery failed: HTTP ${res.status}`)
+  const json = (await res.json()) as { cert?: unknown; caChain?: unknown }
+  const { certPem, caChainPem } = certResponseToPem(json.cert, json.caChain)
+  ks.saveCert(certPem, caChainPem)
+  return 'rotated'
+}
+
 export function createCertRotator(
  cfg: AgentConfig,
  id: AgentIdentity,
@@ -85,6 +160,10 @@ export function createCertRotator(
    parseCert?: (pem: string) => Date
    /** Backoff policy for retrying a FAILED renewal (default 1s→30s). Reset after a success. */
    retryBackoff?: BackoffPolicy
+    /** Plain (NON-mTLS) fetch used only for the expired-leaf `/recover` call. */
+    recoverFetchImpl?: typeof fetch
+    /** Window in which an expired leaf may still be recovered. 0 ⇒ no recovery at all. */
+    expiredGraceMs?: number
  } = {},
 ): CertRotator {
  const renewBeforeMs = opts.renewBeforeMs ?? DEFAULT_RENEW_BEFORE_MS
@@ -96,12 +175,15 @@ export function createCertRotator(
    clearInterval: (h) => clearInterval(h as ReturnType<typeof setInterval>),
  }
  const doFetch = opts.fetchImpl ?? fetch
+  const recoverFetch = opts.recoverFetchImpl ?? fetch
+  const expiredGraceMs = opts.expiredGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS
  const now = opts.now ?? (() => new Date())
  const retryBackoff = opts.retryBackoff ?? createBackoff({ jitter: true })
  let handle: unknown = null
  let rotatedCb: (() => void) | null = null
  let revokedCb: (() => void) | null = null
  let errorCb: ((err: unknown) => void) | null = null
+  let exhaustedCb: ((err: CertExpiredBeyondGraceError) => void) | null = null

  function schedule(): void {
    const certs = ks.loadCert()
@@ -110,8 +192,34 @@ export function createCertRotator(
    handle = timer.setTimeout(runRenewal, delay)
  }

+  /**
+   * How this attempt must be made, derived from how stale the leaf on disk is:
+   *   `normal`    — still valid ⇒ ordinary mTLS `/renew`;
+   *   `recover`   — expired but inside the grace window ⇒ plain `/recover` with the cert in the body;
+   *   `exhausted` — expired past the grace window ⇒ nothing can succeed; only an operator re-pair.
+   */
+  function attemptPlan(): { mode: 'normal' | 'recover' | 'exhausted'; expiredForMs: number } {
+    const certs = ks.loadCert()
+    if (certs === null) return { mode: 'normal', expiredForMs: 0 }
+    const expiredForMs = now().getTime() - parseCert(certs.certPem).getTime()
+    if (expiredForMs <= 0) return { mode: 'normal', expiredForMs: 0 }
+    return { mode: expiredForMs > expiredGraceMs ? 'exhausted' : 'recover', expiredForMs }
+  }
+
  function runRenewal(): void {
-    void renewCert(cfg, id, ks, doFetch)
+    const plan = attemptPlan()
+    if (plan.mode === 'exhausted') {
+      // Terminal: report ONCE and arm nothing. The old code retried forever, which is how a single
+      // real failure turned into 6380 identical warnings that buried the signal.
+      handle = null
+      exhaustedCb?.(new CertExpiredBeyondGraceError(plan.expiredForMs, expiredGraceMs))
+      return
+    }
+    const attempt =
+      plan.mode === 'recover'
+        ? recoverCert(cfg, id, ks, recoverFetch)
+        : renewCert(cfg, id, ks, doFetch)
+    void attempt
      .then((outcome) => {
        if (outcome === 'revoked') {
          revokedCb?.()
@@ -147,5 +255,8 @@ export function createCertRotator(
    onError(cb): void {
      errorCb = cb
    },
+    onExhausted(cb): void {
+      exhaustedCb = cb
+    },
  }
 }
--- a/agent/src/config/agentConfig.ts
+++ b/agent/src/config/agentConfig.ts
@@ -18,6 +18,13 @@ export interface AgentConfig {
  readonly localTargetUrl: string
  readonly subdomain: string | null
  readonly hostId: string | null
+  /**
+   * Renewal endpoint used ONLY when the current leaf has already expired (the strict `/renew` vhost
+   * rejects an expired client cert before it reaches the control-plane). Optional: when unset it is
+   * derived from `enrollUrl` by swapping the `enroll.` label for `recover.` — see
+   * `certs/rotation.ts` `recoveryRenewalUrlFor`.
+   */
+  readonly recoverUrl?: string | null | undefined
 }

 /**
@@ -59,6 +66,11 @@ export const AgentConfigSchema = z
      .refine(isLoopbackWsUrl, 'localTargetUrl must be a ws:// loopback URL (anti-SSRF)'),
    subdomain: z.string().min(1).nullable(),
    hostId: z.string().min(1).nullable(),
+    recoverUrl: z
+      .string()
+      .refine((u) => hasScheme(u, 'https:'), 'recoverUrl must be an https:// URL')
+      .nullable()
+      .optional(),
  })
  .strict()
  .readonly()
@@ -85,6 +97,7 @@ export function loadAgentConfig(
    localTargetUrl: argv.localTargetUrl ?? env.LOCAL_TARGET_URL ?? DEFAULT_LOCAL_TARGET,
    subdomain: argv.subdomain ?? env.SUBDOMAIN ?? null,
    hostId: argv.hostId ?? env.HOST_ID ?? null,
+    recoverUrl: argv.recoverUrl ?? env.RECOVER_URL ?? null,
  }
  return AgentConfigSchema.parse(merged)
 }
--- a/agent/test/nativeRenew.test.ts
+++ b/agent/test/nativeRenew.test.ts
@@ -27,6 +27,7 @@ import {
  type MtlsRequest,
 } from '../src/certs/nativeRenew.js'
 import { FakeTimer } from './fixtures/fakes.js'
+import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js'

 const CFG: AgentConfig = {
  relayUrl: 'wss://relay/agent',
@@ -109,11 +110,21 @@ describe('createMtlsFetch (A5)', () => {
 describe('wireAutoRenew (A5)', () => {
  function fakeRotator(): {
    rotator: CertRotator
-    fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void }
+    fire: {
+      rotated?: () => void
+      revoked?: () => void
+      error?: (e: unknown) => void
+      exhausted?: (e: CertExpiredBeyondGraceError) => void
+    }
    start: ReturnType<typeof vi.fn>
    stop: ReturnType<typeof vi.fn>
  } {
-    const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {}
+    const fire: {
+      rotated?: () => void
+      revoked?: () => void
+      error?: (e: unknown) => void
+      exhausted?: (e: CertExpiredBeyondGraceError) => void
+    } = {}
    const start = vi.fn()
    const stop = vi.fn()
    const rotator: CertRotator = {
@@ -128,6 +139,9 @@ describe('wireAutoRenew (A5)', () => {
      onError: (cb) => {
        fire.error = cb
      },
+      onExhausted: (cb) => {
+        fire.exhausted = cb
+      },
    }
    return { rotator, fire, start, stop }
  }
@@ -277,3 +291,60 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => {
    rmSync(dir, { recursive: true, force: true })
  })
 })
+
+/**
+ * The mTLS renew transport stays STRICT about expiry: nginx refuses to forward an expired client
+ * cert at all, so an expired leaf must be routed to the plain `/recover` endpoint by the rotator
+ * rather than smuggled through this transport.
+ */
+describe('createMtlsFetch stays fail-closed on an expired leaf', () => {
+  it('refuses to present a lapsed leaf (recovery is the rotator\'s job, not this transport\'s)', async () => {
+    const { dir, ks } = enrolledKs()
+    let called = 0
+    const request: MtlsRequest = async () => {
+      called += 1
+      return { status: 201, body: '{}' }
+    }
+    const f = createMtlsFetch(ks, {
+      request,
+      certParser: () => ({ validTo: new Date(Date.now() - 86_400_000) }),
+    })
+    await expect(f('https://cp.example.com/renew', { method: 'POST' })).rejects.toThrow(
+      /expired/i,
+    )
+    expect(called).toBe(0)
+    rmSync(dir, { recursive: true, force: true })
+  })
+})
+
+describe('wireAutoRenew exhausted routing', () => {
+  it('logs an actionable re-pair alarm and leaves the supervisor running', () => {
+    const fire: { exhausted?: (e: CertExpiredBeyondGraceError) => void } = {}
+    const rotator: CertRotator = {
+      start: vi.fn(),
+      stop: vi.fn(),
+      onRotated: () => {},
+      onRevoked: () => {},
+      onError: () => {},
+      onExhausted: (cb) => {
+        fire.exhausted = cb
+      },
+    }
+    const restartChild = vi.fn()
+    const stop = vi.fn()
+    const lines: string[] = []
+    wireAutoRenew(rotator, { restartChild, stop }, createLogger('info', (l) => lines.push(l)), {
+      subdomain: 'h7fd8',
+      hostId: 'h-1',
+    })
+    fire.exhausted?.(new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000))
+
+    const alarm = lines.find((l) => /re-pair/i.test(l))
+    expect(alarm).toBeDefined()
+    expect(alarm).toContain('h7fd8')
+    // The supervisor keeps running: a later `pair` writes fresh cert files that the restarting
+    // frpc child picks up. Tearing down here would make recovery need a manual restart too.
+    expect(stop).not.toHaveBeenCalled()
+    expect(restartChild).not.toHaveBeenCalled()
+  })
+})
--- a/agent/test/rotation.test.ts
+++ b/agent/test/rotation.test.ts
@@ -8,9 +8,12 @@ import { openKeystore } from '../src/keys/keystore.js'
 import {
  computeRenewDelayMs,
  createCertRotator,
+  recoverCert,
+  recoveryUrlFor,
  renewCert,
  renewalUrlFor,
 } from '../src/certs/rotation.js'
+import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js'
 import { createBackoff } from '../src/transport/backoff.js'
 import { FakeTimer } from './fixtures/fakes.js'

@@ -159,3 +162,149 @@ describe('createCertRotator (T13)', () => {
    rmSync(dir, { recursive: true, force: true })
  })
 })
+
+/**
+ * Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf
+ * it renews, so a lapsed leaf can never renew itself — and nginx will not even forward an expired
+ * client cert. Recovery is therefore a PLAIN POST to `/recover` carrying the expired cert in the
+ * body, plus a terminal signal once the grace window is spent so the agent stops retrying forever.
+ */
+describe('expired-leaf recovery', () => {
+  const HOUR = 3_600_000
+
+  it('derives /recover as a sibling PATH on the same enroll host', () => {
+    expect(recoveryUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
+      'https://enroll.terminal.example.com/recover',
+    )
+  })
+
+  it('honours an explicit recoverUrl over the derivation', () => {
+    expect(recoveryUrlFor({ ...CFG, recoverUrl: 'https://elsewhere.example.com/recover' })).toBe(
+      'https://elsewhere.example.com/recover',
+    )
+  })
+
+  it('recoverCert posts the EXPIRED cert plus a CSR, with no client cert, and installs the result', async () => {
+    const { dir, ks } = enrolledKs()
+    let seenUrl = ''
+    let seenBody: { cert?: string; csr?: string } = {}
+    const fetchImpl = (async (u: string, init: { body: string }) => {
+      seenUrl = u
+      seenBody = JSON.parse(init.body)
+      return jsonRes(201, { cert: 'FRESHCERT', caChain: ['FRESHCA'] })
+    }) as unknown as typeof fetch
+
+    const outcome = await recoverCert(
+      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
+      ks.loadIdentity()!,
+      ks,
+      fetchImpl,
+    )
+    expect(outcome).toBe('rotated')
+    expect(seenUrl).toBe('https://enroll.terminal.example.com/recover')
+    expect(seenBody.cert).toBe('OLDCERT') // the lapsed leaf travels in the BODY, not the TLS layer
+    expect(seenBody.csr).toBeTruthy()
+    expect(ks.loadCert()!.certPem).toContain('FRESHCERT')
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('a still-valid leaf uses the mTLS /renew fetch, never the recovery one', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    let renewCalls = 0
+    let recoverCalls = 0
+    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
+      timer,
+      renewBeforeMs: 1000,
+      fetchImpl: (async () => {
+        renewCalls += 1
+        return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+      }) as unknown as typeof fetch,
+      recoverFetchImpl: (async () => {
+        recoverCalls += 1
+        return jsonRes(200, {})
+      }) as unknown as typeof fetch,
+      now: () => new Date(0),
+      parseCert: () => new Date(2000), // valid at now=0
+    })
+    rotator.start()
+    timer.advance(1000)
+    await flush()
+    expect(renewCalls).toBe(1)
+    expect(recoverCalls).toBe(0)
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('an expired leaf INSIDE the grace window switches to the recovery fetch', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    let renewCalls = 0
+    let recoverCalls = 0
+    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
+      timer,
+      renewBeforeMs: 1000,
+      expiredGraceMs: 30 * 24 * HOUR,
+      fetchImpl: (async () => {
+        renewCalls += 1
+        return jsonRes(200, {})
+      }) as unknown as typeof fetch,
+      recoverFetchImpl: (async () => {
+        recoverCalls += 1
+        return jsonRes(201, { cert: 'NEWCERT', caChain: ['NEWCA'] })
+      }) as unknown as typeof fetch,
+      now: () => new Date(8 * 24 * HOUR), // 8 days after the leaf lapsed
+      parseCert: () => new Date(0),
+    })
+    let rotated = 0
+    rotator.onRotated(() => {
+      rotated += 1
+    })
+    rotator.start()
+    timer.advance(0)
+    await flush()
+    expect(recoverCalls).toBe(1)
+    expect(renewCalls).toBe(0)
+    expect(rotated).toBe(1)
+    rotator.stop()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('BEYOND the grace window it fires onExhausted, issues no request, and stops retrying', async () => {
+    const { dir, ks } = enrolledKs()
+    const timer = new FakeTimer()
+    let requests = 0
+    const countingFetch = (async () => {
+      requests += 1
+      return jsonRes(201, {})
+    }) as unknown as typeof fetch
+    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
+      timer,
+      renewBeforeMs: 1000,
+      expiredGraceMs: 30 * 24 * HOUR,
+      retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
+      fetchImpl: countingFetch,
+      recoverFetchImpl: countingFetch,
+      now: () => new Date(31 * 24 * HOUR), // 31 days stale ⇒ past a 30-day grace
+      parseCert: () => new Date(0),
+    })
+    const errors: unknown[] = []
+    let exhausted: CertExpiredBeyondGraceError | null = null
+    rotator.onError((e) => errors.push(e))
+    rotator.onExhausted((e) => {
+      exhausted = e
+    })
+    rotator.start()
+    timer.advance(0)
+    await flush()
+
+    expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError)
+    expect(requests).toBe(0) // nothing is even attempted — it cannot succeed
+    expect(errors).toHaveLength(0)
+    // Terminal: no retry armed. Retrying forever is what produced 6380 identical warnings.
+    timer.advance(60_000)
+    await flush()
+    expect(requests).toBe(0)
+    rmSync(dir, { recursive: true, force: true })
+  })
+})
--- a/control-plane/src/api/renew.ts
+++ b/control-plane/src/api/renew.ts
@@ -51,12 +51,29 @@ export const DEFAULT_RENEW_RATE_WINDOW_MS = 60 * 60 * 1000
 export const RENEW_RATE_MAX_IDENTITIES = 10_000
 /** Max base64 length of a submitted CSR (a P-256 PKCS#10 is well under 1 KB; this is generous slack). */
 export const MAX_CSR_B64_LEN = 8192
+/** Max wire length of a body-carried certificate (a P-256 leaf PEM is ~1 KB; generous slack). */
+export const MAX_PRESENTED_CERT_LEN = 16384
 /**
 * Default header the mTLS terminator forwards the verified client cert in (base64 DER). The terminator
 * MUST set this from `$ssl_client_cert` AND strip any client-supplied copy — a client can never provide
 * its own current cert. Production wiring can swap in a socket-peer-cert resolver instead.
 */
 export const DEFAULT_CLIENT_CERT_HEADER = 'x-client-cert'
+/**
+ * How long after `notAfter` a presented leaf may still authenticate its OWN re-issuance (30 days).
+ *
+ * `/renew` is authenticated by the very leaf it renews, so a strict expiry check means a host whose
+ * leaf lapsed can never renew it — it is bricked until an operator re-pairs (this happened: a laptop
+ * slept through its renewal window and the tunnel stayed down for 8 days). Accepting a recently
+ * expired leaf FOR RE-ISSUANCE ONLY breaks that deadlock.
+ *
+ * Nothing else is relaxed: the leaf must still chain to the CA anchor set, its SPIFFE identity must
+ * still resolve to a registry record that is `active` and account-consistent, and `notBefore` is NOT
+ * graced. The residual risk is that a stale stolen leaf stays usable for this window — an attacker
+ * holding an unexpired stolen leaf can already renew indefinitely, so this widens an existing
+ * exposure rather than opening a new one. Set to 0 to restore the strict behaviour.
+ */
+export const DEFAULT_EXPIRED_RENEW_GRACE_MS = 30 * 24 * 60 * 60 * 1000

 /** Uniform reject: 401 = no/invalid current cert (unauthenticated); 403 = cert valid but not allowed. */
 export class RenewRejectError extends Error {
@@ -170,10 +187,18 @@ export function headerPresentedCert(headerName: string = DEFAULT_CLIENT_CERT_HEA
      const raw = req.headers[name]
      const value = Array.isArray(raw) ? raw[0] : raw
      if (typeof value !== 'string' || value.length === 0) return null
+      return certWireToDer(value)
+    },
+  }
+}
+
+/**
+ * Normalize a certificate carried over the wire to raw DER. Accepts base64 DER, PEM, and nginx's
+ * header-safe `$ssl_client_escaped_cert` (URL-encoded PEM): URL-decode if escaped, then strip PEM
+ * armor + whitespace to recover the base64 body. Returns null on anything unparseable.
+ */
+export function certWireToDer(value: string): Uint8Array | null {
  try {
-        // The terminator may forward the verified cert as base64 DER, OR as PEM — including nginx's
-        // header-safe `$ssl_client_escaped_cert` (URL-encoded PEM). Normalize all three to raw DER:
-        // URL-decode if escaped, then strip PEM armor + whitespace to recover the base64 DER body.
    let s = value.includes('%') ? safeDecodeUri(value) : value
    if (s.includes('BEGIN CERTIFICATE')) {
      s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '')
@@ -183,8 +208,6 @@ export function headerPresentedCert(headerName: string = DEFAULT_CLIENT_CERT_HEA
  } catch {
    return null
  }
-    },
-  }
 }

 /** URL-decode `value`, returning it unchanged if it is not valid percent-encoding. */
@@ -207,6 +230,7 @@ function assertPresentedCertTrusted(
  der: Uint8Array,
  caAnchorsDer: readonly Uint8Array[],
  nowMs: number,
+  expiredGraceMs: number,
 ): void {
  let leaf: NodeX509Certificate
  let anchors: NodeX509Certificate[]
@@ -220,7 +244,10 @@ function assertPresentedCertTrusted(
  const notBefore = new Date(leaf.validFrom).getTime()
  const notAfter = new Date(leaf.validTo).getTime()
  if (Number.isNaN(notBefore) || Number.isNaN(notAfter)) throw new RenewRejectError(401)
-  if (nowMs < notBefore || nowMs > notAfter) throw new RenewRejectError(401)
+  // `notBefore` is never graced — a not-yet-valid cert is nonsense, not a recoverable lapse. Only
+  // `notAfter` gets the bounded renewal grace (see DEFAULT_EXPIRED_RENEW_GRACE_MS).
+  if (nowMs < notBefore) throw new RenewRejectError(401)
+  if (nowMs > notAfter + Math.max(0, expiredGraceMs)) throw new RenewRejectError(401)
 }

 /**
@@ -234,7 +261,11 @@ function assertPresentedCertTrusted(
 function parsePresentedCertIdentity(
  der: Uint8Array,
  expectedKind: SpiffeKind,
-  verify?: { readonly caAnchorsDer: readonly Uint8Array[]; readonly nowMs: number },
+  verify?: {
+    readonly caAnchorsDer: readonly Uint8Array[]
+    readonly nowMs: number
+    readonly expiredGraceMs: number
+  },
 ): CertIdentity {
  let leaf: x509.X509Certificate
  try {
@@ -242,7 +273,8 @@ function parsePresentedCertIdentity(
  } catch {
    throw new RenewRejectError(401)
  }
-  if (verify !== undefined) assertPresentedCertTrusted(der, verify.caAnchorsDer, verify.nowMs)
+  if (verify !== undefined)
+    assertPresentedCertTrusted(der, verify.caAnchorsDer, verify.nowMs, verify.expiredGraceMs)
  const san = leaf.getExtension(x509.SubjectAlternativeNameExtension)
  const uri = san?.names.toJSON().find((n) => n.type === 'url')?.value
  if (uri === undefined) throw new RenewRejectError(401)
@@ -262,6 +294,13 @@ function parsePresentedCertIdentity(
 }

 const RenewBodySchema = z.object({ csr: z.string().min(1).max(MAX_CSR_B64_LEN) }).strict()
+/** `/recover` additionally carries the EXPIRED leaf itself (base64 DER or PEM). */
+const RecoverBodySchema = z
+  .object({
+    cert: z.string().min(1).max(MAX_PRESENTED_CERT_LEN),
+    csr: z.string().min(1).max(MAX_CSR_B64_LEN),
+  })
+  .strict()

 export interface RenewDeps {
  readonly hosts: HostRegistry
@@ -279,6 +318,11 @@ export interface RenewDeps {
  readonly hostCaAnchorsDer?: readonly Uint8Array[]
  /** device-CA anchor DER(s) — same role for the DEVICE renew path. */
  readonly deviceCaAnchorsDer?: readonly Uint8Array[]
+  /**
+   * Window after `notAfter` in which a presented leaf may still authenticate its own re-issuance.
+   * Defaults to `DEFAULT_EXPIRED_RENEW_GRACE_MS`; 0 restores the strict fail-closed behaviour.
+   */
+  readonly expiredRenewGraceMs?: number
 }

 /** Map any thrown error to a uniform HTTP reject — never leak which internal check failed. */
@@ -298,6 +342,9 @@ function sendError(reply: FastifyReply, err: unknown): void {
 export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
  const presentedCert = deps.presentedCert ?? headerPresentedCert()
  const rateLimiter = deps.rateLimiter ?? createRenewRateLimiter()
+  // `/renew` stays STRICT (0): the mTLS terminator would never forward an expired cert to it anyway.
+  // The grace belongs to `/recover`, the route built for exactly that case.
+  const recoverGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS

  return async (app) => {
    app.post('/renew', async (req, reply) => {
@@ -307,7 +354,9 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
        const identity = parsePresentedCertIdentity(
          der,
          'host',
-          deps.hostCaAnchorsDer ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now() } : undefined,
+          deps.hostCaAnchorsDer
+            ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
+            : undefined,
        )
        rateLimiter.check(`host:${identity.accountId}:${identity.id}`)

@@ -333,6 +382,57 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
      }
    })

+    /**
+     * Expired-leaf recovery. The lapsed cert arrives in the BODY because no TLS terminator will
+     * forward an expired client certificate (nginx `optional` → bare 400; `optional_no_ca` tolerates
+     * only chain errors, never `X509_V_ERR_CERT_HAS_EXPIRED`). That makes this route the SOLE
+     * verifier, so it runs the identical trust pipeline as `/renew` — real X.509 path validation to
+     * the frp-client-CA anchors, SPIFFE parse, `notBefore`, registry `active` + account match — and
+     * differs ONLY in allowing a bounded overrun on `notAfter`.
+     *
+     * Possession of the private key is still proven: the CSR is self-signed by it, and the host
+     * signer's delegated gate enforces CSR PoP plus `CSR key == registered key`. A replayed cert
+     * without the key therefore yields, at most, a certificate the attacker cannot authenticate with.
+     * The header channel is deliberately IGNORED here — on this route only the body speaks.
+     */
+    app.post('/recover', async (req, reply) => {
+      try {
+        const body = RecoverBodySchema.parse(req.body)
+        const der = certWireToDer(body.cert)
+        if (der === null) throw new RenewRejectError(401)
+        const identity = parsePresentedCertIdentity(
+          der,
+          'host',
+          deps.hostCaAnchorsDer
+            ? {
+                caAnchorsDer: deps.hostCaAnchorsDer,
+                nowMs: Date.now(),
+                expiredGraceMs: recoverGraceMs,
+              }
+            : undefined,
+        )
+        rateLimiter.check(`recover:${identity.accountId}:${identity.id}`)
+
+        const host = await deps.hosts.getHostBySubdomain(identity.id)
+        if (host === null || host.status === 'revoked' || host.accountId !== identity.accountId) {
+          throw new RenewRejectError(403)
+        }
+
+        const leaf = await deps.renewer.renewHostLeaf(
+          host.hostId,
+          identity.publicKeySpki,
+          decodeCsrWire(body.csr),
+        )
+        await reply.code(201).send({
+          cert: bytesToBase64(leaf.cert),
+          caChain: leaf.caChain.map((c) => bytesToBase64(c)),
+          notAfter: leaf.notAfter.toISOString(),
+        })
+      } catch (err) {
+        sendError(reply, err)
+      }
+    })
+
    app.post('/device/:id/renew', async (req, reply) => {
      try {
        const der = presentedCert.presentedCertDer(req)
@@ -340,7 +440,9 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
        const identity = parsePresentedCertIdentity(
          der,
          'device',
-          deps.deviceCaAnchorsDer ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now() } : undefined,
+          deps.deviceCaAnchorsDer
+            ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
+            : undefined,
        )
        const id = (req.params as { id: string }).id
        rateLimiter.check(`device:${identity.id}`)
--- a/control-plane/test/renew.test.ts
+++ b/control-plane/test/renew.test.ts
@@ -502,6 +502,8 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio
      headers: { 'x-client-cert': certHeader(expired) },
      payload: { csr: b64Csr(ctx.csr) },
    })
+    // STRICT on this route: nginx would never forward an expired client cert here anyway, so the
+    // deadlock escape hatch lives on `/recover` (below) and NOT on the mTLS renewal path.
    expect(res.statusCode).toBe(401)
  })

@@ -521,3 +523,152 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio
    expect(res.statusCode).toBe(401)
  })
 })
+
+/**
+ * CP6d · POST /recover — the expired-leaf escape hatch.
+ *
+ * Unlike `/renew` this route is NOT mTLS-authenticated: nginx cannot forward an expired client cert
+ * (under `ssl_verify_client optional` it 400s, and `optional_no_ca` only tolerates CHAIN errors, not
+ * `X509_V_ERR_CERT_HAS_EXPIRED`). The lapsed cert therefore arrives in the BODY, and the
+ * control-plane becomes the sole verifier. These tests pin down that nothing except the `notAfter`
+ * bound was relaxed — chain, SPIFFE, `notBefore`, and registry status all still decide.
+ */
+describe('CP6d POST /recover — expired-leaf recovery', () => {
+  async function mintLeaf(
+    ctx: HostCtx,
+    notBefore: Date,
+    notAfter: Date,
+    signer = ctx.ca.caSigner,
+  ): Promise<Uint8Array> {
+    const spiffe = `spiffe://relay.terminal.yaojia.wang/account/${ctx.accountId}/host/${ctx.subdomain}`
+    return assembleCertificate({
+      subjectPublicKey: ctx.spki,
+      subject: `CN=${ctx.subdomain}`,
+      issuer: ctx.ca.caCert.subjectName,
+      serialNumber: Uint8Array.from([0x0b]),
+      notBefore,
+      notAfter,
+      extensions: [
+        new x509.SubjectAlternativeNameExtension([
+          { type: 'dns', value: `${ctx.subdomain}.terminal.yaojia.wang` },
+          { type: 'url', value: spiffe },
+        ]),
+        new x509.BasicConstraintsExtension(false, undefined, true),
+        new x509.KeyUsagesExtension(x509.KeyUsageFlags.digitalSignature, true),
+        new x509.ExtendedKeyUsageExtension([x509.ExtendedKeyUsage.clientAuth]),
+      ],
+      signer,
+      sigAlg: 'ecdsa-p256',
+    })
+  }
+
+  const b64 = (der: Uint8Array): string => Buffer.from(der).toString('base64')
+
+  async function post(ctx: HostCtx, cert: Uint8Array, extra?: Partial<RenewDeps>) {
+    const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer], ...extra }))
+    await app.ready()
+    return app.inject({
+      method: 'POST',
+      url: '/recover',
+      payload: { cert: b64(cert), csr: b64Csr(ctx.csr) },
+    })
+  }
+
+  test('a leaf expired INSIDE the grace window is re-issued → 201 (breaks the deadlock)', async () => {
+    const ctx = await hostCtx('alice')
+    const now = Date.now()
+    const res = await post(ctx, await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS)))
+    expect(res.statusCode).toBe(201)
+    expect(JSON.parse(res.payload).cert).toBeTruthy()
+  })
+
+  test('a leaf expired BEYOND the grace window is refused → 401', async () => {
+    const ctx = await hostCtx('alice')
+    const now = Date.now()
+    const res = await post(ctx, await mintLeaf(ctx, new Date(now - 60 * DAY_MS), new Date(now - 31 * DAY_MS)))
+    expect(res.statusCode).toBe(401)
+  })
+
+  test('grace 0 disables recovery entirely → 401', async () => {
+    const ctx = await hostCtx('alice')
+    const now = Date.now()
+    const res = await post(
+      ctx,
+      await mintLeaf(ctx, new Date(now - 2 * DAY_MS), new Date(now - DAY_MS)),
+      { expiredRenewGraceMs: 0 },
+    )
+    expect(res.statusCode).toBe(401)
+  })
+
+  /**
+   * THE load-bearing test for this route. nginx no longer validates the chain here, so a forged
+   * self-signed cert carrying a correct-looking SPIFFE SAN must be rejected by the control-plane
+   * alone. If this ever goes green-to-red, `/recover` becomes an unauthenticated cert vending machine.
+   */
+  test('a self-signed cert with a FORGED SPIFFE SAN is refused → 401', async () => {
+    const ctx = await hostCtx('alice')
+    const rogueCa = await makeP256Ca('rogue-CA')
+    const now = Date.now()
+    const rogue = await mintLeaf(
+      ctx,
+      new Date(now - 9 * DAY_MS),
+      new Date(now - 8 * DAY_MS),
+      rogueCa.caSigner,
+    )
+    const res = await post(ctx, rogue)
+    expect(res.statusCode).toBe(401)
+  })
+
+  test('recovery NEVER bypasses revocation — revoked host → 403', async () => {
+    const ctx = await hostCtx('alice')
+    const now = Date.now()
+    const cert = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS))
+    const host = await ctx.hosts.getHostBySubdomain('alice')
+    await ctx.hosts.setHostStatus(host!.hostId, 'revoked')
+    const res = await post(ctx, cert)
+    expect(res.statusCode).toBe(403)
+  })
+
+  test('grace covers notAfter ONLY — a not-yet-valid leaf is refused → 401', async () => {
+    const ctx = await hostCtx('alice')
+    const now = Date.now()
+    const res = await post(ctx, await mintLeaf(ctx, new Date(now + DAY_MS), new Date(now + 2 * DAY_MS)))
+    expect(res.statusCode).toBe(401)
+  })
+
+  test('a still-valid leaf may also use /recover → 201', async () => {
+    const ctx = await hostCtx('alice')
+    const res = await post(ctx, ctx.currentCertDer)
+    expect(res.statusCode).toBe(201)
+  })
+
+  test('a missing cert field is rejected → 400 (uniform schema reject, same as a missing csr)', async () => {
+    const ctx = await hostCtx('alice')
+    const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
+    await app.ready()
+    const res = await app.inject({ method: 'POST', url: '/recover', payload: { csr: b64Csr(ctx.csr) } })
+    // The module answers with uniform rejects that never say which check failed: 400 for a malformed
+    // body, 401 for a cert that parses but is not trusted, 403 for a trusted cert that is not allowed.
+    expect(res.statusCode).toBe(400)
+  })
+
+  /**
+   * The header is the mTLS terminator's channel. On `/recover` the cert comes from the body, so a
+   * client-supplied header must not be able to substitute an identity.
+   */
+  test('an x-client-cert HEADER cannot override the body identity → 401', async () => {
+    const ctx = await hostCtx('alice')
+    const rogueCa = await makeP256Ca('rogue-CA')
+    const now = Date.now()
+    const rogue = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS), rogueCa.caSigner)
+    const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
+    await app.ready()
+    const res = await app.inject({
+      method: 'POST',
+      url: '/recover',
+      headers: { 'x-client-cert': certHeader(ctx.currentCertDer) },
+      payload: { cert: b64(rogue), csr: b64Csr(ctx.csr) },
+    })
+    expect(res.statusCode).toBe(401)
+  })
+})
--- a/deploy/nginx/enroll-recover-location.md
+++ b/deploy/nginx/enroll-recover-location.md
@@ -0,0 +1,93 @@
+# Enroll vhost addition — `/recover` (expired-leaf recovery)
+
+One `location` block to **merge into the existing** enroll vhost on the VPS
+(`/etc/nginx/conf.d/enroll.conf`, the `127.0.0.1:8471` server). No new server, no new SNI route, no
+DNS work.
+
+> **This is a MERGE, not a file to ship.** The enroll vhost also carries `/enroll`, `/device/enroll`,
+> `/auth/login`, `/crl/`, `/renew` and `/device/:id/renew` — do not replace it.
+
+## Why the route exists
+
+`POST /renew` is mTLS-authenticated by the very leaf it renews, so once that leaf lapses the host can
+never renew it and the tunnel stays down until an operator re-pairs. That is not hypothetical: a
+laptop slept through its 8h renewal window, its 24h leaf expired, and the agent then logged
+`client certificate has expired; renew before dialling` 6380 times over 8 days without recovering.
+
+## Why it cannot be fixed on `/renew` itself
+
+**nginx will not forward an expired client certificate, under any `ssl_verify_client` mode.**
+
+- `optional` → nginx answers a bare `400 The SSL certificate error` as soon as verification fails.
+  The request never reaches the `location`, so no `if ($ssl_client_verify …)` can rescue it.
+- `optional_no_ca` → does **not** help either. It only tolerates *chain* failures; see nginx's
+  `ngx_ssl_verify_error_optional()`, which covers `DEPTH_ZERO_SELF_SIGNED_CERT`,
+  `SELF_SIGNED_CERT_IN_CHAIN`, `UNABLE_TO_GET_ISSUER_CERT_LOCALLY` and
+  `UNABLE_TO_VERIFY_LEAF_SIGNATURE` — and **not** `X509_V_ERR_CERT_HAS_EXPIRED`.
+- `ssl_verify_client` is a `server`-level directive, so it cannot be relaxed per-location anyway.
+
+So recovery drops mTLS: `/recover` takes **no client certificate**, and the lapsed cert travels in
+the request body instead.
+
+## Why that is still authenticated
+
+A certificate is public, so the body alone proves nothing — possession of the **private key** does,
+and it is still proven end to end:
+
+- the accompanying CSR is **self-signed by that key**, and the host signer's delegated gate enforces
+  CSR proof-of-possession plus `CSR key == registered key` (`control-plane/src/ca/csr.ts`
+  `verifyCsrPoP`);
+- `control-plane/src/api/renew.ts` runs the *same* trust pipeline as `/renew` — real X.509 path
+  validation to the frp-client-CA anchors, SPIFFE SAN parse, `notBefore`, and a registry lookup
+  requiring an `active`, account-consistent host — differing **only** in a bounded overrun allowance
+  on `notAfter` (`DEFAULT_EXPIRED_RENEW_GRACE_MS`, 30 days).
+
+Worst case for a replayed cert without the key: the attacker receives a certificate they cannot
+authenticate with. Revocation still bites, via registry status.
+
+## The block to add
+
+```nginx
+    # Expired-leaf recovery: NO client cert (see enroll-recover-location.md). The control-plane is
+    # the sole verifier; strip any client-supplied cert header so only the body can speak.
+    location = /recover {
+        limit_req zone=renew_recover burst=5 nodelay;
+        proxy_pass http://127.0.0.1:8080;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header x-client-cert "";
+        proxy_read_timeout 60s;
+    }
+```
+
+And once, in the `http` context (top of `enroll.conf` is fine) — the route is reachable
+pre-authentication and costs the control-plane an X.509 path validation, so bound it. The
+control-plane additionally rate-limits per identity (`createRenewRateLimiter`, 30/hour):
+
+```nginx
+limit_req_zone $binary_remote_addr zone=renew_recover:1m rate=10r/m;
+```
+
+## Deploy gate
+
+```bash
+cp /etc/nginx/conf.d/enroll.conf{,.bak.$(date +%s)}   # snapshot first
+# ...merge the block above...
+nginx -t && systemctl restart nginx                   # NEVER reload on a failed -t
+```
+
+`restart`, not `reload`: a graceful reload has been observed keeping old workers alive for seconds,
+which makes post-deploy verification race the change.
+
+## Verify
+
+```bash
+# no cert needed — an in-grace expired leaf is re-issued
+curl -sS --resolve enroll.terminal.yaojia.wang:443:<vps> \
+  -X POST -H 'content-type: application/json' \
+  -d "{\"cert\":\"$(base64 -w0 expired.cert.pem)\",\"csr\":\"$(base64 -w0 new.csr.der)\"}" \
+  https://enroll.terminal.yaojia.wang/recover        # → 201 {cert,caChain,notAfter}
+
+# a forged self-signed cert with a correct-looking SPIFFE SAN must still be refused
+# → 401   (this is the load-bearing check; nginx is no longer validating the chain here)
+```
--- a/docs/PROGRESS_LOG.md
+++ b/docs/PROGRESS_LOG.md
@@ -24,6 +24,32 @@

 > 新会话读到的第一块。保持准确,只描述"此刻"。

+### 🔥 [x] 隧道断了 8 天 —— 证书过期后**自动续期死锁**,已修 + 已上线(2026-07-29)
+
+- **现象**: 用户问"远端 relay 是否在运行"。服务端**全绿**(nginx/frps/cp-zte/panel-zte/xray/pg/redis 全 active,LE 通配证书有效至 2026-10-05,`*.terminal.yaojia.wang` 通配 DNS 正常),但 **frps `:7000` 已建立连接数 = 0**,`h7fd8.terminal.yaojia.wang` 打不通。Mac 上 frpc 在跑,每 20s 重试一次,全部 `connect to server error: EOF`(frps 在 mTLS 阶段拒)。
+- **根因(死锁)**: Mac 的 frp-client 叶证书 **2026-07-21 13:55 UTC 过期**(TTL 仅 ~24h)。`POST /renew` 是**用它自己要续的那张证书做 mTLS 鉴权**的 —— 证书一过期就再也换不了证书。触发链:续期窗口(T-8h)那天 Mac 在睡眠,醒来时 DNS 未就绪 → 24 次 `getaddrinfo ENOTFOUND enroll.terminal.yaojia.wang` 错过窗口 → 过期后改报 `client certificate has expired; renew before dialling`,**重复 6380 次、8 天、永不自愈**(agent.log 涨到 3MB)。
+- **三层都 fail-closed**,只改一层没用:① agent `transport/dial.ts` `buildTlsOptions` 抛 `CertExpiredError`,连 socket 都不开;② nginx `enroll.conf` 的 `location = /renew` 要求 `$ssl_client_verify = SUCCESS`;③ CP `api/renew.ts` `assertPresentedCertTrusted` 再查一次有效期 → 401。
+- **两个假设被实测推翻(重要,别再踩)**:
+  1. `ssl_verify_client optional` 在客户端证书**验证失败时直接返回裸 400**(`400 The SSL certificate error`),请求**根本到不了 location** —— 所以任何 `if ($ssl_client_verify …)` 都救不了。
+  2. `optional_no_ca` **也不行**:它只容忍**链**错误。见 nginx 的 `ngx_ssl_verify_error_optional()` —— 只含 `DEPTH_ZERO_SELF_SIGNED_CERT` / `SELF_SIGNED_CERT_IN_CHAIN` / `UNABLE_TO_GET_ISSUER_CERT_LOCALLY` / `UNABLE_TO_VERIFY_LEAF_SIGNATURE`,**不含 `X509_V_ERR_CERT_HAS_EXPIRED`**。已在 VPS 上用真过期证书打 :8472 实测确认。结论:**nginx 在任何模式下都不会转发过期客户端证书**。
+- **最终方案(用户选定:30 天宽限窗口 + 超窗回落配对码)**: 恢复路径**放弃 mTLS**,改为**不带客户端证书的普通 HTTPS POST,过期证书放进 body**。不丢鉴权 —— **CSR 是用同一把私钥自签的**,而签名网关本来就强制 CSR PoP(`ca/csr.ts` `verifyCsrPoP`)+ `CSR key == 注册 key`,持有私钥的证明与原来 TLS 握手给的完全等价。重放一张(公开的)证书而没有私钥,最多拿到一张**自己用不了**的证书。
+  - **agent**: `dial.ts` 与 mTLS renew 通道**保持严格 fail-closed**(放宽已从 TLS 层完全移除)。改由 rotator 逐次决策:有效 → mTLS `/renew`;过期但在宽限内 → 普通 `/recover`;超出宽限 → 终态 `onExhausted`,**一次告警后停止重试**(不再发任何请求),日志直接给出 `web-terminal-agent pair <CODE>`。新增 `recoverCert()`、`recoveryUrlFor()`、`DEFAULT_EXPIRED_RENEW_GRACE_MS`(30d)、`CertExpiredBeyondGraceError`;`AgentConfig` 加可选 `recoverUrl`(env `RECOVER_URL`)。
+  - **CP**: `/renew` **恢复严格**(grace 0,与终端器行为一致)。宽限只给新增的 `POST /recover` —— 证书从 **body** 读、**忽略 `x-client-cert` 头**,然后跑**与 `/renew` 完全相同**的信任链:frp-client-CA 锚点做真 X.509 路径验证 → SPIFFE 解析 → `notBefore`(**永不宽限**)→ registry `active` + 账户一致。**吊销照样生效**。
+  - **deploy**: 不需要新 vhost / 新 SNI / 新 DNS —— 只往现有 enroll vhost 加一个 `location = /recover`(+ `limit_req` 10r/m)。文档 `deploy/nginx/enroll-recover-location.md`(含 nginx 源码引证)。
+- **验证**:
+  - 单测:agent **289/289**、control-plane **290/290**,两侧 `tsc --noEmit` 干净。TDD 全程先 RED 后 GREEN。
+  - 承重测试:**「山寨 CA 签的、SPIFFE SAN 与真证书逐字节相同的伪造证书 → 401」**。这条最关键 —— 这条路径上 nginx 已不再验链,它一旦变红,`/recover` 就成了无鉴权发证机。已在**线上实测 401**。
+  - **线上端到端自愈实测**:把那张真·过期 8 天的证书塞回 keystore → 重启 agent → 日志 `frp-client cert rotated; restarting frpc onto the fresh leaf` → 证书自动换成新的 24h 叶(Jul30 08:06)→ frpc `login to server success` / `start proxy success` → 设备证书走完整链路 `:443 → SNI → :8470 mTLS → frps → frpc → Mac base app` 拿到 **HTTP 200**(返回本机真实 session 列表)→ 健康探针 `healthy: true`。**全程零人工介入** —— 正是生产卡死 8 天的那个场景。
+  - 回归:`/renew` 对过期证书仍 400(nginx 拦住,未放宽);4 区 SNI 探针(frp/通配/enroll/xray-Reality)全部未受影响。
+- **应急恢复(修复前先做的)**: 用 frp-client CA 给 h7fd8 补签了一张 7 天过桥证书(同一把私钥、同子域名、同 SPIFFE SAN,经 `openssl ca` 登记进 index.txt 因此可吊销),先把隧道救活。自愈生效后它已被换成 CP 正常签发的 24h 叶。
+- **遗留 / 待办**:
+  - **`agent/src/dist/buildBinary.ts` 是源码却被 `.gitignore` 的 `dist/` 规则吞掉,从未提交**(既有问题,与本次无关):任何全新 clone/worktree 都缺它 → `agent/src/index.ts` 类型检查失败、已提交的 `test/buildBinary.test.ts` 直接导入失败。建议加 `!agent/src/dist/` 例外。
+  - agent 日志里 `subdomain: (none)` / `host_id: (none)`:enroll 后未把 subdomain/hostId 落进配置,导致告警缺主机标识(既有问题)。
+  - 手机轨(device)目前**没有** `/recover` 对应路径,设备证书过期仍需重新 enroll;CP 侧宽限代码是通用的,补一条路由即可。
+  - 24h TTL + 单次定时续期 + 笔记本睡眠 的组合仍偏脆,宽限窗口是兜底而非根治。
+- **文件**: `agent/src/certs/rotation.ts` · `agent/src/certs/nativeRenew.ts` · `agent/src/config/agentConfig.ts` · `control-plane/src/api/renew.ts` · `deploy/nginx/enroll-recover-location.md`(新) · 各自测试。
+- **commit**: `f3f4d8b`(第一版,方向对但 nginx 那条路走不通)+ `5509c81`(实测后改为 body 传证书,最终版)。
+
 ### 🐛 [x] Mac 版中文全变 `_` + 框线渲染错乱 —— PTY 缺 UTF-8 locale(2026-07-28)

 - **现象**(用户截图,Mac 版):Claude Code TUI 里所有中文变成 `_`;banner/输入框只剩零散横线,圆角边框消失。