fix(tunnel): recover an expired leaf over plain HTTPS, not mTLS

Correction to the previous commit. Its recovery vhost could not work: nginx
will not forward an expired client certificate in ANY `ssl_verify_client` mode.
`optional` answers a bare `400 The SSL certificate error` before any location
runs, and `optional_no_ca` only tolerates CHAIN failures — see nginx's
`ngx_ssl_verify_error_optional()`, which covers self-signed / unknown-issuer /
unverifiable-leaf and NOT `X509_V_ERR_CERT_HAS_EXPIRED`. Verified live against
the deployed :8472 server, which rejected the real expired leaf.

So recovery drops mTLS instead of trying to bend it:

- agent: `buildTlsOptions` and the mTLS renew transport go back to being
  strictly fail-closed on expiry — the relaxation is gone from the TLS layer
  entirely. The rotator now decides per attempt: valid → mTLS `/renew`,
  expired-inside-grace → plain `/recover`, expired-beyond-grace → terminal
  `onExhausted` with no request issued at all.
- new `recoverCert` POSTs `{cert, csr}` with no client certificate. Possession
  of the private key is still proven: the CSR is self-signed by it and the host
  signer already enforces CSR PoP plus `CSR key == registered key`, so a
  replayed (public) cert without the key yields at most a certificate the
  attacker cannot authenticate with.
- control-plane: `/renew` is strict again (grace 0, matching the terminator).
  The grace lives on the new `POST /recover`, which reads the cert from the BODY
  and ignores the `x-client-cert` header, then runs the identical trust
  pipeline: X.509 path validation to the frp-client-CA anchors, SPIFFE parse,
  `notBefore` (never graced), registry `active` + account match. Revocation
  still bites.
- deploy: no new vhost, no new SNI, no DNS. One `location = /recover` merged
  into the existing enroll vhost, documented in
  `deploy/nginx/enroll-recover-location.md` with the nginx source citation.

The load-bearing new test is "a self-signed cert with a FORGED SPIFFE SAN is
refused → 401": nginx no longer validates the chain on this path, so that
assertion is what keeps `/recover` from being a cert vending machine.

Verified: agent 289/289, control-plane 290/290, tsc clean on both.

control-plane/src/api/renew.ts

@@ -51,6 +51,8 @@ export const DEFAULT_RENEW_RATE_WINDOW_MS = 60 * 60 * 1000
 export const RENEW_RATE_MAX_IDENTITIES = 10_000
 /** Max base64 length of a submitted CSR (a P-256 PKCS#10 is well under 1 KB; this is generous slack). */
 export const MAX_CSR_B64_LEN = 8192
+/** Max wire length of a body-carried certificate (a P-256 leaf PEM is ~1 KB; generous slack). */
+export const MAX_PRESENTED_CERT_LEN = 16384
 /**
  * Default header the mTLS terminator forwards the verified client cert in (base64 DER). The terminator
  * MUST set this from `$ssl_client_cert` AND strip any client-supplied copy — a client can never provide
@@ -185,23 +187,29 @@ export function headerPresentedCert(headerName: string = DEFAULT_CLIENT_CERT_HEA
       const raw = req.headers[name]
       const value = Array.isArray(raw) ? raw[0] : raw
       if (typeof value !== 'string' || value.length === 0) return null
-      try {
-        // The terminator may forward the verified cert as base64 DER, OR as PEM — including nginx's
-        // header-safe `$ssl_client_escaped_cert` (URL-encoded PEM). Normalize all three to raw DER:
-        // URL-decode if escaped, then strip PEM armor + whitespace to recover the base64 DER body.
-        let s = value.includes('%') ? safeDecodeUri(value) : value
-        if (s.includes('BEGIN CERTIFICATE')) {
-          s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '')
-        }
-        const der = Buffer.from(s, 'base64')
-        return der.length > 0 ? new Uint8Array(der) : null
-      } catch {
-        return null
-      }
+      return certWireToDer(value)
     },
   }
 }
 
+/**
+ * Normalize a certificate carried over the wire to raw DER. Accepts base64 DER, PEM, and nginx's
+ * header-safe `$ssl_client_escaped_cert` (URL-encoded PEM): URL-decode if escaped, then strip PEM
+ * armor + whitespace to recover the base64 body. Returns null on anything unparseable.
+ */
+export function certWireToDer(value: string): Uint8Array | null {
+  try {
+    let s = value.includes('%') ? safeDecodeUri(value) : value
+    if (s.includes('BEGIN CERTIFICATE')) {
+      s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '')
+    }
+    const der = Buffer.from(s, 'base64')
+    return der.length > 0 ? new Uint8Array(der) : null
+  } catch {
+    return null
+  }
+}
+
 /** URL-decode `value`, returning it unchanged if it is not valid percent-encoding. */
 function safeDecodeUri(value: string): string {
   try {
@@ -286,6 +294,13 @@ function parsePresentedCertIdentity(
 }
 
 const RenewBodySchema = z.object({ csr: z.string().min(1).max(MAX_CSR_B64_LEN) }).strict()
+/** `/recover` additionally carries the EXPIRED leaf itself (base64 DER or PEM). */
+const RecoverBodySchema = z
+  .object({
+    cert: z.string().min(1).max(MAX_PRESENTED_CERT_LEN),
+    csr: z.string().min(1).max(MAX_CSR_B64_LEN),
+  })
+  .strict()
 
 export interface RenewDeps {
   readonly hosts: HostRegistry
@@ -327,7 +342,9 @@ function sendError(reply: FastifyReply, err: unknown): void {
 export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
   const presentedCert = deps.presentedCert ?? headerPresentedCert()
   const rateLimiter = deps.rateLimiter ?? createRenewRateLimiter()
-  const expiredGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS
+  // `/renew` stays STRICT (0): the mTLS terminator would never forward an expired cert to it anyway.
+  // The grace belongs to `/recover`, the route built for exactly that case.
+  const recoverGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS
 
   return async (app) => {
     app.post('/renew', async (req, reply) => {
@@ -338,7 +355,7 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
           der,
           'host',
           deps.hostCaAnchorsDer
-            ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs }
+            ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
             : undefined,
         )
         rateLimiter.check(`host:${identity.accountId}:${identity.id}`)
@@ -365,6 +382,57 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
       }
     })
 
+    /**
+     * Expired-leaf recovery. The lapsed cert arrives in the BODY because no TLS terminator will
+     * forward an expired client certificate (nginx `optional` → bare 400; `optional_no_ca` tolerates
+     * only chain errors, never `X509_V_ERR_CERT_HAS_EXPIRED`). That makes this route the SOLE
+     * verifier, so it runs the identical trust pipeline as `/renew` — real X.509 path validation to
+     * the frp-client-CA anchors, SPIFFE parse, `notBefore`, registry `active` + account match — and
+     * differs ONLY in allowing a bounded overrun on `notAfter`.
+     *
+     * Possession of the private key is still proven: the CSR is self-signed by it, and the host
+     * signer's delegated gate enforces CSR PoP plus `CSR key == registered key`. A replayed cert
+     * without the key therefore yields, at most, a certificate the attacker cannot authenticate with.
+     * The header channel is deliberately IGNORED here — on this route only the body speaks.
+     */
+    app.post('/recover', async (req, reply) => {
+      try {
+        const body = RecoverBodySchema.parse(req.body)
+        const der = certWireToDer(body.cert)
+        if (der === null) throw new RenewRejectError(401)
+        const identity = parsePresentedCertIdentity(
+          der,
+          'host',
+          deps.hostCaAnchorsDer
+            ? {
+                caAnchorsDer: deps.hostCaAnchorsDer,
+                nowMs: Date.now(),
+                expiredGraceMs: recoverGraceMs,
+              }
+            : undefined,
+        )
+        rateLimiter.check(`recover:${identity.accountId}:${identity.id}`)
+
+        const host = await deps.hosts.getHostBySubdomain(identity.id)
+        if (host === null || host.status === 'revoked' || host.accountId !== identity.accountId) {
+          throw new RenewRejectError(403)
+        }
+
+        const leaf = await deps.renewer.renewHostLeaf(
+          host.hostId,
+          identity.publicKeySpki,
+          decodeCsrWire(body.csr),
+        )
+        await reply.code(201).send({
+          cert: bytesToBase64(leaf.cert),
+          caChain: leaf.caChain.map((c) => bytesToBase64(c)),
+          notAfter: leaf.notAfter.toISOString(),
+        })
+      } catch (err) {
+        sendError(reply, err)
+      }
+    })
+
     app.post('/device/:id/renew', async (req, reply) => {
       try {
         const der = presentedCert.presentedCertDer(req)
@@ -373,7 +441,7 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
           der,
           'device',
           deps.deviceCaAnchorsDer
-            ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs }
+            ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
             : undefined,
         )
         const id = (req.params as { id: string }).id