feat(agent): auto-renew the host cert in the native run-loop (A5)

Wire createCertRotator/renewCert into superviseNative so the frp-client cert renews silently at ~2/3 TTL over mTLS /renew and frpc restarts onto the fresh leaf; failures retry with backoff, never crash the supervisor. mTLS transport has a 15s timeout + 64KB response cap. 281 tests pass.
2026-07-18 13:32:05 +02:00
parent fff011bb7f
commit 9a5909f672
8 changed files with 815 additions and 6 deletions
--- a/agent/src/certs/nativeRenew.ts
+++ b/agent/src/certs/nativeRenew.ts
@@ -0,0 +1,240 @@
+/**
+ * Native-tunnel cert auto-renew wiring — TASK A5 (PLAN_ZERO_TOUCH_ROLLOUT).
+ *
+ * The native run-loop (`superviseNative`) used to only MONITOR the frp-client leaf's freshness; the
+ * leaf therefore expired at ~24h and the tunnel dropped until a manual re-pair. This module closes
+ * that gap by driving `createCertRotator`/`renewCert` (crypto is REUSED, never reimplemented):
+ *
+ *   - `createMtlsFetch` — the injected `fetchImpl` the rotator hands to `renewCert`. It POSTs /renew
+ *     over mTLS presenting the CURRENT keystore leaf (re-read on every call, so the first renewal
+ *     after a rotation already authenticates with the freshly issued leaf). mTLS IS the auth — no
+ *     token, `rejectUnauthorized` always true (INV4/INV14). The private key stays in-process.
+ *   - `wireAutoRenew` — routes the rotator callbacks: rotated → restart frpc onto the new leaf and
+ *     log (non-secret); revoked (403) → tear the tunnel down (INV12); error → log + let the rotator
+ *     retry with backoff. A failed renewal NEVER crashes the supervisor.
+ *   - `startNativeAutoRenew` — the builder `superviseNative` calls: loads the identity, builds the
+ *     mTLS fetch + rotator at ~2/3-TTL, and starts it. Returns null (auto-renew disabled) if the host
+ *     is not enrolled (no identity), rather than throwing into the run-loop.
+ */
+import { request as httpsRequest } from 'node:https'
+import type { AgentConfig } from '../config/agentConfig.js'
+import type { Keystore } from '../keys/keystore.js'
+import type { Logger } from '../log/logger.js'
+import type { TimerLike } from '../transport/seams.js'
+import { createBackoff } from '../transport/backoff.js'
+import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js'
+import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js'
+import { createCertRotator, type CertRotator } from './rotation.js'
+
+/** Non-secret message from an unknown thrown value (never serializes cert/key material). */
+function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err)
+}
+
+/**
+ * Socket-idle timeout for a /renew request. A stalled or overloaded control-plane (or a NAT that
+ * silently drops the connection after the TLS handshake) must NOT leave the renewal Promise pending
+ * forever — that would starve the rotator's backoff-retry loop and let the leaf silently expire. On
+ * timeout the request is destroyed and the rejection surfaces through the rotator's onError→backoff.
+ */
+export const RENEW_REQUEST_TIMEOUT_MS = 15_000
+/**
+ * Hard cap on the buffered /renew response body. The reply is a small `{cert,caChain}` JSON; anything
+ * beyond a few KB is malformed or hostile, so we destroy the stream and reject rather than buffer it.
+ */
+export const MAX_RENEW_RESPONSE_BYTES = 64 * 1024
+
+// --- mTLS fetch --------------------------------------------------------------------------------
+
+/** A single mTLS request the fetch shim delegates to (injectable so the shim is offline-testable). */
+export interface MtlsRequestInit {
+  readonly method: string
+  readonly headers: Record<string, string>
+  readonly body?: string
+}
+export interface MtlsResponse {
+  readonly status: number
+  readonly body: string
+}
+export type MtlsRequest = (
+  url: string,
+  tls: TlsClientOptions,
+  init: MtlsRequestInit,
+) => Promise<MtlsResponse>
+
+/** Default mTLS transport: a `node:https` POST presenting the client cert/key + pinned CA. */
+const defaultMtlsRequest: MtlsRequest = (url, tls, init) =>
+  new Promise<MtlsResponse>((resolve, reject) => {
+    const req = httpsRequest(
+      url,
+      {
+        method: init.method,
+        headers: init.headers,
+        cert: tls.cert,
+        key: tls.key,
+        ca: tls.ca,
+        rejectUnauthorized: tls.rejectUnauthorized, // always true (anti-MITM, INV14)
+      },
+      (res) => {
+        const chunks: Buffer[] = []
+        let total = 0
+        res.on('data', (c: Buffer) => {
+          total += c.length
+          if (total > MAX_RENEW_RESPONSE_BYTES) {
+            res.destroy() // MEDIUM: refuse an unbounded body — a renew reply is a few-KB JSON
+            reject(new Error(`renew response body exceeded ${MAX_RENEW_RESPONSE_BYTES} byte cap`))
+            return
+          }
+          chunks.push(c)
+        })
+        res.on('end', () =>
+          resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') }),
+        )
+        res.on('error', reject) // a mid-stream socket error must reject, not hang
+      },
+    )
+    // HIGH: bound the request so a peer that accepts the connection but never replies rejects (and the
+    // rotator re-enters backoff) instead of pending forever — destroy(err) emits 'error' → reject below.
+    req.setTimeout(RENEW_REQUEST_TIMEOUT_MS, () => {
+      req.destroy(new Error(`renew request timed out after ${RENEW_REQUEST_TIMEOUT_MS}ms`))
+    })
+    req.on('error', reject)
+    if (init.body !== undefined) req.write(init.body)
+    req.end()
+  })
+
+function toHeaderRecord(headers: RequestInit['headers']): Record<string, string> {
+  if (!headers) return {}
+  if (headers instanceof Headers) {
+    const out: Record<string, string> = {}
+    headers.forEach((v, k) => {
+      out[k] = v
+    })
+    return out
+  }
+  if (Array.isArray(headers)) return Object.fromEntries(headers)
+  return { ...(headers as Record<string, string>) }
+}
+
+/**
+ * Build the `fetch`-shaped shim `renewCert` uses. Each call re-reads the CURRENT keystore leaf via
+ * `buildTlsOptions` (which fail-fast throws NotEnrolled/CertExpired — the rotator then logs + retries
+ * with backoff, never crashing) and delegates to the mTLS transport, mapping the result to a real
+ * `Response` (so `res.ok`/`res.status`/`res.json()` behave exactly as `renewCert` expects).
+ */
+export function createMtlsFetch(
+  ks: Keystore,
+  opts: { request?: MtlsRequest; certParser?: CertParser } = {},
+): typeof fetch {
+  const request = opts.request ?? defaultMtlsRequest
+  const shim = async (input: Parameters<typeof fetch>[0], init?: RequestInit): Promise<Response> => {
+    const url = typeof input === 'string' ? input : input.toString()
+    const tls = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) })
+    const reqInit: MtlsRequestInit = {
+      method: init?.method ?? 'GET',
+      headers: toHeaderRecord(init?.headers),
+      ...(typeof init?.body === 'string' ? { body: init.body } : {}),
+    }
+    const { status, body } = await request(url, tls, reqInit)
+    return new Response(body, { status })
+  }
+  return shim as typeof fetch
+}
+
+// --- rotator wiring ----------------------------------------------------------------------------
+
+/** Non-secret identifiers logged alongside renew events (INV9). */
+export interface AutoRenewLogIds {
+  readonly subdomain: string | null
+  readonly hostId: string | null
+}
+
+/** The two run-loop effects the rotator drives. */
+export interface AutoRenewHooks {
+  /** Restart the supervised frpc so it re-reads the rotated cert (a leaf rotation only). */
+  restartChild(): void
+  /** Tear the tunnel down (host revoked ⇒ never reconnect, INV12). */
+  stop(): void
+}
+
+/** Handle for the wired auto-renew loop. */
+export interface AutoRenewController {
+  stop(): void
+}
+
+/**
+ * Wire a rotator's callbacks to the run-loop and start it. Rotated → restart frpc; revoked → stop;
+ * error → log (non-secret) and let the rotator retry with backoff. Returns a controller that stops
+ * the rotator's scheduled timer.
+ */
+export function wireAutoRenew(
+  rotator: CertRotator,
+  hooks: AutoRenewHooks,
+  logger: Logger,
+  ids: AutoRenewLogIds,
+): AutoRenewController {
+  const meta = { subdomain: ids.subdomain, hostId: ids.hostId }
+  rotator.onRotated(() => {
+    logger.log('info', 'frp-client cert rotated; restarting frpc onto the fresh leaf', meta)
+    hooks.restartChild()
+  })
+  rotator.onRevoked(() => {
+    logger.log('warn', 'frp-client cert renewal refused (host revoked); tearing down tunnel', meta)
+    hooks.stop()
+  })
+  rotator.onError((err) => {
+    logger.log('warn', 'frp-client cert renewal failed; will retry with backoff', {
+      ...meta,
+      error: errorMessage(err),
+    })
+  })
+  rotator.start()
+  return { stop: () => rotator.stop() }
+}
+
+// --- builder -----------------------------------------------------------------------------------
+
+/** Injection seams for `startNativeAutoRenew` (all optional; unset ⇒ real transport/timers). */
+export interface NativeAutoRenewOpts {
+  readonly mtlsRequest?: MtlsRequest
+  readonly certParser?: CertParser
+  readonly timer?: TimerLike
+  readonly renewBeforeMs?: number
+  readonly retryBaseMs?: number
+  readonly now?: () => Date
+  readonly parseCert?: (pem: string) => Date
+}
+
+/**
+ * Build + start native cert auto-renew for `superviseNative`. Renews at ~2/3 of the leaf TTL
+ * (default `DEFAULT_CERT_RENEW_WINDOW_MS`, the same window the health probe alarms on). Returns null
+ * (auto-renew disabled, logged) when the host has no identity — an unenrolled run-loop must not throw.
+ */
+export function startNativeAutoRenew(
+  cfg: AgentConfig,
+  ks: Keystore,
+  hooks: AutoRenewHooks,
+  logger: Logger,
+  opts: NativeAutoRenewOpts = {},
+): AutoRenewController | null {
+  const id = ks.loadIdentity()
+  if (id === null) {
+    logger.log('warn', 'no identity in keystore — cert auto-renew disabled', {})
+    return null
+  }
+  const fetchImpl = createMtlsFetch(ks, {
+    ...(opts.mtlsRequest ? { request: opts.mtlsRequest } : {}),
+    ...(opts.certParser ? { certParser: opts.certParser } : {}),
+  })
+  const rotator = createCertRotator(cfg, id, ks, {
+    fetchImpl,
+    renewBeforeMs: opts.renewBeforeMs ?? DEFAULT_CERT_RENEW_WINDOW_MS,
+    ...(opts.timer ? { timer: opts.timer } : {}),
+    ...(opts.now ? { now: opts.now } : {}),
+    ...(opts.parseCert ? { parseCert: opts.parseCert } : {}),
+    ...(opts.retryBaseMs !== undefined
+      ? { retryBackoff: createBackoff({ baseMs: opts.retryBaseMs, jitter: false }) }
+      : {}),
+  })
+  return wireAutoRenew(rotator, hooks, logger, { subdomain: cfg.subdomain, hostId: cfg.hostId })
+}
--- a/agent/src/certs/rotation.ts
+++ b/agent/src/certs/rotation.ts
@@ -13,6 +13,7 @@ import type { AgentConfig } from '../config/agentConfig.js'
 import type { AgentIdentity } from '../keys/identity.js'
 import type { Keystore } from '../keys/keystore.js'
 import type { TimerLike } from '../transport/seams.js'
+import { createBackoff, type BackoffPolicy } from '../transport/backoff.js'
 import { buildCsr } from '../enroll/csr.js'

 export const DEFAULT_RENEW_BEFORE_MS = 5 * 60_000 // renew 5 min before expiry
@@ -22,6 +23,8 @@ export interface CertRotator {
  stop(): void
  onRotated(cb: () => void): void
  onRevoked(cb: () => void): void
+  /** A renewal attempt failed (network/HTTP, NOT a 403 revoke). The rotator retries with backoff. */
+  onError(cb: (err: unknown) => void): void
 }

 export type RenewOutcome = 'rotated' | 'revoked'
@@ -79,6 +82,8 @@ export function createCertRotator(
    fetchImpl?: typeof fetch
    now?: () => Date
    parseCert?: (pem: string) => Date
+    /** Backoff policy for retrying a FAILED renewal (default 1s→30s). Reset after a success. */
+    retryBackoff?: BackoffPolicy
  } = {},
 ): CertRotator {
  const renewBeforeMs = opts.renewBeforeMs ?? DEFAULT_RENEW_BEFORE_MS
@@ -91,9 +96,11 @@ export function createCertRotator(
  }
  const doFetch = opts.fetchImpl ?? fetch
  const now = opts.now ?? (() => new Date())
+  const retryBackoff = opts.retryBackoff ?? createBackoff({ jitter: true })
  let handle: unknown = null
  let rotatedCb: (() => void) | null = null
  let revokedCb: (() => void) | null = null
+  let errorCb: ((err: unknown) => void) | null = null

  function schedule(): void {
    const certs = ks.loadCert()
@@ -109,12 +116,16 @@ export function createCertRotator(
          revokedCb?.()
          return
        }
+        retryBackoff.reset() // a healthy renewal clears the retry backoff for the next cycle
        rotatedCb?.()
        schedule()
      })
-      .catch(() => {
-        // network error: retry after renewBeforeMs; the tunnel stays up meanwhile.
-        handle = timer.setTimeout(runRenewal, renewBeforeMs)
+      .catch((err: unknown) => {
+        // Network/HTTP failure (never a 403 revoke): surface it (caller logs, no secret) and retry
+        // with backoff. The cert is still valid until expiry, so the tunnel stays up meanwhile — a
+        // failed renewal must NEVER tear the supervisor down.
+        errorCb?.(err)
+        handle = timer.setTimeout(runRenewal, retryBackoff.nextDelayMs())
      })
  }

@@ -132,5 +143,8 @@ export function createCertRotator(
    onRevoked(cb): void {
      revokedCb = cb
    },
+    onError(cb): void {
+      errorCb = cb
+    },
  }
 }
--- a/agent/src/cli/deps.ts
+++ b/agent/src/cli/deps.ts
@@ -23,6 +23,7 @@ import { runTunnel } from '../transport/runTunnel.js'
 import { buildNativeFrpcToml } from '../transport/frpcToml.js'
 import { superviseFrpc } from '../transport/frpSupervise.js'
 import { provisionFrpc } from '../provision/frpcBinary.js'
+import { startNativeAutoRenew } from '../certs/nativeRenew.js'
 import {
  probeLoopbackBaseApp,
  renderHealthStatus,
@@ -156,9 +157,12 @@ export function readFrpcLog(stateDir: string): string {
 }

 /**
- * Native run-loop (B4/H4): supervise the pinned frpc child with restart-on-exit backoff while a
+ * Native run-loop (B4/H4 + A5): supervise the pinned frpc child with restart-on-exit backoff while a
 * periodic health probe (frpc alive, base-app loopback reachable, proxy-started, cert-not-expiring)
- * logs NON-SECRET status only (INV9). Resolves when the supervisor stops (SIGTERM/SIGINT).
+ * logs NON-SECRET status only (INV9), AND auto-renew the frp-client leaf at ~2/3 TTL so the tunnel
+ * never drops on cert expiry (A5): a successful renewal restarts frpc onto the fresh leaf, a 403
+ * revoke tears the tunnel down, and a failed renewal retries with backoff without crashing the
+ * supervisor. Resolves when the supervisor stops (SIGTERM/SIGINT).
 */
 function superviseNative(cfg: AgentConfig, ks: Keystore): Promise<number> {
  const logger = createLogger('info')
@@ -184,12 +188,28 @@ function superviseNative(cfg: AgentConfig, ks: Keystore): Promise<number> {
      for (const line of renderHealthStatus(ids, report)) logger.log('info', line)
    },
  )
+  // A5: silently renew the leaf before it expires. Restart frpc onto the fresh cert on rotation;
+  // stop the whole supervisor on a 403 revoke (INV12). Null ⇒ unenrolled (auto-renew disabled).
+  const autoRenew = startNativeAutoRenew(
+    cfg,
+    ks,
+    {
+      restartChild: () => handle.restartChild(),
+      stop: () => {
+        void handle.stop()
+      },
+    },
+    logger,
+  )
  const onSignal = (): void => {
    void handle.stop()
  }
  process.once('SIGTERM', onSignal)
  process.once('SIGINT', onSignal)
-  return handle.done.finally(() => monitor.stop())
+  return handle.done.finally(() => {
+    monitor.stop()
+    autoRenew?.stop()
+  })
 }

 /** Build the concrete CliDeps used by the real CLI entrypoint. */
--- a/agent/src/transport/frpSupervise.ts
+++ b/agent/src/transport/frpSupervise.ts
@@ -62,6 +62,12 @@ export interface FrpSuperviseHandle {
  readonly done: Promise<number>
  /** True while a frpc child is currently running (for the health probe). */
  isChildAlive(): boolean
+  /**
+   * Kill the current child WITHOUT stopping the supervisor (A5): the restart-on-exit loop respawns a
+   * fresh frpc that re-reads the (now rotated) cert/key/CA files. A no-op once `stop()` was called —
+   * it must never resurrect a supervisor that is shutting down.
+   */
+  restartChild(): void
 }

 /** A frpc run lasting at least this long is "stable" ⇒ reset the restart backoff. */
@@ -196,5 +202,8 @@ export function superviseFrpc(
    },
    done,
    isChildAlive: () => child?.isAlive() ?? false,
+    restartChild(): void {
+      if (!stopped) child?.kill()
+    },
  }
 }