From 9a5909f67299b36fb3402dd0353d8b0ead3916fd Mon Sep 17 00:00:00 2001 From: Yaojia Wang Date: Sat, 18 Jul 2026 13:32:05 +0200 Subject: [PATCH] feat(agent): auto-renew the host cert in the native run-loop (A5) Wire createCertRotator/renewCert into superviseNative so the frp-client cert renews silently at ~2/3 TTL over mTLS /renew and frpc restarts onto the fresh leaf; failures retry with backoff, never crash the supervisor. mTLS transport has a 15s timeout + 64KB response cap. 281 tests pass. --- agent/src/certs/nativeRenew.ts | 240 ++++++++++++++++++++ agent/src/certs/rotation.ts | 20 +- agent/src/cli/deps.ts | 26 ++- agent/src/transport/frpSupervise.ts | 9 + agent/test/frpSupervise.test.ts | 40 ++++ agent/test/nativeRenew.test.ts | 277 ++++++++++++++++++++++++ agent/test/nativeRenewTransport.test.ts | 168 ++++++++++++++ agent/test/rotation.test.ts | 41 ++++ 8 files changed, 815 insertions(+), 6 deletions(-) create mode 100644 agent/src/certs/nativeRenew.ts create mode 100644 agent/test/nativeRenew.test.ts create mode 100644 agent/test/nativeRenewTransport.test.ts diff --git a/agent/src/certs/nativeRenew.ts b/agent/src/certs/nativeRenew.ts new file mode 100644 index 0000000..34e5910 --- /dev/null +++ b/agent/src/certs/nativeRenew.ts @@ -0,0 +1,240 @@ +/** + * Native-tunnel cert auto-renew wiring — TASK A5 (PLAN_ZERO_TOUCH_ROLLOUT). + * + * The native run-loop (`superviseNative`) used to only MONITOR the frp-client leaf's freshness; the + * leaf therefore expired at ~24h and the tunnel dropped until a manual re-pair. This module closes + * that gap by driving `createCertRotator`/`renewCert` (crypto is REUSED, never reimplemented): + * + * - `createMtlsFetch` — the injected `fetchImpl` the rotator hands to `renewCert`. It POSTs /renew + * over mTLS presenting the CURRENT keystore leaf (re-read on every call, so the first renewal + * after a rotation already authenticates with the freshly issued leaf). mTLS IS the auth — no + * token, `rejectUnauthorized` always true (INV4/INV14). The private key stays in-process. + * - `wireAutoRenew` — routes the rotator callbacks: rotated → restart frpc onto the new leaf and + * log (non-secret); revoked (403) → tear the tunnel down (INV12); error → log + let the rotator + * retry with backoff. A failed renewal NEVER crashes the supervisor. + * - `startNativeAutoRenew` — the builder `superviseNative` calls: loads the identity, builds the + * mTLS fetch + rotator at ~2/3-TTL, and starts it. Returns null (auto-renew disabled) if the host + * is not enrolled (no identity), rather than throwing into the run-loop. + */ +import { request as httpsRequest } from 'node:https' +import type { AgentConfig } from '../config/agentConfig.js' +import type { Keystore } from '../keys/keystore.js' +import type { Logger } from '../log/logger.js' +import type { TimerLike } from '../transport/seams.js' +import { createBackoff } from '../transport/backoff.js' +import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js' +import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js' +import { createCertRotator, type CertRotator } from './rotation.js' + +/** Non-secret message from an unknown thrown value (never serializes cert/key material). */ +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +/** + * Socket-idle timeout for a /renew request. A stalled or overloaded control-plane (or a NAT that + * silently drops the connection after the TLS handshake) must NOT leave the renewal Promise pending + * forever — that would starve the rotator's backoff-retry loop and let the leaf silently expire. On + * timeout the request is destroyed and the rejection surfaces through the rotator's onError→backoff. + */ +export const RENEW_REQUEST_TIMEOUT_MS = 15_000 +/** + * Hard cap on the buffered /renew response body. The reply is a small `{cert,caChain}` JSON; anything + * beyond a few KB is malformed or hostile, so we destroy the stream and reject rather than buffer it. + */ +export const MAX_RENEW_RESPONSE_BYTES = 64 * 1024 + +// --- mTLS fetch -------------------------------------------------------------------------------- + +/** A single mTLS request the fetch shim delegates to (injectable so the shim is offline-testable). */ +export interface MtlsRequestInit { + readonly method: string + readonly headers: Record + readonly body?: string +} +export interface MtlsResponse { + readonly status: number + readonly body: string +} +export type MtlsRequest = ( + url: string, + tls: TlsClientOptions, + init: MtlsRequestInit, +) => Promise + +/** Default mTLS transport: a `node:https` POST presenting the client cert/key + pinned CA. */ +const defaultMtlsRequest: MtlsRequest = (url, tls, init) => + new Promise((resolve, reject) => { + const req = httpsRequest( + url, + { + method: init.method, + headers: init.headers, + cert: tls.cert, + key: tls.key, + ca: tls.ca, + rejectUnauthorized: tls.rejectUnauthorized, // always true (anti-MITM, INV14) + }, + (res) => { + const chunks: Buffer[] = [] + let total = 0 + res.on('data', (c: Buffer) => { + total += c.length + if (total > MAX_RENEW_RESPONSE_BYTES) { + res.destroy() // MEDIUM: refuse an unbounded body — a renew reply is a few-KB JSON + reject(new Error(`renew response body exceeded ${MAX_RENEW_RESPONSE_BYTES} byte cap`)) + return + } + chunks.push(c) + }) + res.on('end', () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') }), + ) + res.on('error', reject) // a mid-stream socket error must reject, not hang + }, + ) + // HIGH: bound the request so a peer that accepts the connection but never replies rejects (and the + // rotator re-enters backoff) instead of pending forever — destroy(err) emits 'error' → reject below. + req.setTimeout(RENEW_REQUEST_TIMEOUT_MS, () => { + req.destroy(new Error(`renew request timed out after ${RENEW_REQUEST_TIMEOUT_MS}ms`)) + }) + req.on('error', reject) + if (init.body !== undefined) req.write(init.body) + req.end() + }) + +function toHeaderRecord(headers: RequestInit['headers']): Record { + if (!headers) return {} + if (headers instanceof Headers) { + const out: Record = {} + headers.forEach((v, k) => { + out[k] = v + }) + return out + } + if (Array.isArray(headers)) return Object.fromEntries(headers) + return { ...(headers as Record) } +} + +/** + * Build the `fetch`-shaped shim `renewCert` uses. Each call re-reads the CURRENT keystore leaf via + * `buildTlsOptions` (which fail-fast throws NotEnrolled/CertExpired — the rotator then logs + retries + * with backoff, never crashing) and delegates to the mTLS transport, mapping the result to a real + * `Response` (so `res.ok`/`res.status`/`res.json()` behave exactly as `renewCert` expects). + */ +export function createMtlsFetch( + ks: Keystore, + opts: { request?: MtlsRequest; certParser?: CertParser } = {}, +): typeof fetch { + const request = opts.request ?? defaultMtlsRequest + const shim = async (input: Parameters[0], init?: RequestInit): Promise => { + const url = typeof input === 'string' ? input : input.toString() + const tls = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) }) + const reqInit: MtlsRequestInit = { + method: init?.method ?? 'GET', + headers: toHeaderRecord(init?.headers), + ...(typeof init?.body === 'string' ? { body: init.body } : {}), + } + const { status, body } = await request(url, tls, reqInit) + return new Response(body, { status }) + } + return shim as typeof fetch +} + +// --- rotator wiring ---------------------------------------------------------------------------- + +/** Non-secret identifiers logged alongside renew events (INV9). */ +export interface AutoRenewLogIds { + readonly subdomain: string | null + readonly hostId: string | null +} + +/** The two run-loop effects the rotator drives. */ +export interface AutoRenewHooks { + /** Restart the supervised frpc so it re-reads the rotated cert (a leaf rotation only). */ + restartChild(): void + /** Tear the tunnel down (host revoked ⇒ never reconnect, INV12). */ + stop(): void +} + +/** Handle for the wired auto-renew loop. */ +export interface AutoRenewController { + stop(): void +} + +/** + * Wire a rotator's callbacks to the run-loop and start it. Rotated → restart frpc; revoked → stop; + * error → log (non-secret) and let the rotator retry with backoff. Returns a controller that stops + * the rotator's scheduled timer. + */ +export function wireAutoRenew( + rotator: CertRotator, + hooks: AutoRenewHooks, + logger: Logger, + ids: AutoRenewLogIds, +): AutoRenewController { + const meta = { subdomain: ids.subdomain, hostId: ids.hostId } + rotator.onRotated(() => { + logger.log('info', 'frp-client cert rotated; restarting frpc onto the fresh leaf', meta) + hooks.restartChild() + }) + rotator.onRevoked(() => { + logger.log('warn', 'frp-client cert renewal refused (host revoked); tearing down tunnel', meta) + hooks.stop() + }) + rotator.onError((err) => { + logger.log('warn', 'frp-client cert renewal failed; will retry with backoff', { + ...meta, + error: errorMessage(err), + }) + }) + rotator.start() + return { stop: () => rotator.stop() } +} + +// --- builder ----------------------------------------------------------------------------------- + +/** Injection seams for `startNativeAutoRenew` (all optional; unset ⇒ real transport/timers). */ +export interface NativeAutoRenewOpts { + readonly mtlsRequest?: MtlsRequest + readonly certParser?: CertParser + readonly timer?: TimerLike + readonly renewBeforeMs?: number + readonly retryBaseMs?: number + readonly now?: () => Date + readonly parseCert?: (pem: string) => Date +} + +/** + * Build + start native cert auto-renew for `superviseNative`. Renews at ~2/3 of the leaf TTL + * (default `DEFAULT_CERT_RENEW_WINDOW_MS`, the same window the health probe alarms on). Returns null + * (auto-renew disabled, logged) when the host has no identity — an unenrolled run-loop must not throw. + */ +export function startNativeAutoRenew( + cfg: AgentConfig, + ks: Keystore, + hooks: AutoRenewHooks, + logger: Logger, + opts: NativeAutoRenewOpts = {}, +): AutoRenewController | null { + const id = ks.loadIdentity() + if (id === null) { + logger.log('warn', 'no identity in keystore — cert auto-renew disabled', {}) + return null + } + const fetchImpl = createMtlsFetch(ks, { + ...(opts.mtlsRequest ? { request: opts.mtlsRequest } : {}), + ...(opts.certParser ? { certParser: opts.certParser } : {}), + }) + const rotator = createCertRotator(cfg, id, ks, { + fetchImpl, + renewBeforeMs: opts.renewBeforeMs ?? DEFAULT_CERT_RENEW_WINDOW_MS, + ...(opts.timer ? { timer: opts.timer } : {}), + ...(opts.now ? { now: opts.now } : {}), + ...(opts.parseCert ? { parseCert: opts.parseCert } : {}), + ...(opts.retryBaseMs !== undefined + ? { retryBackoff: createBackoff({ baseMs: opts.retryBaseMs, jitter: false }) } + : {}), + }) + return wireAutoRenew(rotator, hooks, logger, { subdomain: cfg.subdomain, hostId: cfg.hostId }) +} diff --git a/agent/src/certs/rotation.ts b/agent/src/certs/rotation.ts index f243af5..6d7b345 100644 --- a/agent/src/certs/rotation.ts +++ b/agent/src/certs/rotation.ts @@ -13,6 +13,7 @@ import type { AgentConfig } from '../config/agentConfig.js' import type { AgentIdentity } from '../keys/identity.js' import type { Keystore } from '../keys/keystore.js' import type { TimerLike } from '../transport/seams.js' +import { createBackoff, type BackoffPolicy } from '../transport/backoff.js' import { buildCsr } from '../enroll/csr.js' export const DEFAULT_RENEW_BEFORE_MS = 5 * 60_000 // renew 5 min before expiry @@ -22,6 +23,8 @@ export interface CertRotator { stop(): void onRotated(cb: () => void): void onRevoked(cb: () => void): void + /** A renewal attempt failed (network/HTTP, NOT a 403 revoke). The rotator retries with backoff. */ + onError(cb: (err: unknown) => void): void } export type RenewOutcome = 'rotated' | 'revoked' @@ -79,6 +82,8 @@ export function createCertRotator( fetchImpl?: typeof fetch now?: () => Date parseCert?: (pem: string) => Date + /** Backoff policy for retrying a FAILED renewal (default 1s→30s). Reset after a success. */ + retryBackoff?: BackoffPolicy } = {}, ): CertRotator { const renewBeforeMs = opts.renewBeforeMs ?? DEFAULT_RENEW_BEFORE_MS @@ -91,9 +96,11 @@ export function createCertRotator( } const doFetch = opts.fetchImpl ?? fetch const now = opts.now ?? (() => new Date()) + const retryBackoff = opts.retryBackoff ?? createBackoff({ jitter: true }) let handle: unknown = null let rotatedCb: (() => void) | null = null let revokedCb: (() => void) | null = null + let errorCb: ((err: unknown) => void) | null = null function schedule(): void { const certs = ks.loadCert() @@ -109,12 +116,16 @@ export function createCertRotator( revokedCb?.() return } + retryBackoff.reset() // a healthy renewal clears the retry backoff for the next cycle rotatedCb?.() schedule() }) - .catch(() => { - // network error: retry after renewBeforeMs; the tunnel stays up meanwhile. - handle = timer.setTimeout(runRenewal, renewBeforeMs) + .catch((err: unknown) => { + // Network/HTTP failure (never a 403 revoke): surface it (caller logs, no secret) and retry + // with backoff. The cert is still valid until expiry, so the tunnel stays up meanwhile — a + // failed renewal must NEVER tear the supervisor down. + errorCb?.(err) + handle = timer.setTimeout(runRenewal, retryBackoff.nextDelayMs()) }) } @@ -132,5 +143,8 @@ export function createCertRotator( onRevoked(cb): void { revokedCb = cb }, + onError(cb): void { + errorCb = cb + }, } } diff --git a/agent/src/cli/deps.ts b/agent/src/cli/deps.ts index 6696e74..05b7c86 100644 --- a/agent/src/cli/deps.ts +++ b/agent/src/cli/deps.ts @@ -23,6 +23,7 @@ import { runTunnel } from '../transport/runTunnel.js' import { buildNativeFrpcToml } from '../transport/frpcToml.js' import { superviseFrpc } from '../transport/frpSupervise.js' import { provisionFrpc } from '../provision/frpcBinary.js' +import { startNativeAutoRenew } from '../certs/nativeRenew.js' import { probeLoopbackBaseApp, renderHealthStatus, @@ -156,9 +157,12 @@ export function readFrpcLog(stateDir: string): string { } /** - * Native run-loop (B4/H4): supervise the pinned frpc child with restart-on-exit backoff while a + * Native run-loop (B4/H4 + A5): supervise the pinned frpc child with restart-on-exit backoff while a * periodic health probe (frpc alive, base-app loopback reachable, proxy-started, cert-not-expiring) - * logs NON-SECRET status only (INV9). Resolves when the supervisor stops (SIGTERM/SIGINT). + * logs NON-SECRET status only (INV9), AND auto-renew the frp-client leaf at ~2/3 TTL so the tunnel + * never drops on cert expiry (A5): a successful renewal restarts frpc onto the fresh leaf, a 403 + * revoke tears the tunnel down, and a failed renewal retries with backoff without crashing the + * supervisor. Resolves when the supervisor stops (SIGTERM/SIGINT). */ function superviseNative(cfg: AgentConfig, ks: Keystore): Promise { const logger = createLogger('info') @@ -184,12 +188,28 @@ function superviseNative(cfg: AgentConfig, ks: Keystore): Promise { for (const line of renderHealthStatus(ids, report)) logger.log('info', line) }, ) + // A5: silently renew the leaf before it expires. Restart frpc onto the fresh cert on rotation; + // stop the whole supervisor on a 403 revoke (INV12). Null ⇒ unenrolled (auto-renew disabled). + const autoRenew = startNativeAutoRenew( + cfg, + ks, + { + restartChild: () => handle.restartChild(), + stop: () => { + void handle.stop() + }, + }, + logger, + ) const onSignal = (): void => { void handle.stop() } process.once('SIGTERM', onSignal) process.once('SIGINT', onSignal) - return handle.done.finally(() => monitor.stop()) + return handle.done.finally(() => { + monitor.stop() + autoRenew?.stop() + }) } /** Build the concrete CliDeps used by the real CLI entrypoint. */ diff --git a/agent/src/transport/frpSupervise.ts b/agent/src/transport/frpSupervise.ts index 6ab4667..cddd559 100644 --- a/agent/src/transport/frpSupervise.ts +++ b/agent/src/transport/frpSupervise.ts @@ -62,6 +62,12 @@ export interface FrpSuperviseHandle { readonly done: Promise /** True while a frpc child is currently running (for the health probe). */ isChildAlive(): boolean + /** + * Kill the current child WITHOUT stopping the supervisor (A5): the restart-on-exit loop respawns a + * fresh frpc that re-reads the (now rotated) cert/key/CA files. A no-op once `stop()` was called — + * it must never resurrect a supervisor that is shutting down. + */ + restartChild(): void } /** A frpc run lasting at least this long is "stable" ⇒ reset the restart backoff. */ @@ -196,5 +202,8 @@ export function superviseFrpc( }, done, isChildAlive: () => child?.isAlive() ?? false, + restartChild(): void { + if (!stopped) child?.kill() + }, } } diff --git a/agent/test/frpSupervise.test.ts b/agent/test/frpSupervise.test.ts index e19d0ce..7bc8268 100644 --- a/agent/test/frpSupervise.test.ts +++ b/agent/test/frpSupervise.test.ts @@ -145,4 +145,44 @@ describe('superviseFrpc (B4/H4 — restart-on-exit backoff)', () => { await stopping expect(spawn).toHaveBeenCalledTimes(1) }) + + it('restartChild() kills the running child so the loop respawns onto the fresh cert files', async () => { + const flush = (): Promise => new Promise((r) => setImmediate(r)) + const children = [makeChild(), makeChild()] + let n = 0 + const spawn: SpawnFrpc = () => children[n++]!.child + const handle = superviseFrpc('/frpc', '/toml', { + spawn, + sleep: async () => {}, + logger: silentLogger, + now: () => 1_000_000, // fixed clock: run counts as unstable, but sleep is a no-op → respawn is immediate + }) + await flush() + expect(handle.isChildAlive()).toBe(true) // child[0] + + handle.restartChild() // A5: rotator calls this after a cert rotation to reload the new leaf + expect(children[0]!.killed).toBe(true) + + await flush() + expect(n).toBe(2) // child[1] spawned — frpc now reads the rotated cert on disk + expect(handle.isChildAlive()).toBe(true) + await handle.stop() + }) + + it('restartChild() after stop is a no-op (no spawn past shutdown)', async () => { + const first = makeChild() + let n = 0 + const spawn: SpawnFrpc = vi.fn(() => (n++ === 0 ? first.child : makeChild().child)) + const handle = superviseFrpc('/frpc', '/toml', { + spawn, + sleep: async () => {}, + logger: silentLogger, + }) + await Promise.resolve() + const stopping = handle.stop() + first.exit(0) + await stopping + handle.restartChild() // must not resurrect the supervisor + expect(spawn).toHaveBeenCalledTimes(1) + }) }) diff --git a/agent/test/nativeRenew.test.ts b/agent/test/nativeRenew.test.ts new file mode 100644 index 0000000..def5445 --- /dev/null +++ b/agent/test/nativeRenew.test.ts @@ -0,0 +1,277 @@ +/** + * A5 native cert auto-renew wiring — TDD. + * + * Covers the host-side glue that was the "one real host code gap": the native run-loop must actually + * RENEW the frp-client leaf (not merely monitor its freshness). Three units: + * - `createMtlsFetch` — the injected `fetchImpl` `renewCert` uses: it POSTs /renew over mTLS + * presenting the CURRENT keystore leaf (read fresh on every call, so a post-rotation renewal + * authenticates with the new leaf) and maps the transport response to a `Response`. + * - `wireAutoRenew` — routes the rotator's rotated→restartChild(+log), revoked→stop(+log), + * error→log(no secret) callbacks and starts it. + * - `startNativeAutoRenew` — the end-to-end builder `superviseNative` calls (identity + mTLS fetch + * + rotator), returning null (disabled) when unenrolled. + */ +import { describe, expect, it, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { AgentConfig } from '../src/config/agentConfig.js' +import { generateP256Identity } from '../src/keys/identity.js' +import { openKeystore } from '../src/keys/keystore.js' +import { createLogger } from '../src/log/logger.js' +import type { CertRotator } from '../src/certs/rotation.js' +import { + createMtlsFetch, + startNativeAutoRenew, + wireAutoRenew, + type MtlsRequest, +} from '../src/certs/nativeRenew.js' +import { FakeTimer } from './fixtures/fakes.js' + +const CFG: AgentConfig = { + relayUrl: 'wss://relay/agent', + enrollUrl: 'https://cp.example.com/enroll', + stateDir: '/tmp/x', + localTargetUrl: 'ws://127.0.0.1:3000', + subdomain: 'host-42', + hostId: 'h-1', +} + +function enrolledKs(): { dir: string; ks: ReturnType } { + const dir = mkdtempSync(join(tmpdir(), 'wta-nr-')) + const ks = openKeystore(dir) + ks.saveIdentity(generateP256Identity()) + ks.saveCert('LEAFCERT', 'CACHAIN') + return { dir, ks } +} + +const farFuture = (): { validTo: Date } => ({ validTo: new Date(Date.now() + 86_400_000) }) +const flush = (): Promise => new Promise((r) => setImmediate(r)) + +describe('createMtlsFetch (A5)', () => { + it('presents the current keystore cert/key/CA over mTLS and maps the transport response', async () => { + const { dir, ks } = enrolledKs() + const seen: Array<{ url: string; tls: Record; init: Record }> = [] + const request: MtlsRequest = async (url, tls, init) => { + seen.push({ url, tls: tls as unknown as Record, init: init as unknown as Record }) + return { status: 200, body: JSON.stringify({ cert: 'NEW', caChain: 'NEWCA' }) } + } + const f = createMtlsFetch(ks, { request, certParser: farFuture }) + + const res = await f('https://cp.example.com/renew', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"csr":"x"}', + }) + + expect(res.status).toBe(200) + expect(res.ok).toBe(true) + expect(await res.json()).toEqual({ cert: 'NEW', caChain: 'NEWCA' }) + expect(seen[0]!.url).toBe('https://cp.example.com/renew') + expect(seen[0]!.tls.cert).toBe('LEAFCERT') + expect(seen[0]!.tls.ca).toBe('CACHAIN') + expect(seen[0]!.tls.rejectUnauthorized).toBe(true) + expect(String(seen[0]!.tls.key)).toContain('PRIVATE KEY') // in-process PKCS#8 key, mTLS only + expect(seen[0]!.init.method).toBe('POST') + expect(seen[0]!.init.body).toBe('{"csr":"x"}') + rmSync(dir, { recursive: true, force: true }) + }) + + it('reads the current cert on every call so a post-rotation renewal uses the new leaf', async () => { + const { dir, ks } = enrolledKs() + const certs: string[] = [] + const request: MtlsRequest = async (_url, tls) => { + certs.push(tls.cert) + return { status: 200, body: '{}' } + } + const f = createMtlsFetch(ks, { request, certParser: farFuture }) + + await f('https://x/renew', { method: 'POST' }) + ks.saveCert('ROTATEDLEAF', 'CACHAIN') // simulate a completed rotation + await f('https://x/renew', { method: 'POST' }) + + expect(certs).toEqual(['LEAFCERT', 'ROTATEDLEAF']) + rmSync(dir, { recursive: true, force: true }) + }) + + it('propagates a not-enrolled keystore as a throw (renewCert then retries with backoff)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'wta-nr-')) + const ks = openKeystore(dir) + const request: MtlsRequest = async () => ({ status: 200, body: '{}' }) + const f = createMtlsFetch(ks, { request, certParser: farFuture }) + await expect(f('https://x/renew', { method: 'POST' })).rejects.toThrow() + rmSync(dir, { recursive: true, force: true }) + }) +}) + +describe('wireAutoRenew (A5)', () => { + function fakeRotator(): { + rotator: CertRotator + fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } + start: ReturnType + stop: ReturnType + } { + const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {} + const start = vi.fn() + const stop = vi.fn() + const rotator: CertRotator = { + start, + stop, + onRotated: (cb) => { + fire.rotated = cb + }, + onRevoked: (cb) => { + fire.revoked = cb + }, + onError: (cb) => { + fire.error = cb + }, + } + return { rotator, fire, start, stop } + } + + it('routes rotated→restartChild, revoked→stop, error→log; starts and stops the rotator', () => { + const { rotator, fire, start, stop } = fakeRotator() + const restartChild = vi.fn() + const stopSupervisor = vi.fn() + const lines: string[] = [] + const logger = createLogger('info', (l) => lines.push(l)) + + const controller = wireAutoRenew(rotator, { restartChild, stop: stopSupervisor }, logger, { + subdomain: 'host-42', + hostId: 'h-1', + }) + expect(start).toHaveBeenCalledTimes(1) + + fire.rotated!() + expect(restartChild).toHaveBeenCalledTimes(1) + + fire.revoked!() + expect(stopSupervisor).toHaveBeenCalledTimes(1) + + fire.error!(new Error('network down')) + + controller.stop() + expect(stop).toHaveBeenCalledTimes(1) + + const joined = lines.join('\n') + expect(joined).toContain('host-42') // non-secret identifier is logged + expect(joined).not.toContain('LEAFCERT') // never a leaf/key/CSR + }) +}) + +describe('startNativeAutoRenew (A5 end-to-end)', () => { + it('a scheduled 200 renewal rotates the leaf on disk and restarts frpc with it', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + const request: MtlsRequest = async () => ({ + status: 200, + body: JSON.stringify({ cert: 'FRESHLEAF', caChain: 'CACHAIN' }), + }) + const restartChild = vi.fn() + const stop = vi.fn() + + const controller = startNativeAutoRenew( + CFG, + ks, + { restartChild, stop }, + createLogger('error', () => {}), + { mtlsRequest: request, certParser: farFuture, timer, renewBeforeMs: 1000, now: () => new Date(0), parseCert: () => new Date(2000) }, + )! + expect(controller).not.toBeNull() + + timer.advance(1000) // renewal fires at ~2/3 TTL + await flush() + + expect(ks.loadCert()!.certPem).toBe('FRESHLEAF') // atomic persist + expect(restartChild).toHaveBeenCalledTimes(1) // frpc restarted onto the fresh leaf + expect(stop).not.toHaveBeenCalled() + controller.stop() + rmSync(dir, { recursive: true, force: true }) + }) + + it('a scheduled 403 renewal tears the tunnel down (revoked) and never rotates', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + const request: MtlsRequest = async () => ({ status: 403, body: '' }) + const restartChild = vi.fn() + const stop = vi.fn() + + const controller = startNativeAutoRenew( + CFG, + ks, + { restartChild, stop }, + createLogger('error', () => {}), + { mtlsRequest: request, certParser: farFuture, timer, renewBeforeMs: 1000, now: () => new Date(0), parseCert: () => new Date(2000) }, + )! + + timer.advance(1000) + await flush() + + expect(stop).toHaveBeenCalledTimes(1) + expect(restartChild).not.toHaveBeenCalled() + expect(ks.loadCert()!.certPem).toBe('LEAFCERT') // untouched + controller.stop() + rmSync(dir, { recursive: true, force: true }) + }) + + it('a failing renewal is logged (no secret) and retried without crashing, then rotates', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + let calls = 0 + const request: MtlsRequest = async () => { + calls += 1 + if (calls === 1) throw new Error('ECONNREFUSED') + return { status: 200, body: JSON.stringify({ cert: 'FRESHLEAF', caChain: 'CACHAIN' }) } + } + const restartChild = vi.fn() + const stop = vi.fn() + const lines: string[] = [] + + const controller = startNativeAutoRenew( + CFG, + ks, + { restartChild, stop }, + createLogger('warn', (l) => lines.push(l)), + { + mtlsRequest: request, + certParser: farFuture, + timer, + renewBeforeMs: 1000, + retryBaseMs: 500, + now: () => new Date(0), + parseCert: () => new Date(2000), + }, + )! + + timer.advance(1000) // first attempt throws + await flush() + expect(restartChild).not.toHaveBeenCalled() + expect(stop).not.toHaveBeenCalled() // a failure NEVER tears down + expect(lines.join('\n')).toContain('retry') + + timer.advance(500) // backoff retry fires and succeeds + await flush() + expect(ks.loadCert()!.certPem).toBe('FRESHLEAF') + expect(restartChild).toHaveBeenCalledTimes(1) + expect(lines.join('\n')).not.toContain('LEAFCERT') + controller.stop() + rmSync(dir, { recursive: true, force: true }) + }) + + it('returns null (auto-renew disabled) when the keystore has no identity', () => { + const dir = mkdtempSync(join(tmpdir(), 'wta-nr-')) + const ks = openKeystore(dir) + const lines: string[] = [] + const controller = startNativeAutoRenew( + CFG, + ks, + { restartChild: () => {}, stop: () => {} }, + createLogger('warn', (l) => lines.push(l)), + {}, + ) + expect(controller).toBeNull() + expect(lines.join('\n')).toContain('auto-renew disabled') + rmSync(dir, { recursive: true, force: true }) + }) +}) diff --git a/agent/test/nativeRenewTransport.test.ts b/agent/test/nativeRenewTransport.test.ts new file mode 100644 index 0000000..e8f8d1f --- /dev/null +++ b/agent/test/nativeRenewTransport.test.ts @@ -0,0 +1,168 @@ +/** + * A5 default mTLS transport (`defaultMtlsRequest`) — the ONE seam the other nativeRenew tests inject + * past, so the real `node:https` transport had zero coverage. These tests mock `node:https` and drive + * the actual transport to prove: + * - the exact request options (rejectUnauthorized:true + client cert/key + pinned CA + method/body), + * - the HIGH fix: a request timeout is armed and a stalled peer REJECTS (never hangs forever), + * - the MEDIUM fix: an oversized response body is capped (destroyed + rejected, not buffered). + */ +import { EventEmitter } from 'node:events' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { requestMock } = vi.hoisted(() => ({ requestMock: vi.fn() })) +vi.mock('node:https', () => ({ request: requestMock, default: { request: requestMock } })) + +import { generateP256Identity } from '../src/keys/identity.js' +import { openKeystore } from '../src/keys/keystore.js' +import { + createMtlsFetch, + RENEW_REQUEST_TIMEOUT_MS, + MAX_RENEW_RESPONSE_BYTES, +} from '../src/certs/nativeRenew.js' + +const farFuture = (): { validTo: Date } => ({ validTo: new Date(Date.now() + 86_400_000) }) + +/** Fake `http.ClientRequest`: records setTimeout/write/end and emits 'error' on destroy(err). */ +class FakeClientRequest extends EventEmitter { + readonly setTimeoutCalls: Array<{ ms: number; cb: () => void }> = [] + readonly written: string[] = [] + ended = false + destroyedWith: Error | undefined + setTimeout(ms: number, cb: () => void): this { + this.setTimeoutCalls.push({ ms, cb }) + return this + } + write(chunk: string): boolean { + this.written.push(chunk) + return true + } + end(): this { + this.ended = true + return this + } + destroy(err?: Error): this { + this.destroyedWith = err + if (err) this.emit('error', err) + return this + } +} + +/** Fake `http.IncomingMessage`: an EventEmitter with a statusCode and a real destroy(). */ +class FakeIncomingMessage extends EventEmitter { + statusCode = 200 + destroyed = false + destroy(): this { + this.destroyed = true + return this + } +} + +type ReqCb = (res: FakeIncomingMessage) => void +let dirs: string[] = [] + +function enrolledKs(): ReturnType { + const dir = mkdtempSync(join(tmpdir(), 'wta-nrt-')) + dirs.push(dir) + const ks = openKeystore(dir) + ks.saveIdentity(generateP256Identity()) + ks.saveCert('LEAFCERT', 'CACHAIN') + return ks +} + +beforeEach(() => { + requestMock.mockReset() +}) +afterEach(() => { + for (const d of dirs) rmSync(d, { recursive: true, force: true }) + dirs = [] +}) + +describe('defaultMtlsRequest transport (A5 — real node:https)', () => { + it('passes rejectUnauthorized:true + the client cert/key/CA + method/body, and arms the timeout', async () => { + const ks = enrolledKs() + let seenUrl = '' + let seenOpts: Record = {} + let seenReq: FakeClientRequest | undefined + requestMock.mockImplementation((url: string, opts: Record, cb: ReqCb) => { + seenUrl = url + seenOpts = opts + const req = new FakeClientRequest() + seenReq = req + queueMicrotask(() => { + const res = new FakeIncomingMessage() + res.statusCode = 200 + cb(res) + res.emit('data', Buffer.from('{"cert":"NEW",')) + res.emit('data', Buffer.from('"caChain":"NEWCA"}')) + res.emit('end') + }) + return req + }) + + const f = createMtlsFetch(ks, { certParser: farFuture }) // NO request seam → real defaultMtlsRequest + const res = await f('https://cp.example.com/renew', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"csr":"x"}', + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ cert: 'NEW', caChain: 'NEWCA' }) + expect(seenUrl).toBe('https://cp.example.com/renew') + expect(seenOpts.rejectUnauthorized).toBe(true) // anti-MITM (INV14) + expect(seenOpts.method).toBe('POST') + expect(seenOpts.cert).toBe('LEAFCERT') + expect(seenOpts.ca).toBe('CACHAIN') + expect(String(seenOpts.key)).toContain('PRIVATE KEY') // in-process PKCS#8 key + // HIGH fix: a socket timeout is armed with the sane default so a stall can never hang forever. + expect(seenReq!.setTimeoutCalls).toHaveLength(1) + expect(seenReq!.setTimeoutCalls[0]!.ms).toBe(RENEW_REQUEST_TIMEOUT_MS) + expect(seenReq!.written).toEqual(['{"csr":"x"}']) + expect(seenReq!.ended).toBe(true) + }) + + it('HIGH: a stalled peer (accepts TLS, never responds) REJECTS via the timeout, not hangs', async () => { + const ks = enrolledKs() + let seenReq: FakeClientRequest | undefined + requestMock.mockImplementation(() => { + const req = new FakeClientRequest() + seenReq = req + return req // never invokes the response callback — a stalled control-plane + }) + + const f = createMtlsFetch(ks, { certParser: farFuture }) + const p = f('https://cp.example.com/renew', { method: 'POST', body: '{}' }) + + // Fire the armed socket-timeout callback (what node does when the socket idles past the limit). + expect(seenReq!.setTimeoutCalls).toHaveLength(1) + expect(seenReq!.setTimeoutCalls[0]!.ms).toBe(RENEW_REQUEST_TIMEOUT_MS) + seenReq!.setTimeoutCalls[0]!.cb() + + await expect(p).rejects.toThrow(/timed out/i) + expect(seenReq!.destroyedWith).toBeInstanceOf(Error) // socket was torn down + }) + + it('MEDIUM: an oversized response body is capped — res destroyed + promise rejected', async () => { + const ks = enrolledKs() + let seenRes: FakeIncomingMessage | undefined + requestMock.mockImplementation((_url: string, _opts: unknown, cb: ReqCb) => { + const req = new FakeClientRequest() + queueMicrotask(() => { + const res = new FakeIncomingMessage() + res.statusCode = 200 + seenRes = res + cb(res) + res.emit('data', Buffer.alloc(MAX_RENEW_RESPONSE_BYTES + 1)) // one byte over the cap + // deliberately NO 'end' — a capped stream must reject on its own, not wait for end + }) + return req + }) + + const f = createMtlsFetch(ks, { certParser: farFuture }) + await expect(f('https://cp.example.com/renew', { method: 'POST' })).rejects.toThrow(/cap|exceed/i) + expect(seenRes!.destroyed).toBe(true) + }) +}) diff --git a/agent/test/rotation.test.ts b/agent/test/rotation.test.ts index b5c3359..1410079 100644 --- a/agent/test/rotation.test.ts +++ b/agent/test/rotation.test.ts @@ -11,6 +11,7 @@ import { renewCert, renewalUrlFor, } from '../src/certs/rotation.js' +import { createBackoff } from '../src/transport/backoff.js' import { FakeTimer } from './fixtures/fakes.js' const CFG: AgentConfig = { @@ -117,4 +118,44 @@ describe('createCertRotator (T13)', () => { rotator.stop() rmSync(dir, { recursive: true, force: true }) }) + + it('invokes onError and retries with backoff (not renewBeforeMs) when a renewal throws', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + let calls = 0 + const fetchImpl = (async () => { + calls += 1 + if (calls === 1) throw new Error('network down') + return jsonRes(200, { cert: 'NEWCERT', caChain: 'NEWCA' }) + }) as unknown as typeof fetch + const errors: unknown[] = [] + let rotated = 0 + const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, { + timer, + renewBeforeMs: 1000, + retryBackoff: createBackoff({ baseMs: 500, jitter: false }), + fetchImpl, + now: () => new Date(0), + parseCert: () => new Date(2000), // initial renewal scheduled at ~1000ms + }) + rotator.onError((e) => errors.push(e)) + rotator.onRotated(() => { + rotated += 1 + }) + rotator.start() + + timer.advance(1000) // first attempt fires → throws + await flush() + expect(errors).toHaveLength(1) + expect(rotated).toBe(0) + + // The retry is armed at the 500ms backoff delay, NOT renewBeforeMs (1000): advancing only 500 + // must fire it. A crash-loop never escapes here (the supervisor keeps running). + timer.advance(500) + await flush() + expect(rotated).toBe(1) + expect(ks.loadCert()!.certPem).toBe('NEWCERT') + rotator.stop() + rmSync(dir, { recursive: true, force: true }) + }) })