feat(agent): auto-renew the host cert in the native run-loop (A5)
Wire createCertRotator/renewCert into superviseNative so the frp-client cert renews silently at ~2/3 TTL over mTLS /renew and frpc restarts onto the fresh leaf; failures retry with backoff, never crash the supervisor. mTLS transport has a 15s timeout + 64KB response cap. 281 tests pass.
This commit is contained in:
240
agent/src/certs/nativeRenew.ts
Normal file
240
agent/src/certs/nativeRenew.ts
Normal file
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Native-tunnel cert auto-renew wiring — TASK A5 (PLAN_ZERO_TOUCH_ROLLOUT).
|
||||
*
|
||||
* The native run-loop (`superviseNative`) used to only MONITOR the frp-client leaf's freshness; the
|
||||
* leaf therefore expired at ~24h and the tunnel dropped until a manual re-pair. This module closes
|
||||
* that gap by driving `createCertRotator`/`renewCert` (crypto is REUSED, never reimplemented):
|
||||
*
|
||||
* - `createMtlsFetch` — the injected `fetchImpl` the rotator hands to `renewCert`. It POSTs /renew
|
||||
* over mTLS presenting the CURRENT keystore leaf (re-read on every call, so the first renewal
|
||||
* after a rotation already authenticates with the freshly issued leaf). mTLS IS the auth — no
|
||||
* token, `rejectUnauthorized` always true (INV4/INV14). The private key stays in-process.
|
||||
* - `wireAutoRenew` — routes the rotator callbacks: rotated → restart frpc onto the new leaf and
|
||||
* log (non-secret); revoked (403) → tear the tunnel down (INV12); error → log + let the rotator
|
||||
* retry with backoff. A failed renewal NEVER crashes the supervisor.
|
||||
* - `startNativeAutoRenew` — the builder `superviseNative` calls: loads the identity, builds the
|
||||
* mTLS fetch + rotator at ~2/3-TTL, and starts it. Returns null (auto-renew disabled) if the host
|
||||
* is not enrolled (no identity), rather than throwing into the run-loop.
|
||||
*/
|
||||
import { request as httpsRequest } from 'node:https'
|
||||
import type { AgentConfig } from '../config/agentConfig.js'
|
||||
import type { Keystore } from '../keys/keystore.js'
|
||||
import type { Logger } from '../log/logger.js'
|
||||
import type { TimerLike } from '../transport/seams.js'
|
||||
import { createBackoff } from '../transport/backoff.js'
|
||||
import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js'
|
||||
import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js'
|
||||
import { createCertRotator, type CertRotator } from './rotation.js'
|
||||
|
||||
/** Non-secret message from an unknown thrown value (never serializes cert/key material). */
|
||||
function errorMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err)
|
||||
}
|
||||
|
||||
/**
|
||||
* Socket-idle timeout for a /renew request. A stalled or overloaded control-plane (or a NAT that
|
||||
* silently drops the connection after the TLS handshake) must NOT leave the renewal Promise pending
|
||||
* forever — that would starve the rotator's backoff-retry loop and let the leaf silently expire. On
|
||||
* timeout the request is destroyed and the rejection surfaces through the rotator's onError→backoff.
|
||||
*/
|
||||
export const RENEW_REQUEST_TIMEOUT_MS = 15_000
|
||||
/**
|
||||
* Hard cap on the buffered /renew response body. The reply is a small `{cert,caChain}` JSON; anything
|
||||
* beyond a few KB is malformed or hostile, so we destroy the stream and reject rather than buffer it.
|
||||
*/
|
||||
export const MAX_RENEW_RESPONSE_BYTES = 64 * 1024
|
||||
|
||||
// --- mTLS fetch --------------------------------------------------------------------------------
|
||||
|
||||
/** A single mTLS request the fetch shim delegates to (injectable so the shim is offline-testable). */
|
||||
export interface MtlsRequestInit {
|
||||
readonly method: string
|
||||
readonly headers: Record<string, string>
|
||||
readonly body?: string
|
||||
}
|
||||
export interface MtlsResponse {
|
||||
readonly status: number
|
||||
readonly body: string
|
||||
}
|
||||
export type MtlsRequest = (
|
||||
url: string,
|
||||
tls: TlsClientOptions,
|
||||
init: MtlsRequestInit,
|
||||
) => Promise<MtlsResponse>
|
||||
|
||||
/** Default mTLS transport: a `node:https` POST presenting the client cert/key + pinned CA. */
|
||||
const defaultMtlsRequest: MtlsRequest = (url, tls, init) =>
|
||||
new Promise<MtlsResponse>((resolve, reject) => {
|
||||
const req = httpsRequest(
|
||||
url,
|
||||
{
|
||||
method: init.method,
|
||||
headers: init.headers,
|
||||
cert: tls.cert,
|
||||
key: tls.key,
|
||||
ca: tls.ca,
|
||||
rejectUnauthorized: tls.rejectUnauthorized, // always true (anti-MITM, INV14)
|
||||
},
|
||||
(res) => {
|
||||
const chunks: Buffer[] = []
|
||||
let total = 0
|
||||
res.on('data', (c: Buffer) => {
|
||||
total += c.length
|
||||
if (total > MAX_RENEW_RESPONSE_BYTES) {
|
||||
res.destroy() // MEDIUM: refuse an unbounded body — a renew reply is a few-KB JSON
|
||||
reject(new Error(`renew response body exceeded ${MAX_RENEW_RESPONSE_BYTES} byte cap`))
|
||||
return
|
||||
}
|
||||
chunks.push(c)
|
||||
})
|
||||
res.on('end', () =>
|
||||
resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') }),
|
||||
)
|
||||
res.on('error', reject) // a mid-stream socket error must reject, not hang
|
||||
},
|
||||
)
|
||||
// HIGH: bound the request so a peer that accepts the connection but never replies rejects (and the
|
||||
// rotator re-enters backoff) instead of pending forever — destroy(err) emits 'error' → reject below.
|
||||
req.setTimeout(RENEW_REQUEST_TIMEOUT_MS, () => {
|
||||
req.destroy(new Error(`renew request timed out after ${RENEW_REQUEST_TIMEOUT_MS}ms`))
|
||||
})
|
||||
req.on('error', reject)
|
||||
if (init.body !== undefined) req.write(init.body)
|
||||
req.end()
|
||||
})
|
||||
|
||||
function toHeaderRecord(headers: RequestInit['headers']): Record<string, string> {
|
||||
if (!headers) return {}
|
||||
if (headers instanceof Headers) {
|
||||
const out: Record<string, string> = {}
|
||||
headers.forEach((v, k) => {
|
||||
out[k] = v
|
||||
})
|
||||
return out
|
||||
}
|
||||
if (Array.isArray(headers)) return Object.fromEntries(headers)
|
||||
return { ...(headers as Record<string, string>) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `fetch`-shaped shim `renewCert` uses. Each call re-reads the CURRENT keystore leaf via
|
||||
* `buildTlsOptions` (which fail-fast throws NotEnrolled/CertExpired — the rotator then logs + retries
|
||||
* with backoff, never crashing) and delegates to the mTLS transport, mapping the result to a real
|
||||
* `Response` (so `res.ok`/`res.status`/`res.json()` behave exactly as `renewCert` expects).
|
||||
*/
|
||||
export function createMtlsFetch(
|
||||
ks: Keystore,
|
||||
opts: { request?: MtlsRequest; certParser?: CertParser } = {},
|
||||
): typeof fetch {
|
||||
const request = opts.request ?? defaultMtlsRequest
|
||||
const shim = async (input: Parameters<typeof fetch>[0], init?: RequestInit): Promise<Response> => {
|
||||
const url = typeof input === 'string' ? input : input.toString()
|
||||
const tls = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) })
|
||||
const reqInit: MtlsRequestInit = {
|
||||
method: init?.method ?? 'GET',
|
||||
headers: toHeaderRecord(init?.headers),
|
||||
...(typeof init?.body === 'string' ? { body: init.body } : {}),
|
||||
}
|
||||
const { status, body } = await request(url, tls, reqInit)
|
||||
return new Response(body, { status })
|
||||
}
|
||||
return shim as typeof fetch
|
||||
}
|
||||
|
||||
// --- rotator wiring ----------------------------------------------------------------------------
|
||||
|
||||
/** Non-secret identifiers logged alongside renew events (INV9). */
|
||||
export interface AutoRenewLogIds {
|
||||
readonly subdomain: string | null
|
||||
readonly hostId: string | null
|
||||
}
|
||||
|
||||
/** The two run-loop effects the rotator drives. */
|
||||
export interface AutoRenewHooks {
|
||||
/** Restart the supervised frpc so it re-reads the rotated cert (a leaf rotation only). */
|
||||
restartChild(): void
|
||||
/** Tear the tunnel down (host revoked ⇒ never reconnect, INV12). */
|
||||
stop(): void
|
||||
}
|
||||
|
||||
/** Handle for the wired auto-renew loop. */
|
||||
export interface AutoRenewController {
|
||||
stop(): void
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire a rotator's callbacks to the run-loop and start it. Rotated → restart frpc; revoked → stop;
|
||||
* error → log (non-secret) and let the rotator retry with backoff. Returns a controller that stops
|
||||
* the rotator's scheduled timer.
|
||||
*/
|
||||
export function wireAutoRenew(
|
||||
rotator: CertRotator,
|
||||
hooks: AutoRenewHooks,
|
||||
logger: Logger,
|
||||
ids: AutoRenewLogIds,
|
||||
): AutoRenewController {
|
||||
const meta = { subdomain: ids.subdomain, hostId: ids.hostId }
|
||||
rotator.onRotated(() => {
|
||||
logger.log('info', 'frp-client cert rotated; restarting frpc onto the fresh leaf', meta)
|
||||
hooks.restartChild()
|
||||
})
|
||||
rotator.onRevoked(() => {
|
||||
logger.log('warn', 'frp-client cert renewal refused (host revoked); tearing down tunnel', meta)
|
||||
hooks.stop()
|
||||
})
|
||||
rotator.onError((err) => {
|
||||
logger.log('warn', 'frp-client cert renewal failed; will retry with backoff', {
|
||||
...meta,
|
||||
error: errorMessage(err),
|
||||
})
|
||||
})
|
||||
rotator.start()
|
||||
return { stop: () => rotator.stop() }
|
||||
}
|
||||
|
||||
// --- builder -----------------------------------------------------------------------------------
|
||||
|
||||
/** Injection seams for `startNativeAutoRenew` (all optional; unset ⇒ real transport/timers). */
|
||||
export interface NativeAutoRenewOpts {
|
||||
readonly mtlsRequest?: MtlsRequest
|
||||
readonly certParser?: CertParser
|
||||
readonly timer?: TimerLike
|
||||
readonly renewBeforeMs?: number
|
||||
readonly retryBaseMs?: number
|
||||
readonly now?: () => Date
|
||||
readonly parseCert?: (pem: string) => Date
|
||||
}
|
||||
|
||||
/**
|
||||
* Build + start native cert auto-renew for `superviseNative`. Renews at ~2/3 of the leaf TTL
|
||||
* (default `DEFAULT_CERT_RENEW_WINDOW_MS`, the same window the health probe alarms on). Returns null
|
||||
* (auto-renew disabled, logged) when the host has no identity — an unenrolled run-loop must not throw.
|
||||
*/
|
||||
export function startNativeAutoRenew(
|
||||
cfg: AgentConfig,
|
||||
ks: Keystore,
|
||||
hooks: AutoRenewHooks,
|
||||
logger: Logger,
|
||||
opts: NativeAutoRenewOpts = {},
|
||||
): AutoRenewController | null {
|
||||
const id = ks.loadIdentity()
|
||||
if (id === null) {
|
||||
logger.log('warn', 'no identity in keystore — cert auto-renew disabled', {})
|
||||
return null
|
||||
}
|
||||
const fetchImpl = createMtlsFetch(ks, {
|
||||
...(opts.mtlsRequest ? { request: opts.mtlsRequest } : {}),
|
||||
...(opts.certParser ? { certParser: opts.certParser } : {}),
|
||||
})
|
||||
const rotator = createCertRotator(cfg, id, ks, {
|
||||
fetchImpl,
|
||||
renewBeforeMs: opts.renewBeforeMs ?? DEFAULT_CERT_RENEW_WINDOW_MS,
|
||||
...(opts.timer ? { timer: opts.timer } : {}),
|
||||
...(opts.now ? { now: opts.now } : {}),
|
||||
...(opts.parseCert ? { parseCert: opts.parseCert } : {}),
|
||||
...(opts.retryBaseMs !== undefined
|
||||
? { retryBackoff: createBackoff({ baseMs: opts.retryBaseMs, jitter: false }) }
|
||||
: {}),
|
||||
})
|
||||
return wireAutoRenew(rotator, hooks, logger, { subdomain: cfg.subdomain, hostId: cfg.hostId })
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import type { AgentConfig } from '../config/agentConfig.js'
|
||||
import type { AgentIdentity } from '../keys/identity.js'
|
||||
import type { Keystore } from '../keys/keystore.js'
|
||||
import type { TimerLike } from '../transport/seams.js'
|
||||
import { createBackoff, type BackoffPolicy } from '../transport/backoff.js'
|
||||
import { buildCsr } from '../enroll/csr.js'
|
||||
|
||||
export const DEFAULT_RENEW_BEFORE_MS = 5 * 60_000 // renew 5 min before expiry
|
||||
@@ -22,6 +23,8 @@ export interface CertRotator {
|
||||
stop(): void
|
||||
onRotated(cb: () => void): void
|
||||
onRevoked(cb: () => void): void
|
||||
/** A renewal attempt failed (network/HTTP, NOT a 403 revoke). The rotator retries with backoff. */
|
||||
onError(cb: (err: unknown) => void): void
|
||||
}
|
||||
|
||||
export type RenewOutcome = 'rotated' | 'revoked'
|
||||
@@ -79,6 +82,8 @@ export function createCertRotator(
|
||||
fetchImpl?: typeof fetch
|
||||
now?: () => Date
|
||||
parseCert?: (pem: string) => Date
|
||||
/** Backoff policy for retrying a FAILED renewal (default 1s→30s). Reset after a success. */
|
||||
retryBackoff?: BackoffPolicy
|
||||
} = {},
|
||||
): CertRotator {
|
||||
const renewBeforeMs = opts.renewBeforeMs ?? DEFAULT_RENEW_BEFORE_MS
|
||||
@@ -91,9 +96,11 @@ export function createCertRotator(
|
||||
}
|
||||
const doFetch = opts.fetchImpl ?? fetch
|
||||
const now = opts.now ?? (() => new Date())
|
||||
const retryBackoff = opts.retryBackoff ?? createBackoff({ jitter: true })
|
||||
let handle: unknown = null
|
||||
let rotatedCb: (() => void) | null = null
|
||||
let revokedCb: (() => void) | null = null
|
||||
let errorCb: ((err: unknown) => void) | null = null
|
||||
|
||||
function schedule(): void {
|
||||
const certs = ks.loadCert()
|
||||
@@ -109,12 +116,16 @@ export function createCertRotator(
|
||||
revokedCb?.()
|
||||
return
|
||||
}
|
||||
retryBackoff.reset() // a healthy renewal clears the retry backoff for the next cycle
|
||||
rotatedCb?.()
|
||||
schedule()
|
||||
})
|
||||
.catch(() => {
|
||||
// network error: retry after renewBeforeMs; the tunnel stays up meanwhile.
|
||||
handle = timer.setTimeout(runRenewal, renewBeforeMs)
|
||||
.catch((err: unknown) => {
|
||||
// Network/HTTP failure (never a 403 revoke): surface it (caller logs, no secret) and retry
|
||||
// with backoff. The cert is still valid until expiry, so the tunnel stays up meanwhile — a
|
||||
// failed renewal must NEVER tear the supervisor down.
|
||||
errorCb?.(err)
|
||||
handle = timer.setTimeout(runRenewal, retryBackoff.nextDelayMs())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -132,5 +143,8 @@ export function createCertRotator(
|
||||
onRevoked(cb): void {
|
||||
revokedCb = cb
|
||||
},
|
||||
onError(cb): void {
|
||||
errorCb = cb
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import { runTunnel } from '../transport/runTunnel.js'
|
||||
import { buildNativeFrpcToml } from '../transport/frpcToml.js'
|
||||
import { superviseFrpc } from '../transport/frpSupervise.js'
|
||||
import { provisionFrpc } from '../provision/frpcBinary.js'
|
||||
import { startNativeAutoRenew } from '../certs/nativeRenew.js'
|
||||
import {
|
||||
probeLoopbackBaseApp,
|
||||
renderHealthStatus,
|
||||
@@ -156,9 +157,12 @@ export function readFrpcLog(stateDir: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Native run-loop (B4/H4): supervise the pinned frpc child with restart-on-exit backoff while a
|
||||
* Native run-loop (B4/H4 + A5): supervise the pinned frpc child with restart-on-exit backoff while a
|
||||
* periodic health probe (frpc alive, base-app loopback reachable, proxy-started, cert-not-expiring)
|
||||
* logs NON-SECRET status only (INV9). Resolves when the supervisor stops (SIGTERM/SIGINT).
|
||||
* logs NON-SECRET status only (INV9), AND auto-renew the frp-client leaf at ~2/3 TTL so the tunnel
|
||||
* never drops on cert expiry (A5): a successful renewal restarts frpc onto the fresh leaf, a 403
|
||||
* revoke tears the tunnel down, and a failed renewal retries with backoff without crashing the
|
||||
* supervisor. Resolves when the supervisor stops (SIGTERM/SIGINT).
|
||||
*/
|
||||
function superviseNative(cfg: AgentConfig, ks: Keystore): Promise<number> {
|
||||
const logger = createLogger('info')
|
||||
@@ -184,12 +188,28 @@ function superviseNative(cfg: AgentConfig, ks: Keystore): Promise<number> {
|
||||
for (const line of renderHealthStatus(ids, report)) logger.log('info', line)
|
||||
},
|
||||
)
|
||||
// A5: silently renew the leaf before it expires. Restart frpc onto the fresh cert on rotation;
|
||||
// stop the whole supervisor on a 403 revoke (INV12). Null ⇒ unenrolled (auto-renew disabled).
|
||||
const autoRenew = startNativeAutoRenew(
|
||||
cfg,
|
||||
ks,
|
||||
{
|
||||
restartChild: () => handle.restartChild(),
|
||||
stop: () => {
|
||||
void handle.stop()
|
||||
},
|
||||
},
|
||||
logger,
|
||||
)
|
||||
const onSignal = (): void => {
|
||||
void handle.stop()
|
||||
}
|
||||
process.once('SIGTERM', onSignal)
|
||||
process.once('SIGINT', onSignal)
|
||||
return handle.done.finally(() => monitor.stop())
|
||||
return handle.done.finally(() => {
|
||||
monitor.stop()
|
||||
autoRenew?.stop()
|
||||
})
|
||||
}
|
||||
|
||||
/** Build the concrete CliDeps used by the real CLI entrypoint. */
|
||||
|
||||
@@ -62,6 +62,12 @@ export interface FrpSuperviseHandle {
|
||||
readonly done: Promise<number>
|
||||
/** True while a frpc child is currently running (for the health probe). */
|
||||
isChildAlive(): boolean
|
||||
/**
|
||||
* Kill the current child WITHOUT stopping the supervisor (A5): the restart-on-exit loop respawns a
|
||||
* fresh frpc that re-reads the (now rotated) cert/key/CA files. A no-op once `stop()` was called —
|
||||
* it must never resurrect a supervisor that is shutting down.
|
||||
*/
|
||||
restartChild(): void
|
||||
}
|
||||
|
||||
/** A frpc run lasting at least this long is "stable" ⇒ reset the restart backoff. */
|
||||
@@ -196,5 +202,8 @@ export function superviseFrpc(
|
||||
},
|
||||
done,
|
||||
isChildAlive: () => child?.isAlive() ?? false,
|
||||
restartChild(): void {
|
||||
if (!stopped) child?.kill()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,4 +145,44 @@ describe('superviseFrpc (B4/H4 — restart-on-exit backoff)', () => {
|
||||
await stopping
|
||||
expect(spawn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('restartChild() kills the running child so the loop respawns onto the fresh cert files', async () => {
|
||||
const flush = (): Promise<void> => new Promise((r) => setImmediate(r))
|
||||
const children = [makeChild(), makeChild()]
|
||||
let n = 0
|
||||
const spawn: SpawnFrpc = () => children[n++]!.child
|
||||
const handle = superviseFrpc('/frpc', '/toml', {
|
||||
spawn,
|
||||
sleep: async () => {},
|
||||
logger: silentLogger,
|
||||
now: () => 1_000_000, // fixed clock: run counts as unstable, but sleep is a no-op → respawn is immediate
|
||||
})
|
||||
await flush()
|
||||
expect(handle.isChildAlive()).toBe(true) // child[0]
|
||||
|
||||
handle.restartChild() // A5: rotator calls this after a cert rotation to reload the new leaf
|
||||
expect(children[0]!.killed).toBe(true)
|
||||
|
||||
await flush()
|
||||
expect(n).toBe(2) // child[1] spawned — frpc now reads the rotated cert on disk
|
||||
expect(handle.isChildAlive()).toBe(true)
|
||||
await handle.stop()
|
||||
})
|
||||
|
||||
it('restartChild() after stop is a no-op (no spawn past shutdown)', async () => {
|
||||
const first = makeChild()
|
||||
let n = 0
|
||||
const spawn: SpawnFrpc = vi.fn(() => (n++ === 0 ? first.child : makeChild().child))
|
||||
const handle = superviseFrpc('/frpc', '/toml', {
|
||||
spawn,
|
||||
sleep: async () => {},
|
||||
logger: silentLogger,
|
||||
})
|
||||
await Promise.resolve()
|
||||
const stopping = handle.stop()
|
||||
first.exit(0)
|
||||
await stopping
|
||||
handle.restartChild() // must not resurrect the supervisor
|
||||
expect(spawn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
277
agent/test/nativeRenew.test.ts
Normal file
277
agent/test/nativeRenew.test.ts
Normal file
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* A5 native cert auto-renew wiring — TDD.
|
||||
*
|
||||
* Covers the host-side glue that was the "one real host code gap": the native run-loop must actually
|
||||
* RENEW the frp-client leaf (not merely monitor its freshness). Three units:
|
||||
* - `createMtlsFetch` — the injected `fetchImpl` `renewCert` uses: it POSTs /renew over mTLS
|
||||
* presenting the CURRENT keystore leaf (read fresh on every call, so a post-rotation renewal
|
||||
* authenticates with the new leaf) and maps the transport response to a `Response`.
|
||||
* - `wireAutoRenew` — routes the rotator's rotated→restartChild(+log), revoked→stop(+log),
|
||||
* error→log(no secret) callbacks and starts it.
|
||||
* - `startNativeAutoRenew` — the end-to-end builder `superviseNative` calls (identity + mTLS fetch
|
||||
* + rotator), returning null (disabled) when unenrolled.
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type { AgentConfig } from '../src/config/agentConfig.js'
|
||||
import { generateP256Identity } from '../src/keys/identity.js'
|
||||
import { openKeystore } from '../src/keys/keystore.js'
|
||||
import { createLogger } from '../src/log/logger.js'
|
||||
import type { CertRotator } from '../src/certs/rotation.js'
|
||||
import {
|
||||
createMtlsFetch,
|
||||
startNativeAutoRenew,
|
||||
wireAutoRenew,
|
||||
type MtlsRequest,
|
||||
} from '../src/certs/nativeRenew.js'
|
||||
import { FakeTimer } from './fixtures/fakes.js'
|
||||
|
||||
const CFG: AgentConfig = {
|
||||
relayUrl: 'wss://relay/agent',
|
||||
enrollUrl: 'https://cp.example.com/enroll',
|
||||
stateDir: '/tmp/x',
|
||||
localTargetUrl: 'ws://127.0.0.1:3000',
|
||||
subdomain: 'host-42',
|
||||
hostId: 'h-1',
|
||||
}
|
||||
|
||||
function enrolledKs(): { dir: string; ks: ReturnType<typeof openKeystore> } {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wta-nr-'))
|
||||
const ks = openKeystore(dir)
|
||||
ks.saveIdentity(generateP256Identity())
|
||||
ks.saveCert('LEAFCERT', 'CACHAIN')
|
||||
return { dir, ks }
|
||||
}
|
||||
|
||||
const farFuture = (): { validTo: Date } => ({ validTo: new Date(Date.now() + 86_400_000) })
|
||||
const flush = (): Promise<void> => new Promise((r) => setImmediate(r))
|
||||
|
||||
describe('createMtlsFetch (A5)', () => {
|
||||
it('presents the current keystore cert/key/CA over mTLS and maps the transport response', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const seen: Array<{ url: string; tls: Record<string, unknown>; init: Record<string, unknown> }> = []
|
||||
const request: MtlsRequest = async (url, tls, init) => {
|
||||
seen.push({ url, tls: tls as unknown as Record<string, unknown>, init: init as unknown as Record<string, unknown> })
|
||||
return { status: 200, body: JSON.stringify({ cert: 'NEW', caChain: 'NEWCA' }) }
|
||||
}
|
||||
const f = createMtlsFetch(ks, { request, certParser: farFuture })
|
||||
|
||||
const res = await f('https://cp.example.com/renew', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{"csr":"x"}',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.ok).toBe(true)
|
||||
expect(await res.json()).toEqual({ cert: 'NEW', caChain: 'NEWCA' })
|
||||
expect(seen[0]!.url).toBe('https://cp.example.com/renew')
|
||||
expect(seen[0]!.tls.cert).toBe('LEAFCERT')
|
||||
expect(seen[0]!.tls.ca).toBe('CACHAIN')
|
||||
expect(seen[0]!.tls.rejectUnauthorized).toBe(true)
|
||||
expect(String(seen[0]!.tls.key)).toContain('PRIVATE KEY') // in-process PKCS#8 key, mTLS only
|
||||
expect(seen[0]!.init.method).toBe('POST')
|
||||
expect(seen[0]!.init.body).toBe('{"csr":"x"}')
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('reads the current cert on every call so a post-rotation renewal uses the new leaf', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const certs: string[] = []
|
||||
const request: MtlsRequest = async (_url, tls) => {
|
||||
certs.push(tls.cert)
|
||||
return { status: 200, body: '{}' }
|
||||
}
|
||||
const f = createMtlsFetch(ks, { request, certParser: farFuture })
|
||||
|
||||
await f('https://x/renew', { method: 'POST' })
|
||||
ks.saveCert('ROTATEDLEAF', 'CACHAIN') // simulate a completed rotation
|
||||
await f('https://x/renew', { method: 'POST' })
|
||||
|
||||
expect(certs).toEqual(['LEAFCERT', 'ROTATEDLEAF'])
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('propagates a not-enrolled keystore as a throw (renewCert then retries with backoff)', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wta-nr-'))
|
||||
const ks = openKeystore(dir)
|
||||
const request: MtlsRequest = async () => ({ status: 200, body: '{}' })
|
||||
const f = createMtlsFetch(ks, { request, certParser: farFuture })
|
||||
await expect(f('https://x/renew', { method: 'POST' })).rejects.toThrow()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
describe('wireAutoRenew (A5)', () => {
|
||||
function fakeRotator(): {
|
||||
rotator: CertRotator
|
||||
fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void }
|
||||
start: ReturnType<typeof vi.fn>
|
||||
stop: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {}
|
||||
const start = vi.fn()
|
||||
const stop = vi.fn()
|
||||
const rotator: CertRotator = {
|
||||
start,
|
||||
stop,
|
||||
onRotated: (cb) => {
|
||||
fire.rotated = cb
|
||||
},
|
||||
onRevoked: (cb) => {
|
||||
fire.revoked = cb
|
||||
},
|
||||
onError: (cb) => {
|
||||
fire.error = cb
|
||||
},
|
||||
}
|
||||
return { rotator, fire, start, stop }
|
||||
}
|
||||
|
||||
it('routes rotated→restartChild, revoked→stop, error→log; starts and stops the rotator', () => {
|
||||
const { rotator, fire, start, stop } = fakeRotator()
|
||||
const restartChild = vi.fn()
|
||||
const stopSupervisor = vi.fn()
|
||||
const lines: string[] = []
|
||||
const logger = createLogger('info', (l) => lines.push(l))
|
||||
|
||||
const controller = wireAutoRenew(rotator, { restartChild, stop: stopSupervisor }, logger, {
|
||||
subdomain: 'host-42',
|
||||
hostId: 'h-1',
|
||||
})
|
||||
expect(start).toHaveBeenCalledTimes(1)
|
||||
|
||||
fire.rotated!()
|
||||
expect(restartChild).toHaveBeenCalledTimes(1)
|
||||
|
||||
fire.revoked!()
|
||||
expect(stopSupervisor).toHaveBeenCalledTimes(1)
|
||||
|
||||
fire.error!(new Error('network down'))
|
||||
|
||||
controller.stop()
|
||||
expect(stop).toHaveBeenCalledTimes(1)
|
||||
|
||||
const joined = lines.join('\n')
|
||||
expect(joined).toContain('host-42') // non-secret identifier is logged
|
||||
expect(joined).not.toContain('LEAFCERT') // never a leaf/key/CSR
|
||||
})
|
||||
})
|
||||
|
||||
describe('startNativeAutoRenew (A5 end-to-end)', () => {
|
||||
it('a scheduled 200 renewal rotates the leaf on disk and restarts frpc with it', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const timer = new FakeTimer()
|
||||
const request: MtlsRequest = async () => ({
|
||||
status: 200,
|
||||
body: JSON.stringify({ cert: 'FRESHLEAF', caChain: 'CACHAIN' }),
|
||||
})
|
||||
const restartChild = vi.fn()
|
||||
const stop = vi.fn()
|
||||
|
||||
const controller = startNativeAutoRenew(
|
||||
CFG,
|
||||
ks,
|
||||
{ restartChild, stop },
|
||||
createLogger('error', () => {}),
|
||||
{ mtlsRequest: request, certParser: farFuture, timer, renewBeforeMs: 1000, now: () => new Date(0), parseCert: () => new Date(2000) },
|
||||
)!
|
||||
expect(controller).not.toBeNull()
|
||||
|
||||
timer.advance(1000) // renewal fires at ~2/3 TTL
|
||||
await flush()
|
||||
|
||||
expect(ks.loadCert()!.certPem).toBe('FRESHLEAF') // atomic persist
|
||||
expect(restartChild).toHaveBeenCalledTimes(1) // frpc restarted onto the fresh leaf
|
||||
expect(stop).not.toHaveBeenCalled()
|
||||
controller.stop()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('a scheduled 403 renewal tears the tunnel down (revoked) and never rotates', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const timer = new FakeTimer()
|
||||
const request: MtlsRequest = async () => ({ status: 403, body: '' })
|
||||
const restartChild = vi.fn()
|
||||
const stop = vi.fn()
|
||||
|
||||
const controller = startNativeAutoRenew(
|
||||
CFG,
|
||||
ks,
|
||||
{ restartChild, stop },
|
||||
createLogger('error', () => {}),
|
||||
{ mtlsRequest: request, certParser: farFuture, timer, renewBeforeMs: 1000, now: () => new Date(0), parseCert: () => new Date(2000) },
|
||||
)!
|
||||
|
||||
timer.advance(1000)
|
||||
await flush()
|
||||
|
||||
expect(stop).toHaveBeenCalledTimes(1)
|
||||
expect(restartChild).not.toHaveBeenCalled()
|
||||
expect(ks.loadCert()!.certPem).toBe('LEAFCERT') // untouched
|
||||
controller.stop()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('a failing renewal is logged (no secret) and retried without crashing, then rotates', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const timer = new FakeTimer()
|
||||
let calls = 0
|
||||
const request: MtlsRequest = async () => {
|
||||
calls += 1
|
||||
if (calls === 1) throw new Error('ECONNREFUSED')
|
||||
return { status: 200, body: JSON.stringify({ cert: 'FRESHLEAF', caChain: 'CACHAIN' }) }
|
||||
}
|
||||
const restartChild = vi.fn()
|
||||
const stop = vi.fn()
|
||||
const lines: string[] = []
|
||||
|
||||
const controller = startNativeAutoRenew(
|
||||
CFG,
|
||||
ks,
|
||||
{ restartChild, stop },
|
||||
createLogger('warn', (l) => lines.push(l)),
|
||||
{
|
||||
mtlsRequest: request,
|
||||
certParser: farFuture,
|
||||
timer,
|
||||
renewBeforeMs: 1000,
|
||||
retryBaseMs: 500,
|
||||
now: () => new Date(0),
|
||||
parseCert: () => new Date(2000),
|
||||
},
|
||||
)!
|
||||
|
||||
timer.advance(1000) // first attempt throws
|
||||
await flush()
|
||||
expect(restartChild).not.toHaveBeenCalled()
|
||||
expect(stop).not.toHaveBeenCalled() // a failure NEVER tears down
|
||||
expect(lines.join('\n')).toContain('retry')
|
||||
|
||||
timer.advance(500) // backoff retry fires and succeeds
|
||||
await flush()
|
||||
expect(ks.loadCert()!.certPem).toBe('FRESHLEAF')
|
||||
expect(restartChild).toHaveBeenCalledTimes(1)
|
||||
expect(lines.join('\n')).not.toContain('LEAFCERT')
|
||||
controller.stop()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('returns null (auto-renew disabled) when the keystore has no identity', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wta-nr-'))
|
||||
const ks = openKeystore(dir)
|
||||
const lines: string[] = []
|
||||
const controller = startNativeAutoRenew(
|
||||
CFG,
|
||||
ks,
|
||||
{ restartChild: () => {}, stop: () => {} },
|
||||
createLogger('warn', (l) => lines.push(l)),
|
||||
{},
|
||||
)
|
||||
expect(controller).toBeNull()
|
||||
expect(lines.join('\n')).toContain('auto-renew disabled')
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
168
agent/test/nativeRenewTransport.test.ts
Normal file
168
agent/test/nativeRenewTransport.test.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* A5 default mTLS transport (`defaultMtlsRequest`) — the ONE seam the other nativeRenew tests inject
|
||||
* past, so the real `node:https` transport had zero coverage. These tests mock `node:https` and drive
|
||||
* the actual transport to prove:
|
||||
* - the exact request options (rejectUnauthorized:true + client cert/key + pinned CA + method/body),
|
||||
* - the HIGH fix: a request timeout is armed and a stalled peer REJECTS (never hangs forever),
|
||||
* - the MEDIUM fix: an oversized response body is capped (destroyed + rejected, not buffered).
|
||||
*/
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { requestMock } = vi.hoisted(() => ({ requestMock: vi.fn() }))
|
||||
vi.mock('node:https', () => ({ request: requestMock, default: { request: requestMock } }))
|
||||
|
||||
import { generateP256Identity } from '../src/keys/identity.js'
|
||||
import { openKeystore } from '../src/keys/keystore.js'
|
||||
import {
|
||||
createMtlsFetch,
|
||||
RENEW_REQUEST_TIMEOUT_MS,
|
||||
MAX_RENEW_RESPONSE_BYTES,
|
||||
} from '../src/certs/nativeRenew.js'
|
||||
|
||||
const farFuture = (): { validTo: Date } => ({ validTo: new Date(Date.now() + 86_400_000) })
|
||||
|
||||
/** Fake `http.ClientRequest`: records setTimeout/write/end and emits 'error' on destroy(err). */
|
||||
class FakeClientRequest extends EventEmitter {
|
||||
readonly setTimeoutCalls: Array<{ ms: number; cb: () => void }> = []
|
||||
readonly written: string[] = []
|
||||
ended = false
|
||||
destroyedWith: Error | undefined
|
||||
setTimeout(ms: number, cb: () => void): this {
|
||||
this.setTimeoutCalls.push({ ms, cb })
|
||||
return this
|
||||
}
|
||||
write(chunk: string): boolean {
|
||||
this.written.push(chunk)
|
||||
return true
|
||||
}
|
||||
end(): this {
|
||||
this.ended = true
|
||||
return this
|
||||
}
|
||||
destroy(err?: Error): this {
|
||||
this.destroyedWith = err
|
||||
if (err) this.emit('error', err)
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
/** Fake `http.IncomingMessage`: an EventEmitter with a statusCode and a real destroy(). */
|
||||
class FakeIncomingMessage extends EventEmitter {
|
||||
statusCode = 200
|
||||
destroyed = false
|
||||
destroy(): this {
|
||||
this.destroyed = true
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
type ReqCb = (res: FakeIncomingMessage) => void
|
||||
let dirs: string[] = []
|
||||
|
||||
function enrolledKs(): ReturnType<typeof openKeystore> {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wta-nrt-'))
|
||||
dirs.push(dir)
|
||||
const ks = openKeystore(dir)
|
||||
ks.saveIdentity(generateP256Identity())
|
||||
ks.saveCert('LEAFCERT', 'CACHAIN')
|
||||
return ks
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
requestMock.mockReset()
|
||||
})
|
||||
afterEach(() => {
|
||||
for (const d of dirs) rmSync(d, { recursive: true, force: true })
|
||||
dirs = []
|
||||
})
|
||||
|
||||
describe('defaultMtlsRequest transport (A5 — real node:https)', () => {
|
||||
it('passes rejectUnauthorized:true + the client cert/key/CA + method/body, and arms the timeout', async () => {
|
||||
const ks = enrolledKs()
|
||||
let seenUrl = ''
|
||||
let seenOpts: Record<string, unknown> = {}
|
||||
let seenReq: FakeClientRequest | undefined
|
||||
requestMock.mockImplementation((url: string, opts: Record<string, unknown>, cb: ReqCb) => {
|
||||
seenUrl = url
|
||||
seenOpts = opts
|
||||
const req = new FakeClientRequest()
|
||||
seenReq = req
|
||||
queueMicrotask(() => {
|
||||
const res = new FakeIncomingMessage()
|
||||
res.statusCode = 200
|
||||
cb(res)
|
||||
res.emit('data', Buffer.from('{"cert":"NEW",'))
|
||||
res.emit('data', Buffer.from('"caChain":"NEWCA"}'))
|
||||
res.emit('end')
|
||||
})
|
||||
return req
|
||||
})
|
||||
|
||||
const f = createMtlsFetch(ks, { certParser: farFuture }) // NO request seam → real defaultMtlsRequest
|
||||
const res = await f('https://cp.example.com/renew', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{"csr":"x"}',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.json()).toEqual({ cert: 'NEW', caChain: 'NEWCA' })
|
||||
expect(seenUrl).toBe('https://cp.example.com/renew')
|
||||
expect(seenOpts.rejectUnauthorized).toBe(true) // anti-MITM (INV14)
|
||||
expect(seenOpts.method).toBe('POST')
|
||||
expect(seenOpts.cert).toBe('LEAFCERT')
|
||||
expect(seenOpts.ca).toBe('CACHAIN')
|
||||
expect(String(seenOpts.key)).toContain('PRIVATE KEY') // in-process PKCS#8 key
|
||||
// HIGH fix: a socket timeout is armed with the sane default so a stall can never hang forever.
|
||||
expect(seenReq!.setTimeoutCalls).toHaveLength(1)
|
||||
expect(seenReq!.setTimeoutCalls[0]!.ms).toBe(RENEW_REQUEST_TIMEOUT_MS)
|
||||
expect(seenReq!.written).toEqual(['{"csr":"x"}'])
|
||||
expect(seenReq!.ended).toBe(true)
|
||||
})
|
||||
|
||||
it('HIGH: a stalled peer (accepts TLS, never responds) REJECTS via the timeout, not hangs', async () => {
|
||||
const ks = enrolledKs()
|
||||
let seenReq: FakeClientRequest | undefined
|
||||
requestMock.mockImplementation(() => {
|
||||
const req = new FakeClientRequest()
|
||||
seenReq = req
|
||||
return req // never invokes the response callback — a stalled control-plane
|
||||
})
|
||||
|
||||
const f = createMtlsFetch(ks, { certParser: farFuture })
|
||||
const p = f('https://cp.example.com/renew', { method: 'POST', body: '{}' })
|
||||
|
||||
// Fire the armed socket-timeout callback (what node does when the socket idles past the limit).
|
||||
expect(seenReq!.setTimeoutCalls).toHaveLength(1)
|
||||
expect(seenReq!.setTimeoutCalls[0]!.ms).toBe(RENEW_REQUEST_TIMEOUT_MS)
|
||||
seenReq!.setTimeoutCalls[0]!.cb()
|
||||
|
||||
await expect(p).rejects.toThrow(/timed out/i)
|
||||
expect(seenReq!.destroyedWith).toBeInstanceOf(Error) // socket was torn down
|
||||
})
|
||||
|
||||
it('MEDIUM: an oversized response body is capped — res destroyed + promise rejected', async () => {
|
||||
const ks = enrolledKs()
|
||||
let seenRes: FakeIncomingMessage | undefined
|
||||
requestMock.mockImplementation((_url: string, _opts: unknown, cb: ReqCb) => {
|
||||
const req = new FakeClientRequest()
|
||||
queueMicrotask(() => {
|
||||
const res = new FakeIncomingMessage()
|
||||
res.statusCode = 200
|
||||
seenRes = res
|
||||
cb(res)
|
||||
res.emit('data', Buffer.alloc(MAX_RENEW_RESPONSE_BYTES + 1)) // one byte over the cap
|
||||
// deliberately NO 'end' — a capped stream must reject on its own, not wait for end
|
||||
})
|
||||
return req
|
||||
})
|
||||
|
||||
const f = createMtlsFetch(ks, { certParser: farFuture })
|
||||
await expect(f('https://cp.example.com/renew', { method: 'POST' })).rejects.toThrow(/cap|exceed/i)
|
||||
expect(seenRes!.destroyed).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
renewCert,
|
||||
renewalUrlFor,
|
||||
} from '../src/certs/rotation.js'
|
||||
import { createBackoff } from '../src/transport/backoff.js'
|
||||
import { FakeTimer } from './fixtures/fakes.js'
|
||||
|
||||
const CFG: AgentConfig = {
|
||||
@@ -117,4 +118,44 @@ describe('createCertRotator (T13)', () => {
|
||||
rotator.stop()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('invokes onError and retries with backoff (not renewBeforeMs) when a renewal throws', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const timer = new FakeTimer()
|
||||
let calls = 0
|
||||
const fetchImpl = (async () => {
|
||||
calls += 1
|
||||
if (calls === 1) throw new Error('network down')
|
||||
return jsonRes(200, { cert: 'NEWCERT', caChain: 'NEWCA' })
|
||||
}) as unknown as typeof fetch
|
||||
const errors: unknown[] = []
|
||||
let rotated = 0
|
||||
const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
|
||||
timer,
|
||||
renewBeforeMs: 1000,
|
||||
retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
|
||||
fetchImpl,
|
||||
now: () => new Date(0),
|
||||
parseCert: () => new Date(2000), // initial renewal scheduled at ~1000ms
|
||||
})
|
||||
rotator.onError((e) => errors.push(e))
|
||||
rotator.onRotated(() => {
|
||||
rotated += 1
|
||||
})
|
||||
rotator.start()
|
||||
|
||||
timer.advance(1000) // first attempt fires → throws
|
||||
await flush()
|
||||
expect(errors).toHaveLength(1)
|
||||
expect(rotated).toBe(0)
|
||||
|
||||
// The retry is armed at the 500ms backoff delay, NOT renewBeforeMs (1000): advancing only 500
|
||||
// must fire it. A crash-loop never escapes here (the supervisor keeps running).
|
||||
timer.advance(500)
|
||||
await flush()
|
||||
expect(rotated).toBe(1)
|
||||
expect(ks.loadCert()!.certPem).toBe('NEWCERT')
|
||||
rotator.stop()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user