diff --git a/agent/src/certs/nativeRenew.ts b/agent/src/certs/nativeRenew.ts index ac92762..03b91fa 100644 --- a/agent/src/certs/nativeRenew.ts +++ b/agent/src/certs/nativeRenew.ts @@ -24,7 +24,11 @@ import type { TimerLike } from '../transport/seams.js' import { createBackoff } from '../transport/backoff.js' import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js' import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js' -import { createCertRotator, type CertRotator } from './rotation.js' +import { + createCertRotator, + type CertExpiredBeyondGraceError, + type CertRotator, +} from './rotation.js' /** Non-secret message from an unknown thrown value (never serializes cert/key material). */ function errorMessage(err: unknown): string { @@ -135,6 +139,8 @@ export function createMtlsFetch( // SYSTEM roots — its host (the LE-fronted control-plane) is publicly trusted; pinning the private // enroll caChain here fails with "unable to get local issuer certificate". So drop `ca` (absent → // node uses the default roots); rejectUnauthorized stays true. + // Deliberately still fail-closed on an EXPIRED leaf: nginx would refuse to forward it anyway, so + // a lapsed leaf is routed to the plain `/recover` endpoint by the rotator instead of through here. const full = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) }) const tls: TlsClientOptions = { cert: full.cert, key: full.key, rejectUnauthorized: full.rejectUnauthorized } const reqInit: MtlsRequestInit = { @@ -195,6 +201,16 @@ export function wireAutoRenew( error: errorMessage(err), }) }) + // Terminal: the grace window is spent, so every further attempt is guaranteed to fail. Say so once, + // at error level, naming the fix — and deliberately do NOT stop the supervisor: `pair` writes fresh + // cert files that the restart-on-exit frpc child picks up without a manual service restart. + rotator.onExhausted((err) => { + logger.log('error', 'frp-client cert expired beyond recovery grace — run `web-terminal-agent pair ` to re-pair this host', { + ...meta, + expiredForMs: err.expiredForMs, + graceMs: err.graceMs, + }) + }) rotator.start() return { stop: () => rotator.stop() } } @@ -205,6 +221,10 @@ export function wireAutoRenew( export interface NativeAutoRenewOpts { readonly mtlsRequest?: MtlsRequest readonly certParser?: CertParser + /** Window in which an already-expired leaf may still be recovered via `/recover`. */ + readonly expiredGraceMs?: number + /** Plain (NON-mTLS) fetch for the `/recover` call; unset ⇒ global fetch. */ + readonly recoverFetchImpl?: typeof fetch readonly timer?: TimerLike readonly renewBeforeMs?: number readonly retryBaseMs?: number @@ -235,6 +255,8 @@ export function startNativeAutoRenew( }) const rotator = createCertRotator(cfg, id, ks, { fetchImpl, + ...(opts.expiredGraceMs !== undefined ? { expiredGraceMs: opts.expiredGraceMs } : {}), + ...(opts.recoverFetchImpl ? { recoverFetchImpl: opts.recoverFetchImpl } : {}), renewBeforeMs: opts.renewBeforeMs ?? DEFAULT_CERT_RENEW_WINDOW_MS, ...(opts.timer ? { timer: opts.timer } : {}), ...(opts.now ? { now: opts.now } : {}), diff --git a/agent/src/certs/rotation.ts b/agent/src/certs/rotation.ts index 9ad15e0..20acaad 100644 --- a/agent/src/certs/rotation.ts +++ b/agent/src/certs/rotation.ts @@ -19,6 +19,30 @@ import { certResponseToPem } from './pem.js' export const DEFAULT_RENEW_BEFORE_MS = 5 * 60_000 // renew 5 min before expiry +/** + * How long after `notAfter` a lapsed leaf may still be swapped for a fresh one (30 days). + * + * `/renew` is mTLS-authenticated by the very leaf it renews, so a lapsed leaf cannot renew itself — + * a deadlock that bricked a host for 8 days in production (the laptop slept through its renewal + * window, then the agent logged `client certificate has expired` 6380 times and never recovered). + * Inside this window the agent switches to the `/recover` endpoint instead; past it, only a re-pair + * can help and the rotator says so once and stops. + */ +export const DEFAULT_EXPIRED_RENEW_GRACE_MS = 30 * 24 * 60 * 60 * 1000 + +/** The leaf lapsed longer ago than the recovery grace allows ⇒ operator must re-pair this host. */ +export class CertExpiredBeyondGraceError extends Error { + constructor( + /** How long ago the leaf expired (ms) — non-secret, safe to log. */ + readonly expiredForMs: number, + /** The grace window that was exceeded (ms). */ + readonly graceMs: number, + ) { + super('client certificate expired beyond the recovery grace window; re-pair required') + this.name = 'CertExpiredBeyondGraceError' + } +} + export interface CertRotator { start(): void stop(): void @@ -26,6 +50,11 @@ export interface CertRotator { onRevoked(cb: () => void): void /** A renewal attempt failed (network/HTTP, NOT a 403 revoke). The rotator retries with backoff. */ onError(cb: (err: unknown) => void): void + /** + * TERMINAL: the leaf expired past the renewal grace window, so no future attempt can succeed. The + * rotator has stopped; recovery requires an operator re-pair. + */ + onExhausted(cb: (err: CertExpiredBeyondGraceError) => void): void } export type RenewOutcome = 'rotated' | 'revoked' @@ -35,6 +64,23 @@ export function renewalUrlFor(cfg: AgentConfig): string { return cfg.enrollUrl.replace(/\/enroll$/, '/renew') } +/** + * Recovery route for a leaf that has ALREADY EXPIRED — a sibling PATH on the same enroll host. + * + * It cannot be `/renew`, because nginx will not forward an expired client certificate at all: under + * `ssl_verify_client optional` it answers a bare `400 Bad Request`, and `optional_no_ca` does not + * help either — nginx only tolerates CHAIN errors there (`ngx_ssl_verify_error_optional` covers + * self-signed / unknown-issuer / unverifiable-leaf, NOT `X509_V_ERR_CERT_HAS_EXPIRED`). So recovery + * drops mTLS entirely: it is a plain HTTPS POST carrying the expired cert in the BODY. Nothing is + * lost by that — the accompanying CSR is self-signed by the same private key, and the control-plane + * signer already enforces CSR proof-of-possession plus `CSR key == registered key`, so possession is + * proven exactly as the TLS handshake used to prove it. + */ +export function recoveryUrlFor(cfg: AgentConfig): string { + if (cfg.recoverUrl != null && cfg.recoverUrl.length > 0) return cfg.recoverUrl + return cfg.enrollUrl.replace(/\/enroll$/, '/recover') +} + /** Ms until (validTo − renewBeforeMs), clamped to ≥ 0. */ export function computeRenewDelayMs( certPem: string, @@ -56,9 +102,10 @@ export async function renewCert( id: AgentIdentity, ks: Keystore, fetchImpl: typeof fetch, + opts: { url?: string } = {}, ): Promise { const csr = buildCsr(id, cfg.subdomain ?? 'web-terminal-agent') - const res = await fetchImpl(renewalUrlFor(cfg), { + const res = await fetchImpl(opts.url ?? renewalUrlFor(cfg), { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ csr }), @@ -73,6 +120,34 @@ export async function renewCert( return 'rotated' } +/** + * One recovery round-trip for an EXPIRED leaf: plain HTTPS (no client cert — see `recoveryUrlFor`) + * POSTing the expired cert alongside a fresh CSR over the SAME key. Same outcome contract as + * `renewCert`: 'rotated' installs the new leaf, 403 ⇒ 'revoked', anything else throws to the retry. + */ +export async function recoverCert( + cfg: AgentConfig, + id: AgentIdentity, + ks: Keystore, + fetchImpl: typeof fetch, + url: string = recoveryUrlFor(cfg), +): Promise { + const certs = ks.loadCert() + if (certs === null) throw new Error('cannot recover without the expired leaf on disk') + const csr = buildCsr(id, cfg.subdomain ?? 'web-terminal-agent') + const res = await fetchImpl(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cert: certs.certPem, csr }), + }) + if (res.status === 403) return 'revoked' + if (!res.ok) throw new Error(`cert recovery failed: HTTP ${res.status}`) + const json = (await res.json()) as { cert?: unknown; caChain?: unknown } + const { certPem, caChainPem } = certResponseToPem(json.cert, json.caChain) + ks.saveCert(certPem, caChainPem) + return 'rotated' +} + export function createCertRotator( cfg: AgentConfig, id: AgentIdentity, @@ -85,6 +160,10 @@ export function createCertRotator( parseCert?: (pem: string) => Date /** Backoff policy for retrying a FAILED renewal (default 1s→30s). Reset after a success. */ retryBackoff?: BackoffPolicy + /** Plain (NON-mTLS) fetch used only for the expired-leaf `/recover` call. */ + recoverFetchImpl?: typeof fetch + /** Window in which an expired leaf may still be recovered. 0 ⇒ no recovery at all. */ + expiredGraceMs?: number } = {}, ): CertRotator { const renewBeforeMs = opts.renewBeforeMs ?? DEFAULT_RENEW_BEFORE_MS @@ -96,12 +175,15 @@ export function createCertRotator( clearInterval: (h) => clearInterval(h as ReturnType), } const doFetch = opts.fetchImpl ?? fetch + const recoverFetch = opts.recoverFetchImpl ?? fetch + const expiredGraceMs = opts.expiredGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS const now = opts.now ?? (() => new Date()) const retryBackoff = opts.retryBackoff ?? createBackoff({ jitter: true }) let handle: unknown = null let rotatedCb: (() => void) | null = null let revokedCb: (() => void) | null = null let errorCb: ((err: unknown) => void) | null = null + let exhaustedCb: ((err: CertExpiredBeyondGraceError) => void) | null = null function schedule(): void { const certs = ks.loadCert() @@ -110,8 +192,34 @@ export function createCertRotator( handle = timer.setTimeout(runRenewal, delay) } + /** + * How this attempt must be made, derived from how stale the leaf on disk is: + * `normal` — still valid ⇒ ordinary mTLS `/renew`; + * `recover` — expired but inside the grace window ⇒ plain `/recover` with the cert in the body; + * `exhausted` — expired past the grace window ⇒ nothing can succeed; only an operator re-pair. + */ + function attemptPlan(): { mode: 'normal' | 'recover' | 'exhausted'; expiredForMs: number } { + const certs = ks.loadCert() + if (certs === null) return { mode: 'normal', expiredForMs: 0 } + const expiredForMs = now().getTime() - parseCert(certs.certPem).getTime() + if (expiredForMs <= 0) return { mode: 'normal', expiredForMs: 0 } + return { mode: expiredForMs > expiredGraceMs ? 'exhausted' : 'recover', expiredForMs } + } + function runRenewal(): void { - void renewCert(cfg, id, ks, doFetch) + const plan = attemptPlan() + if (plan.mode === 'exhausted') { + // Terminal: report ONCE and arm nothing. The old code retried forever, which is how a single + // real failure turned into 6380 identical warnings that buried the signal. + handle = null + exhaustedCb?.(new CertExpiredBeyondGraceError(plan.expiredForMs, expiredGraceMs)) + return + } + const attempt = + plan.mode === 'recover' + ? recoverCert(cfg, id, ks, recoverFetch) + : renewCert(cfg, id, ks, doFetch) + void attempt .then((outcome) => { if (outcome === 'revoked') { revokedCb?.() @@ -147,5 +255,8 @@ export function createCertRotator( onError(cb): void { errorCb = cb }, + onExhausted(cb): void { + exhaustedCb = cb + }, } } diff --git a/agent/src/config/agentConfig.ts b/agent/src/config/agentConfig.ts index 05675de..2f3945a 100644 --- a/agent/src/config/agentConfig.ts +++ b/agent/src/config/agentConfig.ts @@ -18,6 +18,13 @@ export interface AgentConfig { readonly localTargetUrl: string readonly subdomain: string | null readonly hostId: string | null + /** + * Renewal endpoint used ONLY when the current leaf has already expired (the strict `/renew` vhost + * rejects an expired client cert before it reaches the control-plane). Optional: when unset it is + * derived from `enrollUrl` by swapping the `enroll.` label for `recover.` — see + * `certs/rotation.ts` `recoveryRenewalUrlFor`. + */ + readonly recoverUrl?: string | null | undefined } /** @@ -59,6 +66,11 @@ export const AgentConfigSchema = z .refine(isLoopbackWsUrl, 'localTargetUrl must be a ws:// loopback URL (anti-SSRF)'), subdomain: z.string().min(1).nullable(), hostId: z.string().min(1).nullable(), + recoverUrl: z + .string() + .refine((u) => hasScheme(u, 'https:'), 'recoverUrl must be an https:// URL') + .nullable() + .optional(), }) .strict() .readonly() @@ -85,6 +97,7 @@ export function loadAgentConfig( localTargetUrl: argv.localTargetUrl ?? env.LOCAL_TARGET_URL ?? DEFAULT_LOCAL_TARGET, subdomain: argv.subdomain ?? env.SUBDOMAIN ?? null, hostId: argv.hostId ?? env.HOST_ID ?? null, + recoverUrl: argv.recoverUrl ?? env.RECOVER_URL ?? null, } return AgentConfigSchema.parse(merged) } diff --git a/agent/test/nativeRenew.test.ts b/agent/test/nativeRenew.test.ts index b4d3a4e..926fc0d 100644 --- a/agent/test/nativeRenew.test.ts +++ b/agent/test/nativeRenew.test.ts @@ -27,6 +27,7 @@ import { type MtlsRequest, } from '../src/certs/nativeRenew.js' import { FakeTimer } from './fixtures/fakes.js' +import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js' const CFG: AgentConfig = { relayUrl: 'wss://relay/agent', @@ -109,11 +110,21 @@ describe('createMtlsFetch (A5)', () => { describe('wireAutoRenew (A5)', () => { function fakeRotator(): { rotator: CertRotator - fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } + fire: { + rotated?: () => void + revoked?: () => void + error?: (e: unknown) => void + exhausted?: (e: CertExpiredBeyondGraceError) => void + } start: ReturnType stop: ReturnType } { - const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {} + const fire: { + rotated?: () => void + revoked?: () => void + error?: (e: unknown) => void + exhausted?: (e: CertExpiredBeyondGraceError) => void + } = {} const start = vi.fn() const stop = vi.fn() const rotator: CertRotator = { @@ -128,6 +139,9 @@ describe('wireAutoRenew (A5)', () => { onError: (cb) => { fire.error = cb }, + onExhausted: (cb) => { + fire.exhausted = cb + }, } return { rotator, fire, start, stop } } @@ -277,3 +291,60 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => { rmSync(dir, { recursive: true, force: true }) }) }) + +/** + * The mTLS renew transport stays STRICT about expiry: nginx refuses to forward an expired client + * cert at all, so an expired leaf must be routed to the plain `/recover` endpoint by the rotator + * rather than smuggled through this transport. + */ +describe('createMtlsFetch stays fail-closed on an expired leaf', () => { + it('refuses to present a lapsed leaf (recovery is the rotator\'s job, not this transport\'s)', async () => { + const { dir, ks } = enrolledKs() + let called = 0 + const request: MtlsRequest = async () => { + called += 1 + return { status: 201, body: '{}' } + } + const f = createMtlsFetch(ks, { + request, + certParser: () => ({ validTo: new Date(Date.now() - 86_400_000) }), + }) + await expect(f('https://cp.example.com/renew', { method: 'POST' })).rejects.toThrow( + /expired/i, + ) + expect(called).toBe(0) + rmSync(dir, { recursive: true, force: true }) + }) +}) + +describe('wireAutoRenew exhausted routing', () => { + it('logs an actionable re-pair alarm and leaves the supervisor running', () => { + const fire: { exhausted?: (e: CertExpiredBeyondGraceError) => void } = {} + const rotator: CertRotator = { + start: vi.fn(), + stop: vi.fn(), + onRotated: () => {}, + onRevoked: () => {}, + onError: () => {}, + onExhausted: (cb) => { + fire.exhausted = cb + }, + } + const restartChild = vi.fn() + const stop = vi.fn() + const lines: string[] = [] + wireAutoRenew(rotator, { restartChild, stop }, createLogger('info', (l) => lines.push(l)), { + subdomain: 'h7fd8', + hostId: 'h-1', + }) + fire.exhausted?.(new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000)) + + const alarm = lines.find((l) => /re-pair/i.test(l)) + expect(alarm).toBeDefined() + expect(alarm).toContain('h7fd8') + // The supervisor keeps running: a later `pair` writes fresh cert files that the restarting + // frpc child picks up. Tearing down here would make recovery need a manual restart too. + expect(stop).not.toHaveBeenCalled() + expect(restartChild).not.toHaveBeenCalled() + }) +}) diff --git a/agent/test/rotation.test.ts b/agent/test/rotation.test.ts index 2567f30..befa082 100644 --- a/agent/test/rotation.test.ts +++ b/agent/test/rotation.test.ts @@ -8,9 +8,12 @@ import { openKeystore } from '../src/keys/keystore.js' import { computeRenewDelayMs, createCertRotator, + recoverCert, + recoveryUrlFor, renewCert, renewalUrlFor, } from '../src/certs/rotation.js' +import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js' import { createBackoff } from '../src/transport/backoff.js' import { FakeTimer } from './fixtures/fakes.js' @@ -159,3 +162,149 @@ describe('createCertRotator (T13)', () => { rmSync(dir, { recursive: true, force: true }) }) }) + +/** + * Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf + * it renews, so a lapsed leaf can never renew itself — and nginx will not even forward an expired + * client cert. Recovery is therefore a PLAIN POST to `/recover` carrying the expired cert in the + * body, plus a terminal signal once the grace window is spent so the agent stops retrying forever. + */ +describe('expired-leaf recovery', () => { + const HOUR = 3_600_000 + + it('derives /recover as a sibling PATH on the same enroll host', () => { + expect(recoveryUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe( + 'https://enroll.terminal.example.com/recover', + ) + }) + + it('honours an explicit recoverUrl over the derivation', () => { + expect(recoveryUrlFor({ ...CFG, recoverUrl: 'https://elsewhere.example.com/recover' })).toBe( + 'https://elsewhere.example.com/recover', + ) + }) + + it('recoverCert posts the EXPIRED cert plus a CSR, with no client cert, and installs the result', async () => { + const { dir, ks } = enrolledKs() + let seenUrl = '' + let seenBody: { cert?: string; csr?: string } = {} + const fetchImpl = (async (u: string, init: { body: string }) => { + seenUrl = u + seenBody = JSON.parse(init.body) + return jsonRes(201, { cert: 'FRESHCERT', caChain: ['FRESHCA'] }) + }) as unknown as typeof fetch + + const outcome = await recoverCert( + { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' }, + ks.loadIdentity()!, + ks, + fetchImpl, + ) + expect(outcome).toBe('rotated') + expect(seenUrl).toBe('https://enroll.terminal.example.com/recover') + expect(seenBody.cert).toBe('OLDCERT') // the lapsed leaf travels in the BODY, not the TLS layer + expect(seenBody.csr).toBeTruthy() + expect(ks.loadCert()!.certPem).toContain('FRESHCERT') + rmSync(dir, { recursive: true, force: true }) + }) + + it('a still-valid leaf uses the mTLS /renew fetch, never the recovery one', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + let renewCalls = 0 + let recoverCalls = 0 + const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, { + timer, + renewBeforeMs: 1000, + fetchImpl: (async () => { + renewCalls += 1 + return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] }) + }) as unknown as typeof fetch, + recoverFetchImpl: (async () => { + recoverCalls += 1 + return jsonRes(200, {}) + }) as unknown as typeof fetch, + now: () => new Date(0), + parseCert: () => new Date(2000), // valid at now=0 + }) + rotator.start() + timer.advance(1000) + await flush() + expect(renewCalls).toBe(1) + expect(recoverCalls).toBe(0) + rotator.stop() + rmSync(dir, { recursive: true, force: true }) + }) + + it('an expired leaf INSIDE the grace window switches to the recovery fetch', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + let renewCalls = 0 + let recoverCalls = 0 + const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, { + timer, + renewBeforeMs: 1000, + expiredGraceMs: 30 * 24 * HOUR, + fetchImpl: (async () => { + renewCalls += 1 + return jsonRes(200, {}) + }) as unknown as typeof fetch, + recoverFetchImpl: (async () => { + recoverCalls += 1 + return jsonRes(201, { cert: 'NEWCERT', caChain: ['NEWCA'] }) + }) as unknown as typeof fetch, + now: () => new Date(8 * 24 * HOUR), // 8 days after the leaf lapsed + parseCert: () => new Date(0), + }) + let rotated = 0 + rotator.onRotated(() => { + rotated += 1 + }) + rotator.start() + timer.advance(0) + await flush() + expect(recoverCalls).toBe(1) + expect(renewCalls).toBe(0) + expect(rotated).toBe(1) + rotator.stop() + rmSync(dir, { recursive: true, force: true }) + }) + + it('BEYOND the grace window it fires onExhausted, issues no request, and stops retrying', async () => { + const { dir, ks } = enrolledKs() + const timer = new FakeTimer() + let requests = 0 + const countingFetch = (async () => { + requests += 1 + return jsonRes(201, {}) + }) as unknown as typeof fetch + const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, { + timer, + renewBeforeMs: 1000, + expiredGraceMs: 30 * 24 * HOUR, + retryBackoff: createBackoff({ baseMs: 500, jitter: false }), + fetchImpl: countingFetch, + recoverFetchImpl: countingFetch, + now: () => new Date(31 * 24 * HOUR), // 31 days stale ⇒ past a 30-day grace + parseCert: () => new Date(0), + }) + const errors: unknown[] = [] + let exhausted: CertExpiredBeyondGraceError | null = null + rotator.onError((e) => errors.push(e)) + rotator.onExhausted((e) => { + exhausted = e + }) + rotator.start() + timer.advance(0) + await flush() + + expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError) + expect(requests).toBe(0) // nothing is even attempted — it cannot succeed + expect(errors).toHaveLength(0) + // Terminal: no retry armed. Retrying forever is what produced 6380 identical warnings. + timer.advance(60_000) + await flush() + expect(requests).toBe(0) + rmSync(dir, { recursive: true, force: true }) + }) +}) diff --git a/control-plane/src/api/renew.ts b/control-plane/src/api/renew.ts index 8d2c5a1..f868f75 100644 --- a/control-plane/src/api/renew.ts +++ b/control-plane/src/api/renew.ts @@ -51,12 +51,29 @@ export const DEFAULT_RENEW_RATE_WINDOW_MS = 60 * 60 * 1000 export const RENEW_RATE_MAX_IDENTITIES = 10_000 /** Max base64 length of a submitted CSR (a P-256 PKCS#10 is well under 1 KB; this is generous slack). */ export const MAX_CSR_B64_LEN = 8192 +/** Max wire length of a body-carried certificate (a P-256 leaf PEM is ~1 KB; generous slack). */ +export const MAX_PRESENTED_CERT_LEN = 16384 /** * Default header the mTLS terminator forwards the verified client cert in (base64 DER). The terminator * MUST set this from `$ssl_client_cert` AND strip any client-supplied copy — a client can never provide * its own current cert. Production wiring can swap in a socket-peer-cert resolver instead. */ export const DEFAULT_CLIENT_CERT_HEADER = 'x-client-cert' +/** + * How long after `notAfter` a presented leaf may still authenticate its OWN re-issuance (30 days). + * + * `/renew` is authenticated by the very leaf it renews, so a strict expiry check means a host whose + * leaf lapsed can never renew it — it is bricked until an operator re-pairs (this happened: a laptop + * slept through its renewal window and the tunnel stayed down for 8 days). Accepting a recently + * expired leaf FOR RE-ISSUANCE ONLY breaks that deadlock. + * + * Nothing else is relaxed: the leaf must still chain to the CA anchor set, its SPIFFE identity must + * still resolve to a registry record that is `active` and account-consistent, and `notBefore` is NOT + * graced. The residual risk is that a stale stolen leaf stays usable for this window — an attacker + * holding an unexpired stolen leaf can already renew indefinitely, so this widens an existing + * exposure rather than opening a new one. Set to 0 to restore the strict behaviour. + */ +export const DEFAULT_EXPIRED_RENEW_GRACE_MS = 30 * 24 * 60 * 60 * 1000 /** Uniform reject: 401 = no/invalid current cert (unauthenticated); 403 = cert valid but not allowed. */ export class RenewRejectError extends Error { @@ -170,23 +187,29 @@ export function headerPresentedCert(headerName: string = DEFAULT_CLIENT_CERT_HEA const raw = req.headers[name] const value = Array.isArray(raw) ? raw[0] : raw if (typeof value !== 'string' || value.length === 0) return null - try { - // The terminator may forward the verified cert as base64 DER, OR as PEM — including nginx's - // header-safe `$ssl_client_escaped_cert` (URL-encoded PEM). Normalize all three to raw DER: - // URL-decode if escaped, then strip PEM armor + whitespace to recover the base64 DER body. - let s = value.includes('%') ? safeDecodeUri(value) : value - if (s.includes('BEGIN CERTIFICATE')) { - s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '') - } - const der = Buffer.from(s, 'base64') - return der.length > 0 ? new Uint8Array(der) : null - } catch { - return null - } + return certWireToDer(value) }, } } +/** + * Normalize a certificate carried over the wire to raw DER. Accepts base64 DER, PEM, and nginx's + * header-safe `$ssl_client_escaped_cert` (URL-encoded PEM): URL-decode if escaped, then strip PEM + * armor + whitespace to recover the base64 body. Returns null on anything unparseable. + */ +export function certWireToDer(value: string): Uint8Array | null { + try { + let s = value.includes('%') ? safeDecodeUri(value) : value + if (s.includes('BEGIN CERTIFICATE')) { + s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '') + } + const der = Buffer.from(s, 'base64') + return der.length > 0 ? new Uint8Array(der) : null + } catch { + return null + } +} + /** URL-decode `value`, returning it unchanged if it is not valid percent-encoding. */ function safeDecodeUri(value: string): string { try { @@ -207,6 +230,7 @@ function assertPresentedCertTrusted( der: Uint8Array, caAnchorsDer: readonly Uint8Array[], nowMs: number, + expiredGraceMs: number, ): void { let leaf: NodeX509Certificate let anchors: NodeX509Certificate[] @@ -220,7 +244,10 @@ function assertPresentedCertTrusted( const notBefore = new Date(leaf.validFrom).getTime() const notAfter = new Date(leaf.validTo).getTime() if (Number.isNaN(notBefore) || Number.isNaN(notAfter)) throw new RenewRejectError(401) - if (nowMs < notBefore || nowMs > notAfter) throw new RenewRejectError(401) + // `notBefore` is never graced — a not-yet-valid cert is nonsense, not a recoverable lapse. Only + // `notAfter` gets the bounded renewal grace (see DEFAULT_EXPIRED_RENEW_GRACE_MS). + if (nowMs < notBefore) throw new RenewRejectError(401) + if (nowMs > notAfter + Math.max(0, expiredGraceMs)) throw new RenewRejectError(401) } /** @@ -234,7 +261,11 @@ function assertPresentedCertTrusted( function parsePresentedCertIdentity( der: Uint8Array, expectedKind: SpiffeKind, - verify?: { readonly caAnchorsDer: readonly Uint8Array[]; readonly nowMs: number }, + verify?: { + readonly caAnchorsDer: readonly Uint8Array[] + readonly nowMs: number + readonly expiredGraceMs: number + }, ): CertIdentity { let leaf: x509.X509Certificate try { @@ -242,7 +273,8 @@ function parsePresentedCertIdentity( } catch { throw new RenewRejectError(401) } - if (verify !== undefined) assertPresentedCertTrusted(der, verify.caAnchorsDer, verify.nowMs) + if (verify !== undefined) + assertPresentedCertTrusted(der, verify.caAnchorsDer, verify.nowMs, verify.expiredGraceMs) const san = leaf.getExtension(x509.SubjectAlternativeNameExtension) const uri = san?.names.toJSON().find((n) => n.type === 'url')?.value if (uri === undefined) throw new RenewRejectError(401) @@ -262,6 +294,13 @@ function parsePresentedCertIdentity( } const RenewBodySchema = z.object({ csr: z.string().min(1).max(MAX_CSR_B64_LEN) }).strict() +/** `/recover` additionally carries the EXPIRED leaf itself (base64 DER or PEM). */ +const RecoverBodySchema = z + .object({ + cert: z.string().min(1).max(MAX_PRESENTED_CERT_LEN), + csr: z.string().min(1).max(MAX_CSR_B64_LEN), + }) + .strict() export interface RenewDeps { readonly hosts: HostRegistry @@ -279,6 +318,11 @@ export interface RenewDeps { readonly hostCaAnchorsDer?: readonly Uint8Array[] /** device-CA anchor DER(s) — same role for the DEVICE renew path. */ readonly deviceCaAnchorsDer?: readonly Uint8Array[] + /** + * Window after `notAfter` in which a presented leaf may still authenticate its own re-issuance. + * Defaults to `DEFAULT_EXPIRED_RENEW_GRACE_MS`; 0 restores the strict fail-closed behaviour. + */ + readonly expiredRenewGraceMs?: number } /** Map any thrown error to a uniform HTTP reject — never leak which internal check failed. */ @@ -298,6 +342,9 @@ function sendError(reply: FastifyReply, err: unknown): void { export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync { const presentedCert = deps.presentedCert ?? headerPresentedCert() const rateLimiter = deps.rateLimiter ?? createRenewRateLimiter() + // `/renew` stays STRICT (0): the mTLS terminator would never forward an expired cert to it anyway. + // The grace belongs to `/recover`, the route built for exactly that case. + const recoverGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS return async (app) => { app.post('/renew', async (req, reply) => { @@ -307,7 +354,9 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync { const identity = parsePresentedCertIdentity( der, 'host', - deps.hostCaAnchorsDer ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now() } : undefined, + deps.hostCaAnchorsDer + ? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 } + : undefined, ) rateLimiter.check(`host:${identity.accountId}:${identity.id}`) @@ -333,6 +382,57 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync { } }) + /** + * Expired-leaf recovery. The lapsed cert arrives in the BODY because no TLS terminator will + * forward an expired client certificate (nginx `optional` → bare 400; `optional_no_ca` tolerates + * only chain errors, never `X509_V_ERR_CERT_HAS_EXPIRED`). That makes this route the SOLE + * verifier, so it runs the identical trust pipeline as `/renew` — real X.509 path validation to + * the frp-client-CA anchors, SPIFFE parse, `notBefore`, registry `active` + account match — and + * differs ONLY in allowing a bounded overrun on `notAfter`. + * + * Possession of the private key is still proven: the CSR is self-signed by it, and the host + * signer's delegated gate enforces CSR PoP plus `CSR key == registered key`. A replayed cert + * without the key therefore yields, at most, a certificate the attacker cannot authenticate with. + * The header channel is deliberately IGNORED here — on this route only the body speaks. + */ + app.post('/recover', async (req, reply) => { + try { + const body = RecoverBodySchema.parse(req.body) + const der = certWireToDer(body.cert) + if (der === null) throw new RenewRejectError(401) + const identity = parsePresentedCertIdentity( + der, + 'host', + deps.hostCaAnchorsDer + ? { + caAnchorsDer: deps.hostCaAnchorsDer, + nowMs: Date.now(), + expiredGraceMs: recoverGraceMs, + } + : undefined, + ) + rateLimiter.check(`recover:${identity.accountId}:${identity.id}`) + + const host = await deps.hosts.getHostBySubdomain(identity.id) + if (host === null || host.status === 'revoked' || host.accountId !== identity.accountId) { + throw new RenewRejectError(403) + } + + const leaf = await deps.renewer.renewHostLeaf( + host.hostId, + identity.publicKeySpki, + decodeCsrWire(body.csr), + ) + await reply.code(201).send({ + cert: bytesToBase64(leaf.cert), + caChain: leaf.caChain.map((c) => bytesToBase64(c)), + notAfter: leaf.notAfter.toISOString(), + }) + } catch (err) { + sendError(reply, err) + } + }) + app.post('/device/:id/renew', async (req, reply) => { try { const der = presentedCert.presentedCertDer(req) @@ -340,7 +440,9 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync { const identity = parsePresentedCertIdentity( der, 'device', - deps.deviceCaAnchorsDer ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now() } : undefined, + deps.deviceCaAnchorsDer + ? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 } + : undefined, ) const id = (req.params as { id: string }).id rateLimiter.check(`device:${identity.id}`) diff --git a/control-plane/test/renew.test.ts b/control-plane/test/renew.test.ts index 852e368..92505af 100644 --- a/control-plane/test/renew.test.ts +++ b/control-plane/test/renew.test.ts @@ -502,6 +502,8 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio headers: { 'x-client-cert': certHeader(expired) }, payload: { csr: b64Csr(ctx.csr) }, }) + // STRICT on this route: nginx would never forward an expired client cert here anyway, so the + // deadlock escape hatch lives on `/recover` (below) and NOT on the mTLS renewal path. expect(res.statusCode).toBe(401) }) @@ -521,3 +523,152 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio expect(res.statusCode).toBe(401) }) }) + +/** + * CP6d · POST /recover — the expired-leaf escape hatch. + * + * Unlike `/renew` this route is NOT mTLS-authenticated: nginx cannot forward an expired client cert + * (under `ssl_verify_client optional` it 400s, and `optional_no_ca` only tolerates CHAIN errors, not + * `X509_V_ERR_CERT_HAS_EXPIRED`). The lapsed cert therefore arrives in the BODY, and the + * control-plane becomes the sole verifier. These tests pin down that nothing except the `notAfter` + * bound was relaxed — chain, SPIFFE, `notBefore`, and registry status all still decide. + */ +describe('CP6d POST /recover — expired-leaf recovery', () => { + async function mintLeaf( + ctx: HostCtx, + notBefore: Date, + notAfter: Date, + signer = ctx.ca.caSigner, + ): Promise { + const spiffe = `spiffe://relay.terminal.yaojia.wang/account/${ctx.accountId}/host/${ctx.subdomain}` + return assembleCertificate({ + subjectPublicKey: ctx.spki, + subject: `CN=${ctx.subdomain}`, + issuer: ctx.ca.caCert.subjectName, + serialNumber: Uint8Array.from([0x0b]), + notBefore, + notAfter, + extensions: [ + new x509.SubjectAlternativeNameExtension([ + { type: 'dns', value: `${ctx.subdomain}.terminal.yaojia.wang` }, + { type: 'url', value: spiffe }, + ]), + new x509.BasicConstraintsExtension(false, undefined, true), + new x509.KeyUsagesExtension(x509.KeyUsageFlags.digitalSignature, true), + new x509.ExtendedKeyUsageExtension([x509.ExtendedKeyUsage.clientAuth]), + ], + signer, + sigAlg: 'ecdsa-p256', + }) + } + + const b64 = (der: Uint8Array): string => Buffer.from(der).toString('base64') + + async function post(ctx: HostCtx, cert: Uint8Array, extra?: Partial) { + const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer], ...extra })) + await app.ready() + return app.inject({ + method: 'POST', + url: '/recover', + payload: { cert: b64(cert), csr: b64Csr(ctx.csr) }, + }) + } + + test('a leaf expired INSIDE the grace window is re-issued → 201 (breaks the deadlock)', async () => { + const ctx = await hostCtx('alice') + const now = Date.now() + const res = await post(ctx, await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS))) + expect(res.statusCode).toBe(201) + expect(JSON.parse(res.payload).cert).toBeTruthy() + }) + + test('a leaf expired BEYOND the grace window is refused → 401', async () => { + const ctx = await hostCtx('alice') + const now = Date.now() + const res = await post(ctx, await mintLeaf(ctx, new Date(now - 60 * DAY_MS), new Date(now - 31 * DAY_MS))) + expect(res.statusCode).toBe(401) + }) + + test('grace 0 disables recovery entirely → 401', async () => { + const ctx = await hostCtx('alice') + const now = Date.now() + const res = await post( + ctx, + await mintLeaf(ctx, new Date(now - 2 * DAY_MS), new Date(now - DAY_MS)), + { expiredRenewGraceMs: 0 }, + ) + expect(res.statusCode).toBe(401) + }) + + /** + * THE load-bearing test for this route. nginx no longer validates the chain here, so a forged + * self-signed cert carrying a correct-looking SPIFFE SAN must be rejected by the control-plane + * alone. If this ever goes green-to-red, `/recover` becomes an unauthenticated cert vending machine. + */ + test('a self-signed cert with a FORGED SPIFFE SAN is refused → 401', async () => { + const ctx = await hostCtx('alice') + const rogueCa = await makeP256Ca('rogue-CA') + const now = Date.now() + const rogue = await mintLeaf( + ctx, + new Date(now - 9 * DAY_MS), + new Date(now - 8 * DAY_MS), + rogueCa.caSigner, + ) + const res = await post(ctx, rogue) + expect(res.statusCode).toBe(401) + }) + + test('recovery NEVER bypasses revocation — revoked host → 403', async () => { + const ctx = await hostCtx('alice') + const now = Date.now() + const cert = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS)) + const host = await ctx.hosts.getHostBySubdomain('alice') + await ctx.hosts.setHostStatus(host!.hostId, 'revoked') + const res = await post(ctx, cert) + expect(res.statusCode).toBe(403) + }) + + test('grace covers notAfter ONLY — a not-yet-valid leaf is refused → 401', async () => { + const ctx = await hostCtx('alice') + const now = Date.now() + const res = await post(ctx, await mintLeaf(ctx, new Date(now + DAY_MS), new Date(now + 2 * DAY_MS))) + expect(res.statusCode).toBe(401) + }) + + test('a still-valid leaf may also use /recover → 201', async () => { + const ctx = await hostCtx('alice') + const res = await post(ctx, ctx.currentCertDer) + expect(res.statusCode).toBe(201) + }) + + test('a missing cert field is rejected → 400 (uniform schema reject, same as a missing csr)', async () => { + const ctx = await hostCtx('alice') + const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] })) + await app.ready() + const res = await app.inject({ method: 'POST', url: '/recover', payload: { csr: b64Csr(ctx.csr) } }) + // The module answers with uniform rejects that never say which check failed: 400 for a malformed + // body, 401 for a cert that parses but is not trusted, 403 for a trusted cert that is not allowed. + expect(res.statusCode).toBe(400) + }) + + /** + * The header is the mTLS terminator's channel. On `/recover` the cert comes from the body, so a + * client-supplied header must not be able to substitute an identity. + */ + test('an x-client-cert HEADER cannot override the body identity → 401', async () => { + const ctx = await hostCtx('alice') + const rogueCa = await makeP256Ca('rogue-CA') + const now = Date.now() + const rogue = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS), rogueCa.caSigner) + const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] })) + await app.ready() + const res = await app.inject({ + method: 'POST', + url: '/recover', + headers: { 'x-client-cert': certHeader(ctx.currentCertDer) }, + payload: { cert: b64(rogue), csr: b64Csr(ctx.csr) }, + }) + expect(res.statusCode).toBe(401) + }) +}) diff --git a/deploy/nginx/enroll-recover-location.md b/deploy/nginx/enroll-recover-location.md new file mode 100644 index 0000000..6a611fe --- /dev/null +++ b/deploy/nginx/enroll-recover-location.md @@ -0,0 +1,93 @@ +# Enroll vhost addition — `/recover` (expired-leaf recovery) + +One `location` block to **merge into the existing** enroll vhost on the VPS +(`/etc/nginx/conf.d/enroll.conf`, the `127.0.0.1:8471` server). No new server, no new SNI route, no +DNS work. + +> **This is a MERGE, not a file to ship.** The enroll vhost also carries `/enroll`, `/device/enroll`, +> `/auth/login`, `/crl/`, `/renew` and `/device/:id/renew` — do not replace it. + +## Why the route exists + +`POST /renew` is mTLS-authenticated by the very leaf it renews, so once that leaf lapses the host can +never renew it and the tunnel stays down until an operator re-pairs. That is not hypothetical: a +laptop slept through its 8h renewal window, its 24h leaf expired, and the agent then logged +`client certificate has expired; renew before dialling` 6380 times over 8 days without recovering. + +## Why it cannot be fixed on `/renew` itself + +**nginx will not forward an expired client certificate, under any `ssl_verify_client` mode.** + +- `optional` → nginx answers a bare `400 The SSL certificate error` as soon as verification fails. + The request never reaches the `location`, so no `if ($ssl_client_verify …)` can rescue it. +- `optional_no_ca` → does **not** help either. It only tolerates *chain* failures; see nginx's + `ngx_ssl_verify_error_optional()`, which covers `DEPTH_ZERO_SELF_SIGNED_CERT`, + `SELF_SIGNED_CERT_IN_CHAIN`, `UNABLE_TO_GET_ISSUER_CERT_LOCALLY` and + `UNABLE_TO_VERIFY_LEAF_SIGNATURE` — and **not** `X509_V_ERR_CERT_HAS_EXPIRED`. +- `ssl_verify_client` is a `server`-level directive, so it cannot be relaxed per-location anyway. + +So recovery drops mTLS: `/recover` takes **no client certificate**, and the lapsed cert travels in +the request body instead. + +## Why that is still authenticated + +A certificate is public, so the body alone proves nothing — possession of the **private key** does, +and it is still proven end to end: + +- the accompanying CSR is **self-signed by that key**, and the host signer's delegated gate enforces + CSR proof-of-possession plus `CSR key == registered key` (`control-plane/src/ca/csr.ts` + `verifyCsrPoP`); +- `control-plane/src/api/renew.ts` runs the *same* trust pipeline as `/renew` — real X.509 path + validation to the frp-client-CA anchors, SPIFFE SAN parse, `notBefore`, and a registry lookup + requiring an `active`, account-consistent host — differing **only** in a bounded overrun allowance + on `notAfter` (`DEFAULT_EXPIRED_RENEW_GRACE_MS`, 30 days). + +Worst case for a replayed cert without the key: the attacker receives a certificate they cannot +authenticate with. Revocation still bites, via registry status. + +## The block to add + +```nginx + # Expired-leaf recovery: NO client cert (see enroll-recover-location.md). The control-plane is + # the sole verifier; strip any client-supplied cert header so only the body can speak. + location = /recover { + limit_req zone=renew_recover burst=5 nodelay; + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header x-client-cert ""; + proxy_read_timeout 60s; + } +``` + +And once, in the `http` context (top of `enroll.conf` is fine) — the route is reachable +pre-authentication and costs the control-plane an X.509 path validation, so bound it. The +control-plane additionally rate-limits per identity (`createRenewRateLimiter`, 30/hour): + +```nginx +limit_req_zone $binary_remote_addr zone=renew_recover:1m rate=10r/m; +``` + +## Deploy gate + +```bash +cp /etc/nginx/conf.d/enroll.conf{,.bak.$(date +%s)} # snapshot first +# ...merge the block above... +nginx -t && systemctl restart nginx # NEVER reload on a failed -t +``` + +`restart`, not `reload`: a graceful reload has been observed keeping old workers alive for seconds, +which makes post-deploy verification race the change. + +## Verify + +```bash +# no cert needed — an in-grace expired leaf is re-issued +curl -sS --resolve enroll.terminal.yaojia.wang:443: \ + -X POST -H 'content-type: application/json' \ + -d "{\"cert\":\"$(base64 -w0 expired.cert.pem)\",\"csr\":\"$(base64 -w0 new.csr.der)\"}" \ + https://enroll.terminal.yaojia.wang/recover # → 201 {cert,caChain,notAfter} + +# a forged self-signed cert with a correct-looking SPIFFE SAN must still be refused +# → 401 (this is the load-bearing check; nginx is no longer validating the chain here) +``` diff --git a/docs/PROGRESS_LOG.md b/docs/PROGRESS_LOG.md index 9069d46..f4f1675 100644 --- a/docs/PROGRESS_LOG.md +++ b/docs/PROGRESS_LOG.md @@ -24,6 +24,32 @@ > 新会话读到的第一块。保持准确,只描述"此刻"。 +### 🔥 [x] 隧道断了 8 天 —— 证书过期后**自动续期死锁**,已修 + 已上线(2026-07-29) + +- **现象**: 用户问"远端 relay 是否在运行"。服务端**全绿**(nginx/frps/cp-zte/panel-zte/xray/pg/redis 全 active,LE 通配证书有效至 2026-10-05,`*.terminal.yaojia.wang` 通配 DNS 正常),但 **frps `:7000` 已建立连接数 = 0**,`h7fd8.terminal.yaojia.wang` 打不通。Mac 上 frpc 在跑,每 20s 重试一次,全部 `connect to server error: EOF`(frps 在 mTLS 阶段拒)。 +- **根因(死锁)**: Mac 的 frp-client 叶证书 **2026-07-21 13:55 UTC 过期**(TTL 仅 ~24h)。`POST /renew` 是**用它自己要续的那张证书做 mTLS 鉴权**的 —— 证书一过期就再也换不了证书。触发链:续期窗口(T-8h)那天 Mac 在睡眠,醒来时 DNS 未就绪 → 24 次 `getaddrinfo ENOTFOUND enroll.terminal.yaojia.wang` 错过窗口 → 过期后改报 `client certificate has expired; renew before dialling`,**重复 6380 次、8 天、永不自愈**(agent.log 涨到 3MB)。 +- **三层都 fail-closed**,只改一层没用:① agent `transport/dial.ts` `buildTlsOptions` 抛 `CertExpiredError`,连 socket 都不开;② nginx `enroll.conf` 的 `location = /renew` 要求 `$ssl_client_verify = SUCCESS`;③ CP `api/renew.ts` `assertPresentedCertTrusted` 再查一次有效期 → 401。 +- **两个假设被实测推翻(重要,别再踩)**: + 1. `ssl_verify_client optional` 在客户端证书**验证失败时直接返回裸 400**(`400 The SSL certificate error`),请求**根本到不了 location** —— 所以任何 `if ($ssl_client_verify …)` 都救不了。 + 2. `optional_no_ca` **也不行**:它只容忍**链**错误。见 nginx 的 `ngx_ssl_verify_error_optional()` —— 只含 `DEPTH_ZERO_SELF_SIGNED_CERT` / `SELF_SIGNED_CERT_IN_CHAIN` / `UNABLE_TO_GET_ISSUER_CERT_LOCALLY` / `UNABLE_TO_VERIFY_LEAF_SIGNATURE`,**不含 `X509_V_ERR_CERT_HAS_EXPIRED`**。已在 VPS 上用真过期证书打 :8472 实测确认。结论:**nginx 在任何模式下都不会转发过期客户端证书**。 +- **最终方案(用户选定:30 天宽限窗口 + 超窗回落配对码)**: 恢复路径**放弃 mTLS**,改为**不带客户端证书的普通 HTTPS POST,过期证书放进 body**。不丢鉴权 —— **CSR 是用同一把私钥自签的**,而签名网关本来就强制 CSR PoP(`ca/csr.ts` `verifyCsrPoP`)+ `CSR key == 注册 key`,持有私钥的证明与原来 TLS 握手给的完全等价。重放一张(公开的)证书而没有私钥,最多拿到一张**自己用不了**的证书。 + - **agent**: `dial.ts` 与 mTLS renew 通道**保持严格 fail-closed**(放宽已从 TLS 层完全移除)。改由 rotator 逐次决策:有效 → mTLS `/renew`;过期但在宽限内 → 普通 `/recover`;超出宽限 → 终态 `onExhausted`,**一次告警后停止重试**(不再发任何请求),日志直接给出 `web-terminal-agent pair `。新增 `recoverCert()`、`recoveryUrlFor()`、`DEFAULT_EXPIRED_RENEW_GRACE_MS`(30d)、`CertExpiredBeyondGraceError`;`AgentConfig` 加可选 `recoverUrl`(env `RECOVER_URL`)。 + - **CP**: `/renew` **恢复严格**(grace 0,与终端器行为一致)。宽限只给新增的 `POST /recover` —— 证书从 **body** 读、**忽略 `x-client-cert` 头**,然后跑**与 `/renew` 完全相同**的信任链:frp-client-CA 锚点做真 X.509 路径验证 → SPIFFE 解析 → `notBefore`(**永不宽限**)→ registry `active` + 账户一致。**吊销照样生效**。 + - **deploy**: 不需要新 vhost / 新 SNI / 新 DNS —— 只往现有 enroll vhost 加一个 `location = /recover`(+ `limit_req` 10r/m)。文档 `deploy/nginx/enroll-recover-location.md`(含 nginx 源码引证)。 +- **验证**: + - 单测:agent **289/289**、control-plane **290/290**,两侧 `tsc --noEmit` 干净。TDD 全程先 RED 后 GREEN。 + - 承重测试:**「山寨 CA 签的、SPIFFE SAN 与真证书逐字节相同的伪造证书 → 401」**。这条最关键 —— 这条路径上 nginx 已不再验链,它一旦变红,`/recover` 就成了无鉴权发证机。已在**线上实测 401**。 + - **线上端到端自愈实测**:把那张真·过期 8 天的证书塞回 keystore → 重启 agent → 日志 `frp-client cert rotated; restarting frpc onto the fresh leaf` → 证书自动换成新的 24h 叶(Jul30 08:06)→ frpc `login to server success` / `start proxy success` → 设备证书走完整链路 `:443 → SNI → :8470 mTLS → frps → frpc → Mac base app` 拿到 **HTTP 200**(返回本机真实 session 列表)→ 健康探针 `healthy: true`。**全程零人工介入** —— 正是生产卡死 8 天的那个场景。 + - 回归:`/renew` 对过期证书仍 400(nginx 拦住,未放宽);4 区 SNI 探针(frp/通配/enroll/xray-Reality)全部未受影响。 +- **应急恢复(修复前先做的)**: 用 frp-client CA 给 h7fd8 补签了一张 7 天过桥证书(同一把私钥、同子域名、同 SPIFFE SAN,经 `openssl ca` 登记进 index.txt 因此可吊销),先把隧道救活。自愈生效后它已被换成 CP 正常签发的 24h 叶。 +- **遗留 / 待办**: + - **`agent/src/dist/buildBinary.ts` 是源码却被 `.gitignore` 的 `dist/` 规则吞掉,从未提交**(既有问题,与本次无关):任何全新 clone/worktree 都缺它 → `agent/src/index.ts` 类型检查失败、已提交的 `test/buildBinary.test.ts` 直接导入失败。建议加 `!agent/src/dist/` 例外。 + - agent 日志里 `subdomain: (none)` / `host_id: (none)`:enroll 后未把 subdomain/hostId 落进配置,导致告警缺主机标识(既有问题)。 + - 手机轨(device)目前**没有** `/recover` 对应路径,设备证书过期仍需重新 enroll;CP 侧宽限代码是通用的,补一条路由即可。 + - 24h TTL + 单次定时续期 + 笔记本睡眠 的组合仍偏脆,宽限窗口是兜底而非根治。 +- **文件**: `agent/src/certs/rotation.ts` · `agent/src/certs/nativeRenew.ts` · `agent/src/config/agentConfig.ts` · `control-plane/src/api/renew.ts` · `deploy/nginx/enroll-recover-location.md`(新) · 各自测试。 +- **commit**: `f3f4d8b`(第一版,方向对但 nginx 那条路走不通)+ `5509c81`(实测后改为 body 传证书,最终版)。 + ### 🐛 [x] Mac 版中文全变 `_` + 框线渲染错乱 —— PTY 缺 UTF-8 locale(2026-07-28) - **现象**(用户截图,Mac 版):Claude Code TUI 里所有中文变成 `_`;banner/输入框只剩零散横线,圆角边框消失。