fix(tunnel): break the expired-leaf renewal deadlock

`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
Yaojia Wang
2026-07-29 09:41:03 +02:00
parent b1bc50ccd1
commit f3f4d8baa6
11 changed files with 643 additions and 16 deletions

View File

@@ -22,7 +22,12 @@ import type { Keystore } from '../keys/keystore.js'
import type { Logger } from '../log/logger.js'
import type { TimerLike } from '../transport/seams.js'
import { createBackoff } from '../transport/backoff.js'
import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js'
import {
buildTlsOptions,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
type CertParser,
type TlsClientOptions,
} from '../transport/dial.js'
import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js'
import { createCertRotator, type CertRotator } from './rotation.js'
@@ -126,7 +131,7 @@ function toHeaderRecord(headers: RequestInit['headers']): Record<string, string>
*/
export function createMtlsFetch(
ks: Keystore,
opts: { request?: MtlsRequest; certParser?: CertParser } = {},
opts: { request?: MtlsRequest; certParser?: CertParser; expiredGraceMs?: number } = {},
): typeof fetch {
const request = opts.request ?? defaultMtlsRequest
const shim = async (input: Parameters<typeof fetch>[0], init?: RequestInit): Promise<Response> => {
@@ -135,7 +140,14 @@ export function createMtlsFetch(
// SYSTEM roots — its host (the LE-fronted control-plane) is publicly trusted; pinning the private
// enroll caChain here fails with "unable to get local issuer certificate". So drop `ca` (absent →
// node uses the default roots); rejectUnauthorized stays true.
const full = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) })
// This transport exists ONLY to renew, so it opts into the expired-leaf grace: refusing here is
// exactly the deadlock we are fixing (mTLS renew needs the very leaf that lapsed). Past the
// window `buildTlsOptions` throws CertExpiredBeyondGraceError, which the rotator treats as
// terminal rather than retrying forever.
const full = buildTlsOptions(ks, {
expiredGraceMs: opts.expiredGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS,
...(opts.certParser ? { certParser: opts.certParser } : {}),
})
const tls: TlsClientOptions = { cert: full.cert, key: full.key, rejectUnauthorized: full.rejectUnauthorized }
const reqInit: MtlsRequestInit = {
method: init?.method ?? 'GET',
@@ -195,6 +207,16 @@ export function wireAutoRenew(
error: errorMessage(err),
})
})
// Terminal: the grace window is spent, so every further attempt is guaranteed to fail. Say so once,
// at error level, naming the fix — and deliberately do NOT stop the supervisor: `pair` writes fresh
// cert files that the restart-on-exit frpc child picks up without a manual service restart.
rotator.onExhausted((err) => {
logger.log('error', 'frp-client cert expired beyond renewal grace — run `web-terminal-agent pair <CODE>` to re-pair this host', {
...meta,
expiredForMs: err.expiredForMs,
graceMs: err.graceMs,
})
})
rotator.start()
return { stop: () => rotator.stop() }
}
@@ -205,6 +227,8 @@ export function wireAutoRenew(
export interface NativeAutoRenewOpts {
readonly mtlsRequest?: MtlsRequest
readonly certParser?: CertParser
/** Window in which an already-expired leaf may still authenticate its own renewal. */
readonly expiredGraceMs?: number
readonly timer?: TimerLike
readonly renewBeforeMs?: number
readonly retryBaseMs?: number
@@ -232,6 +256,7 @@ export function startNativeAutoRenew(
const fetchImpl = createMtlsFetch(ks, {
...(opts.mtlsRequest ? { request: opts.mtlsRequest } : {}),
...(opts.certParser ? { certParser: opts.certParser } : {}),
...(opts.expiredGraceMs !== undefined ? { expiredGraceMs: opts.expiredGraceMs } : {}),
})
const rotator = createCertRotator(cfg, id, ks, {
fetchImpl,