fix(tunnel): break the expired-leaf renewal deadlock
`POST /renew` is authenticated by mTLS with the very leaf it renews, so once that leaf lapsed the host could never renew it and the tunnel stayed down until an operator re-paired by hand. Production hit exactly this: the Mac slept through its 8h renewal window, the 24h leaf expired, and the agent then logged `client certificate has expired; renew before dialling` 6380 times over 8 days without recovering. Three layers independently refused an expired leaf, so all three had to move: - agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the historical fail-closed behaviour, and the TUNNEL dial never passes it — only the renew transport does. Past the window it throws the new `CertExpiredBeyondGraceError`. - agent rotator routes an already-expired leaf to a separate recovery endpoint and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop retrying, and name the fix (re-pair) instead of spamming warnings forever. - control-plane `assertPresentedCertTrusted` grants a bounded grace on `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the registry `active`/account checks are all unchanged, so revocation still bites. - new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon as a presented cert fails verification, so an expired leaf never reaches the location — and the directive is server-level, not per-location. Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`, `expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen leaf stays reusable for the window, which widens an existing exposure (an unexpired stolen leaf already renews indefinitely) rather than opening a new one. Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
@@ -22,7 +22,12 @@ import type { Keystore } from '../keys/keystore.js'
|
||||
import type { Logger } from '../log/logger.js'
|
||||
import type { TimerLike } from '../transport/seams.js'
|
||||
import { createBackoff } from '../transport/backoff.js'
|
||||
import { buildTlsOptions, type CertParser, type TlsClientOptions } from '../transport/dial.js'
|
||||
import {
|
||||
buildTlsOptions,
|
||||
DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
type CertParser,
|
||||
type TlsClientOptions,
|
||||
} from '../transport/dial.js'
|
||||
import { DEFAULT_CERT_RENEW_WINDOW_MS } from '../health/probe.js'
|
||||
import { createCertRotator, type CertRotator } from './rotation.js'
|
||||
|
||||
@@ -126,7 +131,7 @@ function toHeaderRecord(headers: RequestInit['headers']): Record<string, string>
|
||||
*/
|
||||
export function createMtlsFetch(
|
||||
ks: Keystore,
|
||||
opts: { request?: MtlsRequest; certParser?: CertParser } = {},
|
||||
opts: { request?: MtlsRequest; certParser?: CertParser; expiredGraceMs?: number } = {},
|
||||
): typeof fetch {
|
||||
const request = opts.request ?? defaultMtlsRequest
|
||||
const shim = async (input: Parameters<typeof fetch>[0], init?: RequestInit): Promise<Response> => {
|
||||
@@ -135,7 +140,14 @@ export function createMtlsFetch(
|
||||
// SYSTEM roots — its host (the LE-fronted control-plane) is publicly trusted; pinning the private
|
||||
// enroll caChain here fails with "unable to get local issuer certificate". So drop `ca` (absent →
|
||||
// node uses the default roots); rejectUnauthorized stays true.
|
||||
const full = buildTlsOptions(ks, { ...(opts.certParser ? { certParser: opts.certParser } : {}) })
|
||||
// This transport exists ONLY to renew, so it opts into the expired-leaf grace: refusing here is
|
||||
// exactly the deadlock we are fixing (mTLS renew needs the very leaf that lapsed). Past the
|
||||
// window `buildTlsOptions` throws CertExpiredBeyondGraceError, which the rotator treats as
|
||||
// terminal rather than retrying forever.
|
||||
const full = buildTlsOptions(ks, {
|
||||
expiredGraceMs: opts.expiredGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
...(opts.certParser ? { certParser: opts.certParser } : {}),
|
||||
})
|
||||
const tls: TlsClientOptions = { cert: full.cert, key: full.key, rejectUnauthorized: full.rejectUnauthorized }
|
||||
const reqInit: MtlsRequestInit = {
|
||||
method: init?.method ?? 'GET',
|
||||
@@ -195,6 +207,16 @@ export function wireAutoRenew(
|
||||
error: errorMessage(err),
|
||||
})
|
||||
})
|
||||
// Terminal: the grace window is spent, so every further attempt is guaranteed to fail. Say so once,
|
||||
// at error level, naming the fix — and deliberately do NOT stop the supervisor: `pair` writes fresh
|
||||
// cert files that the restart-on-exit frpc child picks up without a manual service restart.
|
||||
rotator.onExhausted((err) => {
|
||||
logger.log('error', 'frp-client cert expired beyond renewal grace — run `web-terminal-agent pair <CODE>` to re-pair this host', {
|
||||
...meta,
|
||||
expiredForMs: err.expiredForMs,
|
||||
graceMs: err.graceMs,
|
||||
})
|
||||
})
|
||||
rotator.start()
|
||||
return { stop: () => rotator.stop() }
|
||||
}
|
||||
@@ -205,6 +227,8 @@ export function wireAutoRenew(
|
||||
export interface NativeAutoRenewOpts {
|
||||
readonly mtlsRequest?: MtlsRequest
|
||||
readonly certParser?: CertParser
|
||||
/** Window in which an already-expired leaf may still authenticate its own renewal. */
|
||||
readonly expiredGraceMs?: number
|
||||
readonly timer?: TimerLike
|
||||
readonly renewBeforeMs?: number
|
||||
readonly retryBaseMs?: number
|
||||
@@ -232,6 +256,7 @@ export function startNativeAutoRenew(
|
||||
const fetchImpl = createMtlsFetch(ks, {
|
||||
...(opts.mtlsRequest ? { request: opts.mtlsRequest } : {}),
|
||||
...(opts.certParser ? { certParser: opts.certParser } : {}),
|
||||
...(opts.expiredGraceMs !== undefined ? { expiredGraceMs: opts.expiredGraceMs } : {}),
|
||||
})
|
||||
const rotator = createCertRotator(cfg, id, ks, {
|
||||
fetchImpl,
|
||||
|
||||
Reference in New Issue
Block a user