fix(tunnel): break the expired-leaf renewal deadlock

`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
Yaojia Wang
2026-07-29 09:41:03 +02:00
parent b1bc50ccd1
commit f3f4d8baa6
11 changed files with 643 additions and 16 deletions

View File

@@ -27,6 +27,10 @@ import {
type MtlsRequest,
} from '../src/certs/nativeRenew.js'
import { FakeTimer } from './fixtures/fakes.js'
import {
CertExpiredBeyondGraceError,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
} from '../src/transport/dial.js'
const CFG: AgentConfig = {
relayUrl: 'wss://relay/agent',
@@ -109,11 +113,21 @@ describe('createMtlsFetch (A5)', () => {
describe('wireAutoRenew (A5)', () => {
function fakeRotator(): {
rotator: CertRotator
fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void }
fire: {
rotated?: () => void
revoked?: () => void
error?: (e: unknown) => void
exhausted?: (e: CertExpiredBeyondGraceError) => void
}
start: ReturnType<typeof vi.fn>
stop: ReturnType<typeof vi.fn>
} {
const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {}
const fire: {
rotated?: () => void
revoked?: () => void
error?: (e: unknown) => void
exhausted?: (e: CertExpiredBeyondGraceError) => void
} = {}
const start = vi.fn()
const stop = vi.fn()
const rotator: CertRotator = {
@@ -128,6 +142,9 @@ describe('wireAutoRenew (A5)', () => {
onError: (cb) => {
fire.error = cb
},
onExhausted: (cb) => {
fire.exhausted = cb
},
}
return { rotator, fire, start, stop }
}
@@ -277,3 +294,72 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => {
rmSync(dir, { recursive: true, force: true })
})
})
/**
* Expired-leaf recovery. `createMtlsFetch` is the ONE place that decides whether a lapsed leaf may
* still be presented; if it keeps refusing (the pre-fix behaviour) the renewal can never leave the
* host and the tunnel stays dead until a manual re-pair.
*/
describe('createMtlsFetch expired-leaf recovery', () => {
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
const DAY = 86_400_000
it('still presents a leaf that expired INSIDE the grace window (renewal can go out)', async () => {
const { dir, ks } = enrolledKs()
let presented = ''
const request: MtlsRequest = async (_url, tls) => {
presented = tls.cert
return { status: 201, body: '{}' }
}
const f = createMtlsFetch(ks, { request, certParser: expiredBy(8 * DAY) })
const res = await f('https://recover.example.com/renew', { method: 'POST', body: '{}' })
expect(res.status).toBe(201)
expect(presented).toBe('LEAFCERT')
rmSync(dir, { recursive: true, force: true })
})
it('refuses once the leaf is BEYOND the grace window (terminal — re-pair required)', async () => {
const { dir, ks } = enrolledKs()
const request: MtlsRequest = async () => ({ status: 201, body: '{}' })
const f = createMtlsFetch(ks, {
request,
certParser: expiredBy(DEFAULT_EXPIRED_RENEW_GRACE_MS + DAY),
})
await expect(f('https://recover.example.com/renew', { method: 'POST' })).rejects.toThrow(
CertExpiredBeyondGraceError,
)
rmSync(dir, { recursive: true, force: true })
})
})
describe('wireAutoRenew exhausted routing', () => {
it('logs an actionable re-pair alarm and leaves the supervisor running', () => {
const fire: { exhausted?: (e: CertExpiredBeyondGraceError) => void } = {}
const rotator: CertRotator = {
start: vi.fn(),
stop: vi.fn(),
onRotated: () => {},
onRevoked: () => {},
onError: () => {},
onExhausted: (cb) => {
fire.exhausted = cb
},
}
const restartChild = vi.fn()
const stop = vi.fn()
const lines: string[] = []
wireAutoRenew(rotator, { restartChild, stop }, createLogger('info', (l) => lines.push(l)), {
subdomain: 'h7fd8',
hostId: 'h-1',
})
fire.exhausted?.(new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000))
const alarm = lines.find((l) => /re-pair/i.test(l))
expect(alarm).toBeDefined()
expect(alarm).toContain('h7fd8')
// The supervisor keeps running: a later `pair` writes fresh cert files that the restarting
// frpc child picks up. Tearing down here would make recovery need a manual restart too.
expect(stop).not.toHaveBeenCalled()
expect(restartChild).not.toHaveBeenCalled()
})
})