fix(tunnel): break the expired-leaf renewal deadlock

`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
Yaojia Wang
2026-07-29 09:41:03 +02:00
parent b1bc50ccd1
commit f3f4d8baa6
11 changed files with 643 additions and 16 deletions

View File

@@ -6,7 +6,9 @@ import type { AgentConfig } from '../src/config/agentConfig.js'
import { generateIdentity } from '../src/keys/identity.js'
import { openKeystore } from '../src/keys/keystore.js'
import {
CertExpiredBeyondGraceError,
CertExpiredError,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
NotEnrolledError,
buildTlsOptions,
dialRelay,
@@ -65,6 +67,82 @@ describe('buildTlsOptions (T12, INV14/INV4)', () => {
})
})
/**
* Expired-leaf RENEWAL grace. The tunnel dial stays fail-closed forever; only the /renew recovery
* path may opt in, and only inside a bounded window — otherwise an expired leaf deadlocks the agent
* (mTLS renew needs a cert, and the only cert has expired) and the host is bricked until a re-pair.
*/
describe('buildTlsOptions expired-renewal grace (deadlock recovery)', () => {
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
const DAY = 86_400_000
it('stays fail-closed by DEFAULT — no grace opt-in ⇒ CertExpiredError', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(DAY) })).toThrow(CertExpiredError)
rmSync(dir, { recursive: true, force: true })
})
it('expired INSIDE the grace window → returns options so /renew can authenticate', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: expiredBy(8 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
expect(tls.rejectUnauthorized).toBe(true)
rmSync(dir, { recursive: true, force: true })
})
it('expired BEYOND the grace window → CertExpiredBeyondGraceError (re-pair required)', () => {
const { dir, ks } = enrolledKs()
expect(() =>
buildTlsOptions(ks, {
certParser: expiredBy(31 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
}),
).toThrow(CertExpiredBeyondGraceError)
rmSync(dir, { recursive: true, force: true })
})
it('grace of 0 is not a bypass — an expired leaf still throws', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(1000), expiredGraceMs: 0 })).toThrow(
CertExpiredError,
)
rmSync(dir, { recursive: true, force: true })
})
it('a still-valid cert is unaffected by the grace option', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: future,
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
rmSync(dir, { recursive: true, force: true })
})
it('the TUNNEL dial never gets grace — an expired leaf still refuses to dial', () => {
const { dir, ks } = enrolledKs()
let constructed = 0
class NeverCtor implements RawTlsWs {
constructor() {
constructed += 1
}
send(): void {}
close(): void {}
on(): void {}
once(): void {}
}
// `dialRelay` has no grace seam at all: it fails fast (synchronously) before any socket exists.
expect(() =>
dialRelay(CFG, ks, { Ctor: NeverCtor as unknown as TlsWsConstructor, certParser: past }),
).toThrow(CertExpiredError)
expect(constructed).toBe(0)
rmSync(dir, { recursive: true, force: true })
})
})
describe('dialRelay (T12)', () => {
it('constructs the wss client with the TLS options and resolves on open', async () => {
const { dir, ks } = enrolledKs()