fix(tunnel): break the expired-leaf renewal deadlock
`POST /renew` is authenticated by mTLS with the very leaf it renews, so once that leaf lapsed the host could never renew it and the tunnel stayed down until an operator re-paired by hand. Production hit exactly this: the Mac slept through its 8h renewal window, the 24h leaf expired, and the agent then logged `client certificate has expired; renew before dialling` 6380 times over 8 days without recovering. Three layers independently refused an expired leaf, so all three had to move: - agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the historical fail-closed behaviour, and the TUNNEL dial never passes it — only the renew transport does. Past the window it throws the new `CertExpiredBeyondGraceError`. - agent rotator routes an already-expired leaf to a separate recovery endpoint and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop retrying, and name the fix (re-pair) instead of spamming warnings forever. - control-plane `assertPresentedCertTrusted` grants a bounded grace on `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the registry `active`/account checks are all unchanged, so revocation still bites. - new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon as a presented cert fails verification, so an expired leaf never reaches the location — and the directive is server-level, not per-location. Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`, `expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen leaf stays reusable for the window, which widens an existing exposure (an unexpired stolen leaf already renews indefinitely) rather than opening a new one. Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
@@ -6,7 +6,9 @@ import type { AgentConfig } from '../src/config/agentConfig.js'
|
||||
import { generateIdentity } from '../src/keys/identity.js'
|
||||
import { openKeystore } from '../src/keys/keystore.js'
|
||||
import {
|
||||
CertExpiredBeyondGraceError,
|
||||
CertExpiredError,
|
||||
DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
NotEnrolledError,
|
||||
buildTlsOptions,
|
||||
dialRelay,
|
||||
@@ -65,6 +67,82 @@ describe('buildTlsOptions (T12, INV14/INV4)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* Expired-leaf RENEWAL grace. The tunnel dial stays fail-closed forever; only the /renew recovery
|
||||
* path may opt in, and only inside a bounded window — otherwise an expired leaf deadlocks the agent
|
||||
* (mTLS renew needs a cert, and the only cert has expired) and the host is bricked until a re-pair.
|
||||
*/
|
||||
describe('buildTlsOptions expired-renewal grace (deadlock recovery)', () => {
|
||||
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
|
||||
const DAY = 86_400_000
|
||||
|
||||
it('stays fail-closed by DEFAULT — no grace opt-in ⇒ CertExpiredError', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
expect(() => buildTlsOptions(ks, { certParser: expiredBy(DAY) })).toThrow(CertExpiredError)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('expired INSIDE the grace window → returns options so /renew can authenticate', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const tls = buildTlsOptions(ks, {
|
||||
certParser: expiredBy(8 * DAY),
|
||||
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
})
|
||||
expect(tls.cert).toBe('CERTPEM')
|
||||
expect(tls.rejectUnauthorized).toBe(true)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('expired BEYOND the grace window → CertExpiredBeyondGraceError (re-pair required)', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
expect(() =>
|
||||
buildTlsOptions(ks, {
|
||||
certParser: expiredBy(31 * DAY),
|
||||
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
}),
|
||||
).toThrow(CertExpiredBeyondGraceError)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('grace of 0 is not a bypass — an expired leaf still throws', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
expect(() => buildTlsOptions(ks, { certParser: expiredBy(1000), expiredGraceMs: 0 })).toThrow(
|
||||
CertExpiredError,
|
||||
)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('a still-valid cert is unaffected by the grace option', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const tls = buildTlsOptions(ks, {
|
||||
certParser: future,
|
||||
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
})
|
||||
expect(tls.cert).toBe('CERTPEM')
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('the TUNNEL dial never gets grace — an expired leaf still refuses to dial', () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
let constructed = 0
|
||||
class NeverCtor implements RawTlsWs {
|
||||
constructor() {
|
||||
constructed += 1
|
||||
}
|
||||
send(): void {}
|
||||
close(): void {}
|
||||
on(): void {}
|
||||
once(): void {}
|
||||
}
|
||||
// `dialRelay` has no grace seam at all: it fails fast (synchronously) before any socket exists.
|
||||
expect(() =>
|
||||
dialRelay(CFG, ks, { Ctor: NeverCtor as unknown as TlsWsConstructor, certParser: past }),
|
||||
).toThrow(CertExpiredError)
|
||||
expect(constructed).toBe(0)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
describe('dialRelay (T12)', () => {
|
||||
it('constructs the wss client with the TLS options and resolves on open', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
|
||||
Reference in New Issue
Block a user