fix(tunnel): break the expired-leaf renewal deadlock
`POST /renew` is authenticated by mTLS with the very leaf it renews, so once that leaf lapsed the host could never renew it and the tunnel stayed down until an operator re-paired by hand. Production hit exactly this: the Mac slept through its 8h renewal window, the 24h leaf expired, and the agent then logged `client certificate has expired; renew before dialling` 6380 times over 8 days without recovering. Three layers independently refused an expired leaf, so all three had to move: - agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the historical fail-closed behaviour, and the TUNNEL dial never passes it — only the renew transport does. Past the window it throws the new `CertExpiredBeyondGraceError`. - agent rotator routes an already-expired leaf to a separate recovery endpoint and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop retrying, and name the fix (re-pair) instead of spamming warnings forever. - control-plane `assertPresentedCertTrusted` grants a bounded grace on `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the registry `active`/account checks are all unchanged, so revocation still bites. - new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon as a presented cert fails verification, so an expired leaf never reaches the location — and the directive is server-level, not per-location. Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`, `expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen leaf stays reusable for the window, which widens an existing exposure (an unexpired stolen leaf already renews indefinitely) rather than opening a new one. Verified: agent 296/296, control-plane 286/286, tsc clean on both.
This commit is contained in:
@@ -27,6 +27,10 @@ import {
|
||||
type MtlsRequest,
|
||||
} from '../src/certs/nativeRenew.js'
|
||||
import { FakeTimer } from './fixtures/fakes.js'
|
||||
import {
|
||||
CertExpiredBeyondGraceError,
|
||||
DEFAULT_EXPIRED_RENEW_GRACE_MS,
|
||||
} from '../src/transport/dial.js'
|
||||
|
||||
const CFG: AgentConfig = {
|
||||
relayUrl: 'wss://relay/agent',
|
||||
@@ -109,11 +113,21 @@ describe('createMtlsFetch (A5)', () => {
|
||||
describe('wireAutoRenew (A5)', () => {
|
||||
function fakeRotator(): {
|
||||
rotator: CertRotator
|
||||
fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void }
|
||||
fire: {
|
||||
rotated?: () => void
|
||||
revoked?: () => void
|
||||
error?: (e: unknown) => void
|
||||
exhausted?: (e: CertExpiredBeyondGraceError) => void
|
||||
}
|
||||
start: ReturnType<typeof vi.fn>
|
||||
stop: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const fire: { rotated?: () => void; revoked?: () => void; error?: (e: unknown) => void } = {}
|
||||
const fire: {
|
||||
rotated?: () => void
|
||||
revoked?: () => void
|
||||
error?: (e: unknown) => void
|
||||
exhausted?: (e: CertExpiredBeyondGraceError) => void
|
||||
} = {}
|
||||
const start = vi.fn()
|
||||
const stop = vi.fn()
|
||||
const rotator: CertRotator = {
|
||||
@@ -128,6 +142,9 @@ describe('wireAutoRenew (A5)', () => {
|
||||
onError: (cb) => {
|
||||
fire.error = cb
|
||||
},
|
||||
onExhausted: (cb) => {
|
||||
fire.exhausted = cb
|
||||
},
|
||||
}
|
||||
return { rotator, fire, start, stop }
|
||||
}
|
||||
@@ -277,3 +294,72 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* Expired-leaf recovery. `createMtlsFetch` is the ONE place that decides whether a lapsed leaf may
|
||||
* still be presented; if it keeps refusing (the pre-fix behaviour) the renewal can never leave the
|
||||
* host and the tunnel stays dead until a manual re-pair.
|
||||
*/
|
||||
describe('createMtlsFetch expired-leaf recovery', () => {
|
||||
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
|
||||
const DAY = 86_400_000
|
||||
|
||||
it('still presents a leaf that expired INSIDE the grace window (renewal can go out)', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
let presented = ''
|
||||
const request: MtlsRequest = async (_url, tls) => {
|
||||
presented = tls.cert
|
||||
return { status: 201, body: '{}' }
|
||||
}
|
||||
const f = createMtlsFetch(ks, { request, certParser: expiredBy(8 * DAY) })
|
||||
const res = await f('https://recover.example.com/renew', { method: 'POST', body: '{}' })
|
||||
expect(res.status).toBe(201)
|
||||
expect(presented).toBe('LEAFCERT')
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('refuses once the leaf is BEYOND the grace window (terminal — re-pair required)', async () => {
|
||||
const { dir, ks } = enrolledKs()
|
||||
const request: MtlsRequest = async () => ({ status: 201, body: '{}' })
|
||||
const f = createMtlsFetch(ks, {
|
||||
request,
|
||||
certParser: expiredBy(DEFAULT_EXPIRED_RENEW_GRACE_MS + DAY),
|
||||
})
|
||||
await expect(f('https://recover.example.com/renew', { method: 'POST' })).rejects.toThrow(
|
||||
CertExpiredBeyondGraceError,
|
||||
)
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
describe('wireAutoRenew exhausted routing', () => {
|
||||
it('logs an actionable re-pair alarm and leaves the supervisor running', () => {
|
||||
const fire: { exhausted?: (e: CertExpiredBeyondGraceError) => void } = {}
|
||||
const rotator: CertRotator = {
|
||||
start: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
onRotated: () => {},
|
||||
onRevoked: () => {},
|
||||
onError: () => {},
|
||||
onExhausted: (cb) => {
|
||||
fire.exhausted = cb
|
||||
},
|
||||
}
|
||||
const restartChild = vi.fn()
|
||||
const stop = vi.fn()
|
||||
const lines: string[] = []
|
||||
wireAutoRenew(rotator, { restartChild, stop }, createLogger('info', (l) => lines.push(l)), {
|
||||
subdomain: 'h7fd8',
|
||||
hostId: 'h-1',
|
||||
})
|
||||
fire.exhausted?.(new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000))
|
||||
|
||||
const alarm = lines.find((l) => /re-pair/i.test(l))
|
||||
expect(alarm).toBeDefined()
|
||||
expect(alarm).toContain('h7fd8')
|
||||
// The supervisor keeps running: a later `pair` writes fresh cert files that the restarting
|
||||
// frpc child picks up. Tearing down here would make recovery need a manual restart too.
|
||||
expect(stop).not.toHaveBeenCalled()
|
||||
expect(restartChild).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user