Files
web-terminal/agent/test/dial.test.ts
Yaojia Wang f3f4d8baa6 fix(tunnel): break the expired-leaf renewal deadlock
`POST /renew` is authenticated by mTLS with the very leaf it renews, so once
that leaf lapsed the host could never renew it and the tunnel stayed down until
an operator re-paired by hand. Production hit exactly this: the Mac slept
through its 8h renewal window, the 24h leaf expired, and the agent then logged
`client certificate has expired; renew before dialling` 6380 times over 8 days
without recovering.

Three layers independently refused an expired leaf, so all three had to move:

- agent `buildTlsOptions` gains an opt-in `expiredGraceMs`. Absent/0 keeps the
  historical fail-closed behaviour, and the TUNNEL dial never passes it — only
  the renew transport does. Past the window it throws the new
  `CertExpiredBeyondGraceError`.
- agent rotator routes an already-expired leaf to a separate recovery endpoint
  and treats "beyond grace" as TERMINAL: report once via `onExhausted`, stop
  retrying, and name the fix (re-pair) instead of spamming warnings forever.
- control-plane `assertPresentedCertTrusted` grants a bounded grace on
  `notAfter` only. Chain validation, SPIFFE identity, `notBefore`, and the
  registry `active`/account checks are all unchanged, so revocation still bites.
- new `deploy/nginx/recover-mtls.conf` (:8472). The strict enroll vhost cannot
  host this: under `ssl_verify_client optional` nginx answers a bare 400 as soon
  as a presented cert fails verification, so an expired leaf never reaches the
  location — and the directive is server-level, not per-location.

Grace defaults to 30 days on both ends and is configurable (`RECOVER_URL`,
`expiredRenewGraceMs`). The honest trade is recorded in the code: a stale stolen
leaf stays reusable for the window, which widens an existing exposure (an
unexpired stolen leaf already renews indefinitely) rather than opening a new one.

Verified: agent 296/296, control-plane 286/286, tsc clean on both.
2026-07-29 09:41:03 +02:00

191 lines
6.8 KiB
TypeScript

import { describe, expect, it, vi } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type { AgentConfig } from '../src/config/agentConfig.js'
import { generateIdentity } from '../src/keys/identity.js'
import { openKeystore } from '../src/keys/keystore.js'
import {
CertExpiredBeyondGraceError,
CertExpiredError,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
NotEnrolledError,
buildTlsOptions,
dialRelay,
type RawTlsWs,
type TlsWsConstructor,
} from '../src/transport/dial.js'
const CFG: AgentConfig = {
relayUrl: 'wss://relay.example.com/agent',
enrollUrl: 'https://example.com/enroll',
stateDir: '/tmp/x',
localTargetUrl: 'ws://127.0.0.1:3000',
subdomain: 'host-42',
hostId: 'h-1',
}
function enrolledKs() {
const dir = mkdtempSync(join(tmpdir(), 'wta-dial-'))
const ks = openKeystore(dir)
ks.saveIdentity(generateIdentity())
ks.saveCert('CERTPEM', 'CAPEM')
return { dir, ks }
}
const future: () => { validTo: Date } = () => ({ validTo: new Date(Date.now() + 86_400_000) })
const past: () => { validTo: Date } = () => ({ validTo: new Date(Date.now() - 1000) })
describe('buildTlsOptions (T12, INV14/INV4)', () => {
it('wires cert + key + CA and forces rejectUnauthorized true', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, { certParser: future })
expect(tls.cert).toBe('CERTPEM')
expect(tls.ca).toBe('CAPEM')
expect(tls.key).toContain('PRIVATE KEY') // in-process key PEM
expect(tls.rejectUnauthorized).toBe(true)
rmSync(dir, { recursive: true, force: true })
})
it('carries NO bearer/agent token (mTLS is the auth)', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, { certParser: future })
expect(JSON.stringify(tls)).not.toMatch(/token|authorization|bearer/i)
rmSync(dir, { recursive: true, force: true })
})
it('missing cert → NotEnrolledError (fail-fast)', () => {
const dir = mkdtempSync(join(tmpdir(), 'wta-dial-'))
expect(() => buildTlsOptions(openKeystore(dir))).toThrow(NotEnrolledError)
rmSync(dir, { recursive: true, force: true })
})
it('expired cert → CertExpiredError', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: past })).toThrow(CertExpiredError)
rmSync(dir, { recursive: true, force: true })
})
})
/**
* Expired-leaf RENEWAL grace. The tunnel dial stays fail-closed forever; only the /renew recovery
* path may opt in, and only inside a bounded window — otherwise an expired leaf deadlocks the agent
* (mTLS renew needs a cert, and the only cert has expired) and the host is bricked until a re-pair.
*/
describe('buildTlsOptions expired-renewal grace (deadlock recovery)', () => {
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
const DAY = 86_400_000
it('stays fail-closed by DEFAULT — no grace opt-in ⇒ CertExpiredError', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(DAY) })).toThrow(CertExpiredError)
rmSync(dir, { recursive: true, force: true })
})
it('expired INSIDE the grace window → returns options so /renew can authenticate', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: expiredBy(8 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
expect(tls.rejectUnauthorized).toBe(true)
rmSync(dir, { recursive: true, force: true })
})
it('expired BEYOND the grace window → CertExpiredBeyondGraceError (re-pair required)', () => {
const { dir, ks } = enrolledKs()
expect(() =>
buildTlsOptions(ks, {
certParser: expiredBy(31 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
}),
).toThrow(CertExpiredBeyondGraceError)
rmSync(dir, { recursive: true, force: true })
})
it('grace of 0 is not a bypass — an expired leaf still throws', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(1000), expiredGraceMs: 0 })).toThrow(
CertExpiredError,
)
rmSync(dir, { recursive: true, force: true })
})
it('a still-valid cert is unaffected by the grace option', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: future,
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
rmSync(dir, { recursive: true, force: true })
})
it('the TUNNEL dial never gets grace — an expired leaf still refuses to dial', () => {
const { dir, ks } = enrolledKs()
let constructed = 0
class NeverCtor implements RawTlsWs {
constructor() {
constructed += 1
}
send(): void {}
close(): void {}
on(): void {}
once(): void {}
}
// `dialRelay` has no grace seam at all: it fails fast (synchronously) before any socket exists.
expect(() =>
dialRelay(CFG, ks, { Ctor: NeverCtor as unknown as TlsWsConstructor, certParser: past }),
).toThrow(CertExpiredError)
expect(constructed).toBe(0)
rmSync(dir, { recursive: true, force: true })
})
})
describe('dialRelay (T12)', () => {
it('constructs the wss client with the TLS options and resolves on open', async () => {
const { dir, ks } = enrolledKs()
let capturedUrl = ''
let capturedOpts: unknown
class FakeTlsWs implements RawTlsWs {
constructor(url: string, opts: unknown) {
capturedUrl = url
capturedOpts = opts
queueMicrotask(() => this.openCb?.())
}
private openCb?: () => void
send(): void {}
close(): void {}
on(): void {}
once(ev: string, cb: () => void): void {
if (ev === 'open') this.openCb = cb
}
}
const ws = await dialRelay(CFG, ks, { Ctor: FakeTlsWs as unknown as TlsWsConstructor, certParser: future })
expect(capturedUrl).toBe('wss://relay.example.com/agent')
expect((capturedOpts as { rejectUnauthorized: boolean }).rejectUnauthorized).toBe(true)
expect(ws).toBeDefined()
rmSync(dir, { recursive: true, force: true })
})
it('rejects when the server errors before open (bad chain / MITM)', async () => {
const { dir, ks } = enrolledKs()
class FakeTlsWs implements RawTlsWs {
private errCb?: (e: unknown) => void
constructor() {
queueMicrotask(() => this.errCb?.(new Error('unable to verify leaf signature')))
}
send(): void {}
close(): void {}
on(): void {}
once(ev: string, cb: (e: unknown) => void): void {
if (ev === 'error') this.errCb = cb
}
}
const p = dialRelay(CFG, ks, { Ctor: FakeTlsWs as unknown as TlsWsConstructor, certParser: future })
await expect(p).rejects.toThrow(/verify/)
rmSync(dir, { recursive: true, force: true })
})
})