kai/web-terminal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tunnel): recover an expired leaf over plain HTTPS, not mTLS

Browse Source 
Correction to the previous commit. Its recovery vhost could not work: nginx
will not forward an expired client certificate in ANY `ssl_verify_client` mode.
`optional` answers a bare `400 The SSL certificate error` before any location
runs, and `optional_no_ca` only tolerates CHAIN failures — see nginx's
`ngx_ssl_verify_error_optional()`, which covers self-signed / unknown-issuer /
unverifiable-leaf and NOT `X509_V_ERR_CERT_HAS_EXPIRED`. Verified live against
the deployed :8472 server, which rejected the real expired leaf.

So recovery drops mTLS instead of trying to bend it:

- agent: `buildTlsOptions` and the mTLS renew transport go back to being
  strictly fail-closed on expiry — the relaxation is gone from the TLS layer
  entirely. The rotator now decides per attempt: valid → mTLS `/renew`,
  expired-inside-grace → plain `/recover`, expired-beyond-grace → terminal
  `onExhausted` with no request issued at all.
- new `recoverCert` POSTs `{cert, csr}` with no client certificate. Possession
  of the private key is still proven: the CSR is self-signed by it and the host
  signer already enforces CSR PoP plus `CSR key == registered key`, so a
  replayed (public) cert without the key yields at most a certificate the
  attacker cannot authenticate with.
- control-plane: `/renew` is strict again (grace 0, matching the terminator).
  The grace lives on the new `POST /recover`, which reads the cert from the BODY
  and ignores the `x-client-cert` header, then runs the identical trust
  pipeline: X.509 path validation to the frp-client-CA anchors, SPIFFE parse,
  `notBefore` (never graced), registry `active` + account match. Revocation
  still bites.
- deploy: no new vhost, no new SNI, no DNS. One `location = /recover` merged
  into the existing enroll vhost, documented in
  `deploy/nginx/enroll-recover-location.md` with the nginx source citation.

The load-bearing new test is "a self-signed cert with a FORGED SPIFFE SAN is
refused → 401": nginx no longer validates the chain on this path, so that
assertion is what keeps `/recover` from being a cert vending machine.

Verified: agent 289/289, control-plane 290/290, tsc clean on both.
This commit is contained in:
Yaojia Wang
2026-07-29 09:52:17 +02:00
parent f3f4d8baa6
commit 5509c81eee
13 changed files with 559 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
agent/test/dial.test.ts
View File

@@ -6,9 +6,7 @@ import type { AgentConfig } from '../src/config/agentConfig.js'
import { generateIdentity } from '../src/keys/identity.js'
import { openKeystore } from '../src/keys/keystore.js'
import {
CertExpiredBeyondGraceError,
CertExpiredError,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
NotEnrolledError,
buildTlsOptions,
dialRelay,
@@ -67,82 +65,6 @@ describe('buildTlsOptions (T12, INV14/INV4)', () => {
})
})
/**
* Expired-leaf RENEWAL grace. The tunnel dial stays fail-closed forever; only the /renew recovery
* path may opt in, and only inside a bounded window — otherwise an expired leaf deadlocks the agent
* (mTLS renew needs a cert, and the only cert has expired) and the host is bricked until a re-pair.
*/
describe('buildTlsOptions expired-renewal grace (deadlock recovery)', () => {
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
const DAY = 86_400_000
it('stays fail-closed by DEFAULT — no grace opt-in ⇒ CertExpiredError', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(DAY) })).toThrow(CertExpiredError)
rmSync(dir, { recursive: true, force: true })
})
it('expired INSIDE the grace window → returns options so /renew can authenticate', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: expiredBy(8 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
expect(tls.rejectUnauthorized).toBe(true)
rmSync(dir, { recursive: true, force: true })
})
it('expired BEYOND the grace window → CertExpiredBeyondGraceError (re-pair required)', () => {
const { dir, ks } = enrolledKs()
expect(() =>
buildTlsOptions(ks, {
certParser: expiredBy(31 * DAY),
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
}),
).toThrow(CertExpiredBeyondGraceError)
rmSync(dir, { recursive: true, force: true })
})
it('grace of 0 is not a bypass — an expired leaf still throws', () => {
const { dir, ks } = enrolledKs()
expect(() => buildTlsOptions(ks, { certParser: expiredBy(1000), expiredGraceMs: 0 })).toThrow(
CertExpiredError,
)
rmSync(dir, { recursive: true, force: true })
})
it('a still-valid cert is unaffected by the grace option', () => {
const { dir, ks } = enrolledKs()
const tls = buildTlsOptions(ks, {
certParser: future,
expiredGraceMs: DEFAULT_EXPIRED_RENEW_GRACE_MS,
})
expect(tls.cert).toBe('CERTPEM')
rmSync(dir, { recursive: true, force: true })
})
it('the TUNNEL dial never gets grace — an expired leaf still refuses to dial', () => {
const { dir, ks } = enrolledKs()
let constructed = 0
class NeverCtor implements RawTlsWs {
constructor() {
constructed += 1
}
send(): void {}
close(): void {}
on(): void {}
once(): void {}
}
// `dialRelay` has no grace seam at all: it fails fast (synchronously) before any socket exists.
expect(() =>
dialRelay(CFG, ks, { Ctor: NeverCtor as unknown as TlsWsConstructor, certParser: past }),
).toThrow(CertExpiredError)
expect(constructed).toBe(0)
rmSync(dir, { recursive: true, force: true })
})
})
describe('dialRelay (T12)', () => {
it('constructs the wss client with the TLS options and resolves on open', async () => {
const { dir, ks } = enrolledKs()

41
agent/test/nativeRenew.test.ts
View File

@@ -27,10 +27,7 @@ import {
type MtlsRequest,
} from '../src/certs/nativeRenew.js'
import { FakeTimer } from './fixtures/fakes.js'
import {
CertExpiredBeyondGraceError,
DEFAULT_EXPIRED_RENEW_GRACE_MS,
} from '../src/transport/dial.js'
import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js'
const CFG: AgentConfig = {
relayUrl: 'wss://relay/agent',
@@ -296,38 +293,26 @@ describe('startNativeAutoRenew (A5 end-to-end)', () => {
})
/**
* Expired-leaf recovery. `createMtlsFetch` is the ONE place that decides whether a lapsed leaf may
* still be presented; if it keeps refusing (the pre-fix behaviour) the renewal can never leave the
* host and the tunnel stays dead until a manual re-pair.
* The mTLS renew transport stays STRICT about expiry: nginx refuses to forward an expired client
* cert at all, so an expired leaf must be routed to the plain `/recover` endpoint by the rotator
* rather than smuggled through this transport.
*/
describe('createMtlsFetch expired-leaf recovery', () => {
const expiredBy = (ms: number) => () => ({ validTo: new Date(Date.now() - ms) })
const DAY = 86_400_000
it('still presents a leaf that expired INSIDE the grace window (renewal can go out)', async () => {
describe('createMtlsFetch stays fail-closed on an expired leaf', () => {
it('refuses to present a lapsed leaf (recovery is the rotator\'s job, not this transport\'s)', async () => {
const { dir, ks } = enrolledKs()
let presented = ''
const request: MtlsRequest = async (_url, tls) => {
presented = tls.cert
let called = 0
const request: MtlsRequest = async () => {
called += 1
return { status: 201, body: '{}' }
}
const f = createMtlsFetch(ks, { request, certParser: expiredBy(8 * DAY) })
const res = await f('https://recover.example.com/renew', { method: 'POST', body: '{}' })
expect(res.status).toBe(201)
expect(presented).toBe('LEAFCERT')
rmSync(dir, { recursive: true, force: true })
})
it('refuses once the leaf is BEYOND the grace window (terminal — re-pair required)', async () => {
const { dir, ks } = enrolledKs()
const request: MtlsRequest = async () => ({ status: 201, body: '{}' })
const f = createMtlsFetch(ks, {
request,
certParser: expiredBy(DEFAULT_EXPIRED_RENEW_GRACE_MS + DAY),
certParser: () => ({ validTo: new Date(Date.now() - 86_400_000) }),
})
await expect(f('https://recover.example.com/renew', { method: 'POST' })).rejects.toThrow(
CertExpiredBeyondGraceError,
await expect(f('https://cp.example.com/renew', { method: 'POST' })).rejects.toThrow(
/expired/i,
)
expect(called).toBe(0)
rmSync(dir, { recursive: true, force: true })
})
})

183
agent/test/rotation.test.ts
View File

@@ -8,11 +8,12 @@ import { openKeystore } from '../src/keys/keystore.js'
import {
computeRenewDelayMs,
createCertRotator,
recoveryRenewalUrlFor,
recoverCert,
recoveryUrlFor,
renewCert,
renewalUrlFor,
} from '../src/certs/rotation.js'
import { CertExpiredBeyondGraceError } from '../src/transport/dial.js'
import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js'
import { createBackoff } from '../src/transport/backoff.js'
import { FakeTimer } from './fixtures/fakes.js'
@@ -164,99 +165,128 @@ describe('createCertRotator (T13)', () => {
/**
* Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf
* it renews, so a lapsed leaf can never renew itself. Recovery needs BOTH a different endpoint (the
* strict nginx vhost rejects an expired client cert with a bare 400 before any location runs) and a
* terminal signal when even the grace window is spent, so the agent stops retrying forever.
* it renews, so a lapsed leaf can never renew itself — and nginx will not even forward an expired
* client cert. Recovery is therefore a PLAIN POST to `/recover` carrying the expired cert in the
* body, plus a terminal signal once the grace window is spent so the agent stops retrying forever.
*/
describe('expired-leaf recovery routing', () => {
it('derives the recovery endpoint from an `enroll.` host', () => {
expect(recoveryRenewalUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
'https://recover.terminal.example.com/renew',
describe('expired-leaf recovery', () => {
const HOUR = 3_600_000
it('derives /recover as a sibling PATH on the same enroll host', () => {
expect(recoveryUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
'https://enroll.terminal.example.com/recover',
)
})
it('honours an explicit recoverUrl over the derivation', () => {
expect(
recoveryRenewalUrlFor({
...CFG,
enrollUrl: 'https://enroll.terminal.example.com/enroll',
recoverUrl: 'https://elsewhere.example.com/renew',
}),
).toBe('https://elsewhere.example.com/renew')
expect(recoveryUrlFor({ ...CFG, recoverUrl: 'https://elsewhere.example.com/recover' })).toBe(
'https://elsewhere.example.com/recover',
)
})
it('is null when the enroll host carries no `enroll.` label (nothing to derive)', () => {
expect(recoveryRenewalUrlFor(CFG)).toBeNull()
})
it('posts to the NORMAL endpoint while the leaf is still valid', async () => {
it('recoverCert posts the EXPIRED cert plus a CSR, with no client cert, and installs the result', async () => {
const { dir, ks } = enrolledKs()
const timer = new FakeTimer()
const urls: string[] = []
const rotator = createCertRotator(
let seenUrl = ''
let seenBody: { cert?: string; csr?: string } = {}
const fetchImpl = (async (u: string, init: { body: string }) => {
seenUrl = u
seenBody = JSON.parse(init.body)
return jsonRes(201, { cert: 'FRESHCERT', caChain: ['FRESHCA'] })
}) as unknown as typeof fetch
const outcome = await recoverCert(
{ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
ks.loadIdentity()!,
ks,
{
timer,
renewBeforeMs: 1000,
fetchImpl: (async (u: string) => {
urls.push(u)
return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
}) as unknown as typeof fetch,
now: () => new Date(0),
parseCert: () => new Date(2000), // still valid at now=0
},
fetchImpl,
)
rotator.start()
timer.advance(1000)
await flush()
expect(urls).toEqual(['https://enroll.terminal.example.com/renew'])
rotator.stop()
expect(outcome).toBe('rotated')
expect(seenUrl).toBe('https://enroll.terminal.example.com/recover')
expect(seenBody.cert).toBe('OLDCERT') // the lapsed leaf travels in the BODY, not the TLS layer
expect(seenBody.csr).toBeTruthy()
expect(ks.loadCert()!.certPem).toContain('FRESHCERT')
rmSync(dir, { recursive: true, force: true })
})
it('posts to the RECOVERY endpoint once the leaf has already expired', async () => {
it('a still-valid leaf uses the mTLS /renew fetch, never the recovery one', async () => {
const { dir, ks } = enrolledKs()
const timer = new FakeTimer()
const urls: string[] = []
const rotator = createCertRotator(
{ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
ks.loadIdentity()!,
ks,
{
timer,
renewBeforeMs: 1000,
fetchImpl: (async (u: string) => {
urls.push(u)
return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
}) as unknown as typeof fetch,
now: () => new Date(10_000),
parseCert: () => new Date(2000), // expired 8s ago ⇒ delay clamps to 0
},
)
rotator.start()
timer.advance(0)
await flush()
expect(urls).toEqual(['https://recover.terminal.example.com/renew'])
rotator.stop()
rmSync(dir, { recursive: true, force: true })
})
it('fires onExhausted and STOPS retrying when the grace window is spent', async () => {
const { dir, ks } = enrolledKs()
const timer = new FakeTimer()
let attempts = 0
let renewCalls = 0
let recoverCalls = 0
const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
timer,
renewBeforeMs: 1000,
retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
fetchImpl: (async () => {
attempts += 1
throw new CertExpiredBeyondGraceError(40 * 86_400_000, 30 * 86_400_000)
renewCalls += 1
return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
}) as unknown as typeof fetch,
recoverFetchImpl: (async () => {
recoverCalls += 1
return jsonRes(200, {})
}) as unknown as typeof fetch,
now: () => new Date(0),
parseCert: () => new Date(2000),
parseCert: () => new Date(2000), // valid at now=0
})
rotator.start()
timer.advance(1000)
await flush()
expect(renewCalls).toBe(1)
expect(recoverCalls).toBe(0)
rotator.stop()
rmSync(dir, { recursive: true, force: true })
})
it('an expired leaf INSIDE the grace window switches to the recovery fetch', async () => {
const { dir, ks } = enrolledKs()
const timer = new FakeTimer()
let renewCalls = 0
let recoverCalls = 0
const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
timer,
renewBeforeMs: 1000,
expiredGraceMs: 30 * 24 * HOUR,
fetchImpl: (async () => {
renewCalls += 1
return jsonRes(200, {})
}) as unknown as typeof fetch,
recoverFetchImpl: (async () => {
recoverCalls += 1
return jsonRes(201, { cert: 'NEWCERT', caChain: ['NEWCA'] })
}) as unknown as typeof fetch,
now: () => new Date(8 * 24 * HOUR), // 8 days after the leaf lapsed
parseCert: () => new Date(0),
})
let rotated = 0
rotator.onRotated(() => {
rotated += 1
})
rotator.start()
timer.advance(0)
await flush()
expect(recoverCalls).toBe(1)
expect(renewCalls).toBe(0)
expect(rotated).toBe(1)
rotator.stop()
rmSync(dir, { recursive: true, force: true })
})
it('BEYOND the grace window it fires onExhausted, issues no request, and stops retrying', async () => {
const { dir, ks } = enrolledKs()
const timer = new FakeTimer()
let requests = 0
const countingFetch = (async () => {
requests += 1
return jsonRes(201, {})
}) as unknown as typeof fetch
const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
timer,
renewBeforeMs: 1000,
expiredGraceMs: 30 * 24 * HOUR,
retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
fetchImpl: countingFetch,
recoverFetchImpl: countingFetch,
now: () => new Date(31 * 24 * HOUR), // 31 days stale ⇒ past a 30-day grace
parseCert: () => new Date(0),
})
const errors: unknown[] = []
let exhausted: CertExpiredBeyondGraceError | null = null
@@ -265,17 +295,16 @@ describe('expired-leaf recovery routing', () => {
exhausted = e
})
rotator.start()
timer.advance(1000)
timer.advance(0)
await flush()
expect(attempts).toBe(1)
expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError)
// Terminal: no retry is armed, because only a re-pair can help — retrying forever is the log
// spam that buried the real signal in production (6380 identical warnings).
expect(requests).toBe(0) // nothing is even attempted — it cannot succeed
expect(errors).toHaveLength(0)
// Terminal: no retry armed. Retrying forever is what produced 6380 identical warnings.
timer.advance(60_000)
await flush()
expect(attempts).toBe(1)
expect(requests).toBe(0)
rmSync(dir, { recursive: true, force: true })
})
})