kai/web-terminal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tunnel): recover an expired leaf over plain HTTPS, not mTLS

Browse Source 
Correction to the previous commit. Its recovery vhost could not work: nginx
will not forward an expired client certificate in ANY `ssl_verify_client` mode.
`optional` answers a bare `400 The SSL certificate error` before any location
runs, and `optional_no_ca` only tolerates CHAIN failures — see nginx's
`ngx_ssl_verify_error_optional()`, which covers self-signed / unknown-issuer /
unverifiable-leaf and NOT `X509_V_ERR_CERT_HAS_EXPIRED`. Verified live against
the deployed :8472 server, which rejected the real expired leaf.

So recovery drops mTLS instead of trying to bend it:

- agent: `buildTlsOptions` and the mTLS renew transport go back to being
  strictly fail-closed on expiry — the relaxation is gone from the TLS layer
  entirely. The rotator now decides per attempt: valid → mTLS `/renew`,
  expired-inside-grace → plain `/recover`, expired-beyond-grace → terminal
  `onExhausted` with no request issued at all.
- new `recoverCert` POSTs `{cert, csr}` with no client certificate. Possession
  of the private key is still proven: the CSR is self-signed by it and the host
  signer already enforces CSR PoP plus `CSR key == registered key`, so a
  replayed (public) cert without the key yields at most a certificate the
  attacker cannot authenticate with.
- control-plane: `/renew` is strict again (grace 0, matching the terminator).
  The grace lives on the new `POST /recover`, which reads the cert from the BODY
  and ignores the `x-client-cert` header, then runs the identical trust
  pipeline: X.509 path validation to the frp-client-CA anchors, SPIFFE parse,
  `notBefore` (never graced), registry `active` + account match. Revocation
  still bites.
- deploy: no new vhost, no new SNI, no DNS. One `location = /recover` merged
  into the existing enroll vhost, documented in
  `deploy/nginx/enroll-recover-location.md` with the nginx source citation.

The load-bearing new test is "a self-signed cert with a FORGED SPIFFE SAN is
refused → 401": nginx no longer validates the chain on this path, so that
assertion is what keeps `/recover` from being a cert vending machine.

Verified: agent 289/289, control-plane 290/290, tsc clean on both.
This commit is contained in:
Yaojia Wang
2026-07-29 09:52:17 +02:00
parent f3f4d8baa6
commit 5509c81eee
13 changed files with 559 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
control-plane/node_modules Symbolic link
View File

@@ -0,0 +1 @@
/Users/yiukai/Documents/git/web-terminal/control-plane/node_modules

100
control-plane/src/api/renew.ts
View File

@@ -51,6 +51,8 @@ export const DEFAULT_RENEW_RATE_WINDOW_MS = 60 * 60 * 1000
export const RENEW_RATE_MAX_IDENTITIES = 10_000
/** Max base64 length of a submitted CSR (a P-256 PKCS#10 is well under 1 KB; this is generous slack). */
export const MAX_CSR_B64_LEN = 8192
/** Max wire length of a body-carried certificate (a P-256 leaf PEM is ~1 KB; generous slack). */
export const MAX_PRESENTED_CERT_LEN = 16384
/**
* Default header the mTLS terminator forwards the verified client cert in (base64 DER). The terminator
* MUST set this from `$ssl_client_cert` AND strip any client-supplied copy — a client can never provide
@@ -185,23 +187,29 @@ export function headerPresentedCert(headerName: string = DEFAULT_CLIENT_CERT_HEA
const raw = req.headers[name]
const value = Array.isArray(raw) ? raw[0] : raw
if (typeof value !== 'string' || value.length === 0) return null
try {
// The terminator may forward the verified cert as base64 DER, OR as PEM — including nginx's
// header-safe `$ssl_client_escaped_cert` (URL-encoded PEM). Normalize all three to raw DER:
// URL-decode if escaped, then strip PEM armor + whitespace to recover the base64 DER body.
let s = value.includes('%') ? safeDecodeUri(value) : value
if (s.includes('BEGIN CERTIFICATE')) {
s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '')
}
const der = Buffer.from(s, 'base64')
return der.length > 0 ? new Uint8Array(der) : null
} catch {
return null
}
return certWireToDer(value)
},
}
}
/**
* Normalize a certificate carried over the wire to raw DER. Accepts base64 DER, PEM, and nginx's
* header-safe `$ssl_client_escaped_cert` (URL-encoded PEM): URL-decode if escaped, then strip PEM
* armor + whitespace to recover the base64 body. Returns null on anything unparseable.
*/
export function certWireToDer(value: string): Uint8Array | null {
try {
let s = value.includes('%') ? safeDecodeUri(value) : value
if (s.includes('BEGIN CERTIFICATE')) {
s = s.replace(/-----[^-]+-----/g, '').replace(/\s+/g, '')
}
const der = Buffer.from(s, 'base64')
return der.length > 0 ? new Uint8Array(der) : null
} catch {
return null
}
}
/** URL-decode `value`, returning it unchanged if it is not valid percent-encoding. */
function safeDecodeUri(value: string): string {
try {
@@ -286,6 +294,13 @@ function parsePresentedCertIdentity(
}
const RenewBodySchema = z.object({ csr: z.string().min(1).max(MAX_CSR_B64_LEN) }).strict()
/** `/recover` additionally carries the EXPIRED leaf itself (base64 DER or PEM). */
const RecoverBodySchema = z
.object({
cert: z.string().min(1).max(MAX_PRESENTED_CERT_LEN),
csr: z.string().min(1).max(MAX_CSR_B64_LEN),
})
.strict()
export interface RenewDeps {
readonly hosts: HostRegistry
@@ -327,7 +342,9 @@ function sendError(reply: FastifyReply, err: unknown): void {
export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
const presentedCert = deps.presentedCert ?? headerPresentedCert()
const rateLimiter = deps.rateLimiter ?? createRenewRateLimiter()
const expiredGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS
// `/renew` stays STRICT (0): the mTLS terminator would never forward an expired cert to it anyway.
// The grace belongs to `/recover`, the route built for exactly that case.
const recoverGraceMs = deps.expiredRenewGraceMs ?? DEFAULT_EXPIRED_RENEW_GRACE_MS
return async (app) => {
app.post('/renew', async (req, reply) => {
@@ -338,7 +355,7 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
der,
'host',
deps.hostCaAnchorsDer
? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs }
? { caAnchorsDer: deps.hostCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
: undefined,
)
rateLimiter.check(`host:${identity.accountId}:${identity.id}`)
@@ -365,6 +382,57 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
}
})
/**
* Expired-leaf recovery. The lapsed cert arrives in the BODY because no TLS terminator will
* forward an expired client certificate (nginx `optional` → bare 400; `optional_no_ca` tolerates
* only chain errors, never `X509_V_ERR_CERT_HAS_EXPIRED`). That makes this route the SOLE
* verifier, so it runs the identical trust pipeline as `/renew` — real X.509 path validation to
* the frp-client-CA anchors, SPIFFE parse, `notBefore`, registry `active` + account match — and
* differs ONLY in allowing a bounded overrun on `notAfter`.
*
* Possession of the private key is still proven: the CSR is self-signed by it, and the host
* signer's delegated gate enforces CSR PoP plus `CSR key == registered key`. A replayed cert
* without the key therefore yields, at most, a certificate the attacker cannot authenticate with.
* The header channel is deliberately IGNORED here — on this route only the body speaks.
*/
app.post('/recover', async (req, reply) => {
try {
const body = RecoverBodySchema.parse(req.body)
const der = certWireToDer(body.cert)
if (der === null) throw new RenewRejectError(401)
const identity = parsePresentedCertIdentity(
der,
'host',
deps.hostCaAnchorsDer
? {
caAnchorsDer: deps.hostCaAnchorsDer,
nowMs: Date.now(),
expiredGraceMs: recoverGraceMs,
}
: undefined,
)
rateLimiter.check(`recover:${identity.accountId}:${identity.id}`)
const host = await deps.hosts.getHostBySubdomain(identity.id)
if (host === null || host.status === 'revoked' || host.accountId !== identity.accountId) {
throw new RenewRejectError(403)
}
const leaf = await deps.renewer.renewHostLeaf(
host.hostId,
identity.publicKeySpki,
decodeCsrWire(body.csr),
)
await reply.code(201).send({
cert: bytesToBase64(leaf.cert),
caChain: leaf.caChain.map((c) => bytesToBase64(c)),
notAfter: leaf.notAfter.toISOString(),
})
} catch (err) {
sendError(reply, err)
}
})
app.post('/device/:id/renew', async (req, reply) => {
try {
const der = presentedCert.presentedCertDer(req)
@@ -373,7 +441,7 @@ export function buildRenewRouter(deps: RenewDeps): FastifyPluginAsync {
der,
'device',
deps.deviceCaAnchorsDer
? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs }
? { caAnchorsDer: deps.deviceCaAnchorsDer, nowMs: Date.now(), expiredGraceMs: 0 }
: undefined,
)
const id = (req.params as { id: string }).id

244
control-plane/test/renew.test.ts
View File

@@ -490,65 +490,10 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio
expect(res.statusCode).toBe(201)
})
/**
* POLICY CHANGE (expired-leaf deadlock fix): `/renew` is mTLS-authenticated by the very leaf it
* renews, so refusing every expired leaf meant a host whose leaf lapsed could NEVER renew and was
* bricked until a manual re-pair. A recently-expired leaf is now accepted for RE-ISSUANCE ONLY,
* inside a bounded window. Everything else stays fail-closed — the tests below pin that down.
*/
test('a leaf expired INSIDE the grace window renews → 201 (breaks the deadlock)', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const expired = await mintHostLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS))
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/renew',
headers: { 'x-client-cert': certHeader(expired) },
payload: { csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(201)
})
test('a leaf expired BEYOND the grace window is rejected → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const stale = await mintHostLeaf(ctx, new Date(now - 60 * DAY_MS), new Date(now - 31 * DAY_MS))
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/renew',
headers: { 'x-client-cert': certHeader(stale) },
payload: { csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(401)
})
test('grace 0 restores the strict fail-closed behaviour → 401', async () => {
test('an EXPIRED current cert (valid chain, notAfter in the past) is rejected → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const expired = await mintHostLeaf(ctx, new Date(now - 2 * DAY_MS), new Date(now - DAY_MS))
const app = appWith(
hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer], expiredRenewGraceMs: 0 }),
)
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/renew',
headers: { 'x-client-cert': certHeader(expired) },
payload: { csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(401)
})
test('grace NEVER bypasses revocation — revoked host with an in-grace leaf → 403', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const expired = await mintHostLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS))
const host = await ctx.hosts.getHostBySubdomain('alice')
await ctx.hosts.setHostStatus(host!.hostId, 'revoked')
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
@@ -557,42 +502,8 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio
headers: { 'x-client-cert': certHeader(expired) },
payload: { csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(403)
})
test('grace NEVER bypasses chain validation — untrusted CA + in-grace leaf → 401', async () => {
const ctx = await hostCtx('alice')
const rogueCa = await makeP256Ca('rogue-CA')
const now = Date.now()
const rogue = await mintHostLeaf(
ctx,
new Date(now - 9 * DAY_MS),
new Date(now - 8 * DAY_MS),
rogueCa.caSigner,
)
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/renew',
headers: { 'x-client-cert': certHeader(rogue) },
payload: { csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(401)
})
test('grace applies ONLY to notAfter — a not-yet-valid leaf is still rejected → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const future = await mintHostLeaf(ctx, new Date(now + DAY_MS), new Date(now + 2 * DAY_MS))
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/renew',
headers: { 'x-client-cert': certHeader(future) },
payload: { csr: b64Csr(ctx.csr) },
})
// STRICT on this route: nginx would never forward an expired client cert here anyway, so the
// deadlock escape hatch lives on `/recover` (below) and NOT on the mTLS renewal path.
expect(res.statusCode).toBe(401)
})
@@ -612,3 +523,152 @@ describe('CP6c POST /renew — presented current-cert chain + expiry verificatio
expect(res.statusCode).toBe(401)
})
})
/**
* CP6d · POST /recover — the expired-leaf escape hatch.
*
* Unlike `/renew` this route is NOT mTLS-authenticated: nginx cannot forward an expired client cert
* (under `ssl_verify_client optional` it 400s, and `optional_no_ca` only tolerates CHAIN errors, not
* `X509_V_ERR_CERT_HAS_EXPIRED`). The lapsed cert therefore arrives in the BODY, and the
* control-plane becomes the sole verifier. These tests pin down that nothing except the `notAfter`
* bound was relaxed — chain, SPIFFE, `notBefore`, and registry status all still decide.
*/
describe('CP6d POST /recover — expired-leaf recovery', () => {
async function mintLeaf(
ctx: HostCtx,
notBefore: Date,
notAfter: Date,
signer = ctx.ca.caSigner,
): Promise<Uint8Array> {
const spiffe = `spiffe://relay.terminal.yaojia.wang/account/${ctx.accountId}/host/${ctx.subdomain}`
return assembleCertificate({
subjectPublicKey: ctx.spki,
subject: `CN=${ctx.subdomain}`,
issuer: ctx.ca.caCert.subjectName,
serialNumber: Uint8Array.from([0x0b]),
notBefore,
notAfter,
extensions: [
new x509.SubjectAlternativeNameExtension([
{ type: 'dns', value: `${ctx.subdomain}.terminal.yaojia.wang` },
{ type: 'url', value: spiffe },
]),
new x509.BasicConstraintsExtension(false, undefined, true),
new x509.KeyUsagesExtension(x509.KeyUsageFlags.digitalSignature, true),
new x509.ExtendedKeyUsageExtension([x509.ExtendedKeyUsage.clientAuth]),
],
signer,
sigAlg: 'ecdsa-p256',
})
}
const b64 = (der: Uint8Array): string => Buffer.from(der).toString('base64')
async function post(ctx: HostCtx, cert: Uint8Array, extra?: Partial<RenewDeps>) {
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer], ...extra }))
await app.ready()
return app.inject({
method: 'POST',
url: '/recover',
payload: { cert: b64(cert), csr: b64Csr(ctx.csr) },
})
}
test('a leaf expired INSIDE the grace window is re-issued → 201 (breaks the deadlock)', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const res = await post(ctx, await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS)))
expect(res.statusCode).toBe(201)
expect(JSON.parse(res.payload).cert).toBeTruthy()
})
test('a leaf expired BEYOND the grace window is refused → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const res = await post(ctx, await mintLeaf(ctx, new Date(now - 60 * DAY_MS), new Date(now - 31 * DAY_MS)))
expect(res.statusCode).toBe(401)
})
test('grace 0 disables recovery entirely → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const res = await post(
ctx,
await mintLeaf(ctx, new Date(now - 2 * DAY_MS), new Date(now - DAY_MS)),
{ expiredRenewGraceMs: 0 },
)
expect(res.statusCode).toBe(401)
})
/**
* THE load-bearing test for this route. nginx no longer validates the chain here, so a forged
* self-signed cert carrying a correct-looking SPIFFE SAN must be rejected by the control-plane
* alone. If this ever goes green-to-red, `/recover` becomes an unauthenticated cert vending machine.
*/
test('a self-signed cert with a FORGED SPIFFE SAN is refused → 401', async () => {
const ctx = await hostCtx('alice')
const rogueCa = await makeP256Ca('rogue-CA')
const now = Date.now()
const rogue = await mintLeaf(
ctx,
new Date(now - 9 * DAY_MS),
new Date(now - 8 * DAY_MS),
rogueCa.caSigner,
)
const res = await post(ctx, rogue)
expect(res.statusCode).toBe(401)
})
test('recovery NEVER bypasses revocation — revoked host → 403', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const cert = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS))
const host = await ctx.hosts.getHostBySubdomain('alice')
await ctx.hosts.setHostStatus(host!.hostId, 'revoked')
const res = await post(ctx, cert)
expect(res.statusCode).toBe(403)
})
test('grace covers notAfter ONLY — a not-yet-valid leaf is refused → 401', async () => {
const ctx = await hostCtx('alice')
const now = Date.now()
const res = await post(ctx, await mintLeaf(ctx, new Date(now + DAY_MS), new Date(now + 2 * DAY_MS)))
expect(res.statusCode).toBe(401)
})
test('a still-valid leaf may also use /recover → 201', async () => {
const ctx = await hostCtx('alice')
const res = await post(ctx, ctx.currentCertDer)
expect(res.statusCode).toBe(201)
})
test('a missing cert field is rejected → 400 (uniform schema reject, same as a missing csr)', async () => {
const ctx = await hostCtx('alice')
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({ method: 'POST', url: '/recover', payload: { csr: b64Csr(ctx.csr) } })
// The module answers with uniform rejects that never say which check failed: 400 for a malformed
// body, 401 for a cert that parses but is not trusted, 403 for a trusted cert that is not allowed.
expect(res.statusCode).toBe(400)
})
/**
* The header is the mTLS terminator's channel. On `/recover` the cert comes from the body, so a
* client-supplied header must not be able to substitute an identity.
*/
test('an x-client-cert HEADER cannot override the body identity → 401', async () => {
const ctx = await hostCtx('alice')
const rogueCa = await makeP256Ca('rogue-CA')
const now = Date.now()
const rogue = await mintLeaf(ctx, new Date(now - 9 * DAY_MS), new Date(now - 8 * DAY_MS), rogueCa.caSigner)
const app = appWith(hostDeps(ctx, { hostCaAnchorsDer: [ctx.ca.caDer] }))
await app.ready()
const res = await app.inject({
method: 'POST',
url: '/recover',
headers: { 'x-client-cert': certHeader(ctx.currentCertDer) },
payload: { cert: b64(rogue), csr: b64Csr(ctx.csr) },
})
expect(res.statusCode).toBe(401)
})
})