web-terminal/agent/test/rotation.test.ts

import { describe, expect, it, vi } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type { AgentConfig } from '../src/config/agentConfig.js'
import { generateIdentity } from '../src/keys/identity.js'
import { openKeystore } from '../src/keys/keystore.js'
import {
  computeRenewDelayMs,
  createCertRotator,
  recoverCert,
  recoveryUrlFor,
  renewCert,
  renewalUrlFor,
} from '../src/certs/rotation.js'
import { CertExpiredBeyondGraceError } from '../src/certs/rotation.js'
import { createBackoff } from '../src/transport/backoff.js'
import { FakeTimer } from './fixtures/fakes.js'

const CFG: AgentConfig = {
  relayUrl: 'wss://relay/agent',
  enrollUrl: 'https://example.com/enroll',
  stateDir: '/tmp/x',
  localTargetUrl: 'ws://127.0.0.1:3000',
  subdomain: 'host-42',
  hostId: 'h-1',
}

function enrolledKs() {
  const dir = mkdtempSync(join(tmpdir(), 'wta-rot-'))
  const ks = openKeystore(dir)
  ks.saveIdentity(generateIdentity())
  ks.saveCert('OLDCERT', 'OLDCA')
  return { dir, ks }
}

function jsonRes(status: number, body: unknown): Response {
  return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response
}

describe('cert rotation math (T13)', () => {
  it('derives P3 renewal URL from enrollUrl', () => {
    expect(renewalUrlFor(CFG)).toBe('https://example.com/renew')
  })

  it('renews renewBeforeMs before expiry, clamped at 0', () => {
    const now = new Date('2026-01-01T00:00:00Z')
    const parse = () => new Date('2026-01-01T01:00:00Z') // expires in 1h
    expect(computeRenewDelayMs('C', 5 * 60_000, now, parse)).toBe(55 * 60_000)
    const parsePast = () => new Date('2025-01-01T00:00:00Z')
    expect(computeRenewDelayMs('C', 5 * 60_000, now, parsePast)).toBe(0)
  })
})

describe('renewCert (T13)', () => {
  it('installs a fresh cert atomically on success (same key)', async () => {
    const { dir, ks } = enrolledKs()
    const before = ks.loadIdentity()!.publicKey
    const fetchImpl = vi.fn(async () => jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] }))
    const out = await renewCert(CFG, ks.loadIdentity()!, ks, fetchImpl as unknown as typeof fetch)
    expect(out).toBe('rotated')
    expect(ks.loadCert()!.certPem).toContain('NEWCERT'); expect(ks.loadCert()!.caChainPem).toContain('NEWCA')
    // pubkey unchanged — only the cert rotated
    expect(Buffer.from(ks.loadIdentity()!.publicKey).equals(Buffer.from(before))).toBe(true)
    rmSync(dir, { recursive: true, force: true })
  })

  it('403 → revoked (⇒ T14 teardown, INV12)', async () => {
    const { dir, ks } = enrolledKs()
    const fetchImpl = async () => jsonRes(403, {})
    const out = await renewCert(CFG, ks.loadIdentity()!, ks, fetchImpl as unknown as typeof fetch)
    expect(out).toBe('revoked')
    expect(ks.loadCert()).toEqual({ certPem: 'OLDCERT', caChainPem: 'OLDCA' }) // untouched
    rmSync(dir, { recursive: true, force: true })
  })
})

const flush = () => new Promise((r) => setImmediate(r))

describe('createCertRotator (T13)', () => {
  it('fires onRevoked when the scheduled renewal returns 403', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      fetchImpl: (async () => jsonRes(403, {})) as unknown as typeof fetch,
      now: () => new Date(0),
      parseCert: () => new Date(2000), // expires 2s after epoch → delay ~1000ms
    })
    let revoked = false
    rotator.onRevoked(() => {
      revoked = true
    })
    rotator.start()
    timer.advance(1000) // scheduled renewal fires
    await flush()
    expect(revoked).toBe(true)
    rmSync(dir, { recursive: true, force: true })
  })

  it('rotates and reschedules on a successful renewal (seamless)', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      fetchImpl: (async () => jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })) as unknown as typeof fetch,
      now: () => new Date(0),
      parseCert: () => new Date(2000),
    })
    let rotated = 0
    rotator.onRotated(() => {
      rotated += 1
    })
    rotator.start()
    timer.advance(1000)
    await flush()
    expect(rotated).toBe(1)
    expect(ks.loadCert()!.certPem).toContain('NEWCERT')
    rotator.stop()
    rmSync(dir, { recursive: true, force: true })
  })

  it('invokes onError and retries with backoff (not renewBeforeMs) when a renewal throws', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    let calls = 0
    const fetchImpl = (async () => {
      calls += 1
      if (calls === 1) throw new Error('network down')
      return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
    }) as unknown as typeof fetch
    const errors: unknown[] = []
    let rotated = 0
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
      fetchImpl,
      now: () => new Date(0),
      parseCert: () => new Date(2000), // initial renewal scheduled at ~1000ms
    })
    rotator.onError((e) => errors.push(e))
    rotator.onRotated(() => {
      rotated += 1
    })
    rotator.start()

    timer.advance(1000) // first attempt fires → throws
    await flush()
    expect(errors).toHaveLength(1)
    expect(rotated).toBe(0)

    // The retry is armed at the 500ms backoff delay, NOT renewBeforeMs (1000): advancing only 500
    // must fire it. A crash-loop never escapes here (the supervisor keeps running).
    timer.advance(500)
    await flush()
    expect(rotated).toBe(1)
    expect(ks.loadCert()!.certPem).toContain('NEWCERT')
    rotator.stop()
    rmSync(dir, { recursive: true, force: true })
  })
})

/**
 * Expired-leaf recovery (production deadlock, 2026-07): `/renew` is mTLS-authenticated by the leaf
 * it renews, so a lapsed leaf can never renew itself — and nginx will not even forward an expired
 * client cert. Recovery is therefore a PLAIN POST to `/recover` carrying the expired cert in the
 * body, plus a terminal signal once the grace window is spent so the agent stops retrying forever.
 */
describe('expired-leaf recovery', () => {
  const HOUR = 3_600_000

  it('derives /recover as a sibling PATH on the same enroll host', () => {
    expect(recoveryUrlFor({ ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' })).toBe(
      'https://enroll.terminal.example.com/recover',
    )
  })

  it('honours an explicit recoverUrl over the derivation', () => {
    expect(recoveryUrlFor({ ...CFG, recoverUrl: 'https://elsewhere.example.com/recover' })).toBe(
      'https://elsewhere.example.com/recover',
    )
  })

  it('recoverCert posts the EXPIRED cert plus a CSR, with no client cert, and installs the result', async () => {
    const { dir, ks } = enrolledKs()
    let seenUrl = ''
    let seenBody: { cert?: string; csr?: string } = {}
    const fetchImpl = (async (u: string, init: { body: string }) => {
      seenUrl = u
      seenBody = JSON.parse(init.body)
      return jsonRes(201, { cert: 'FRESHCERT', caChain: ['FRESHCA'] })
    }) as unknown as typeof fetch

    const outcome = await recoverCert(
      { ...CFG, enrollUrl: 'https://enroll.terminal.example.com/enroll' },
      ks.loadIdentity()!,
      ks,
      fetchImpl,
    )
    expect(outcome).toBe('rotated')
    expect(seenUrl).toBe('https://enroll.terminal.example.com/recover')
    expect(seenBody.cert).toBe('OLDCERT') // the lapsed leaf travels in the BODY, not the TLS layer
    expect(seenBody.csr).toBeTruthy()
    expect(ks.loadCert()!.certPem).toContain('FRESHCERT')
    rmSync(dir, { recursive: true, force: true })
  })

  it('a still-valid leaf uses the mTLS /renew fetch, never the recovery one', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    let renewCalls = 0
    let recoverCalls = 0
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      fetchImpl: (async () => {
        renewCalls += 1
        return jsonRes(200, { cert: 'NEWCERT', caChain: ['NEWCA'] })
      }) as unknown as typeof fetch,
      recoverFetchImpl: (async () => {
        recoverCalls += 1
        return jsonRes(200, {})
      }) as unknown as typeof fetch,
      now: () => new Date(0),
      parseCert: () => new Date(2000), // valid at now=0
    })
    rotator.start()
    timer.advance(1000)
    await flush()
    expect(renewCalls).toBe(1)
    expect(recoverCalls).toBe(0)
    rotator.stop()
    rmSync(dir, { recursive: true, force: true })
  })

  it('an expired leaf INSIDE the grace window switches to the recovery fetch', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    let renewCalls = 0
    let recoverCalls = 0
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      expiredGraceMs: 30 * 24 * HOUR,
      fetchImpl: (async () => {
        renewCalls += 1
        return jsonRes(200, {})
      }) as unknown as typeof fetch,
      recoverFetchImpl: (async () => {
        recoverCalls += 1
        return jsonRes(201, { cert: 'NEWCERT', caChain: ['NEWCA'] })
      }) as unknown as typeof fetch,
      now: () => new Date(8 * 24 * HOUR), // 8 days after the leaf lapsed
      parseCert: () => new Date(0),
    })
    let rotated = 0
    rotator.onRotated(() => {
      rotated += 1
    })
    rotator.start()
    timer.advance(0)
    await flush()
    expect(recoverCalls).toBe(1)
    expect(renewCalls).toBe(0)
    expect(rotated).toBe(1)
    rotator.stop()
    rmSync(dir, { recursive: true, force: true })
  })

  it('BEYOND the grace window it fires onExhausted, issues no request, and stops retrying', async () => {
    const { dir, ks } = enrolledKs()
    const timer = new FakeTimer()
    let requests = 0
    const countingFetch = (async () => {
      requests += 1
      return jsonRes(201, {})
    }) as unknown as typeof fetch
    const rotator = createCertRotator(CFG, ks.loadIdentity()!, ks, {
      timer,
      renewBeforeMs: 1000,
      expiredGraceMs: 30 * 24 * HOUR,
      retryBackoff: createBackoff({ baseMs: 500, jitter: false }),
      fetchImpl: countingFetch,
      recoverFetchImpl: countingFetch,
      now: () => new Date(31 * 24 * HOUR), // 31 days stale ⇒ past a 30-day grace
      parseCert: () => new Date(0),
    })
    const errors: unknown[] = []
    let exhausted: CertExpiredBeyondGraceError | null = null
    rotator.onError((e) => errors.push(e))
    rotator.onExhausted((e) => {
      exhausted = e
    })
    rotator.start()
    timer.advance(0)
    await flush()

    expect(exhausted).toBeInstanceOf(CertExpiredBeyondGraceError)
    expect(requests).toBe(0) // nothing is even attempted — it cannot succeed
    expect(errors).toHaveLength(0)
    // Terminal: no retry armed. Retrying forever is what produced 6380 identical warnings.
    timer.advance(60_000)
    await flush()
    expect(requests).toBe(0)
    rmSync(dir, { recursive: true, force: true })
  })
})