Files
web-terminal/ios/App/WebTerm/Components/SpeechDictation.swift
Yaojia Wang 284cfd193a feat(ios,android): P2 wave, git panel, token UX, per-host WS token, docs
App layer, four sequential slices (a shared .xcodeproj means adding files
regenerates it, so these could not run in parallel):

- token UX end to end: pairing prompts for a token when a host 401s, POST /auth
  validates it, and 204-without-Set-Cookie is correctly read as "this server has
  auth disabled" rather than "authenticated". A host paired before the token was
  turned on recovers by re-pairing in place. Remove-host now exists and finally
  gives PushRegistrar.handleHostRemoved a caller.
- project git panel + worktree lifecycle (T-iOS-32) + claude --resume history —
  the parity gap with Android and the web front end.
- terminal search (T-iOS-33) and voice PTT (T-iOS-31) with an epoch guard so a
  session switch between dictation and confirm cannot inject into the wrong
  session.
- theme + Dynamic Type (T-iOS-34) and web ?join= interop (T-iOS-35). RootView no
  longer hard-locks .preferredColorScheme(.dark).

Also unpins SwiftTerm to 1.15.0 by dropping the local hasActiveSelection that
collided with the upstream one, verified green from a fresh derivedDataPath.

Includes the two HIGH fixes the security review found:
- iOS resolved the WS token host-independently, so a token-gated host sitting
  next to an open one could never open a terminal and no on-screen remedy could
  fix it. Now one transport per host; cross-host leakage is structurally
  impossible since both read paths return only that host's own value.
- Android reported the host's own git-credential 401 (git-ops.ts:108, "Push
  authentication required on the host.") as "your access token is wrong", because
  a blanket 401 mapping ran ahead of the per-route one. Git-write routes are now
  ROUTE_DEFINED and keep the server's message.

And the doc sync: README/ios README no longer claim the client is unmerged on
feat/ios-client, the Clients section finally lists Android, and the plan
checkboxes reflect what is actually built.

iOS 534 app tests + 452 package tests; Android 687 tests.
2026-07-30 15:58:01 +02:00

153 lines
5.9 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import AVFoundation
import Foundation
import Speech
/// T-iOS-31 · `VoiceDictating` Speech + AVAudioEngine
/// `VoicePTT.swift` 800 plan §4
/// ****`requiresOnDeviceRecognition`
/// Apple
/// web SEC-L2 `VoiceTranscript.sanitize`
///
/// PTT **** 12
@MainActor
final class SpeechDictation: VoiceDictating {
enum DictationError: LocalizedError {
case recognizerUnavailable
case microphoneUnavailable
var errorDescription: String? {
switch self {
case .recognizerUnavailable: "设备上的语音识别当前不可用。"
case .microphoneUnavailable: "拿不到麦克风输入。"
}
}
}
/// tap Apple
private static let tapBufferSize: AVAudioFrameCount = 1024
private lazy var audioEngine = AVAudioEngine()
private lazy var recognizer = SFSpeechRecognizer(locale: .current) ?? SFSpeechRecognizer()
private var request: SFSpeechAudioBufferRecognitionRequest?
private var task: SFSpeechRecognitionTask?
private var onPartial: (@MainActor (String) -> Void)?
private var isTapInstalled = false
private var latest = VoiceDictationResult(transcript: "", confidence: nil)
func requestAuthorization() async -> VoiceAuthorization {
let speech = await Self.requestSpeechAuthorization()
switch speech {
case .authorized: break
case .restricted: return .restricted
case .denied, .notDetermined: return .deniedSpeech
@unknown default: return .deniedSpeech
}
return await Self.requestMicrophoneAuthorization() ? .granted : .deniedMicrophone
}
func start(onPartial: @escaping @MainActor (String) -> Void) async throws {
guard let recognizer, recognizer.isAvailable else {
throw DictationError.recognizerUnavailable
}
self.onPartial = onPartial
latest = VoiceDictationResult(transcript: "", confidence: nil)
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record, mode: .measurement, options: .duckOthers)
try session.setActive(true, options: .notifyOthersOnDeactivation)
let request = SFSpeechAudioBufferRecognitionRequest()
request.shouldReportPartialResults = true
request.requiresOnDeviceRecognition = recognizer.supportsOnDeviceRecognition
self.request = request
let input = audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
// installTap ObjC
guard format.sampleRate > 0, format.channelCount > 0 else {
teardown()
throw DictationError.microphoneUnavailable
}
input.installTap(onBus: 0, bufferSize: Self.tapBufferSize, format: format) { buffer, _ in
request.append(buffer)
}
isTapInstalled = true
audioEngine.prepare()
try audioEngine.start()
task = recognizer.recognitionTask(with: request) { [weak self] result, error in
// Sendable result actor
let transcript = result?.bestTranscription.formattedString
let confidence = result.flatMap { Self.averageConfidence(of: $0.bestTranscription) }
let isFinal = result?.isFinal ?? false
let didFail = error != nil
Task { @MainActor in
self?.ingest(
transcript: transcript, confidence: confidence,
isFinal: isFinal, didFail: didFail
)
}
}
}
func stop() async -> VoiceDictationResult {
teardown()
return latest
}
// MARK: Internals
private func ingest(
transcript: String?, confidence: Double?, isFinal: Bool, didFail: Bool
) {
if let transcript, !transcript.isEmpty {
latest = VoiceDictationResult(transcript: transcript, confidence: confidence)
onPartial?(transcript)
}
guard isFinal || didFail else { return }
teardown()
}
/// tap
private func teardown() {
request?.endAudio()
task?.cancel()
task = nil
request = nil
onPartial = nil
if isTapInstalled {
audioEngine.inputNode.removeTap(onBus: 0)
isTapInstalled = false
}
if audioEngine.isRunning {
audioEngine.stop()
}
try? AVAudioSession.sharedInstance()
.setActive(false, options: .notifyOthersOnDeactivation)
}
private static func averageConfidence(of transcription: SFTranscription) -> Double? {
let values = transcription.segments.map { Double($0.confidence) }.filter { $0 > 0 }
guard !values.isEmpty else { return nil }
return values.reduce(0, +) / Double(values.count)
}
private static func requestSpeechAuthorization() async
-> SFSpeechRecognizerAuthorizationStatus {
await withCheckedContinuation { continuation in
SFSpeechRecognizer.requestAuthorization { status in
continuation.resume(returning: status)
}
}
}
private static func requestMicrophoneAuthorization() async -> Bool {
await withCheckedContinuation { continuation in
AVAudioApplication.requestRecordPermission { granted in
continuation.resume(returning: granted)
}
}
}
}