#12 multi-device tiles + #5 native iOS transcript

#12 — same user on two devices now shows as two independent tiles (was: LiveKit kicked the older connection, "audio jumps to whichever joined last"): - LiveKit identity is now the per-connection mesh peerId, not the user id. /api/meetings/token + guest-token mint identity=peerId when the client supplies it (anti-hijack: never mint another live user's peerId). Web maps SFU tracks by identity==peerId, keeping peerIdForUid as a fallback for the transition/native. - Mesh dedup (dropDupPeers) now keys on a stable per-device clientId (persisted, sent on meeting-join, echoed by the server) instead of user id — so two real devices keep separate tiles while a same-device reconnect ghost still collapses. Verified in a real browser: 2 devices -> 2 tiles; same-device reconnect -> 1. - Native: plugin gains reconnectRoom(); after the native WebView joins the mesh it re-homes the LiveKit media onto its peerId identity. syncVideoTiles keys by peerId. Token-identity + anti-hijack + clientId echo verified by a server test. #5 — iOS live transcript (WKWebView has no Web Speech API, so an iOS participant was never transcribed; desktop already works): - native-call plugin transcribes the local mic with SFSpeechRecognizer, fed by a LiveKit AudioRenderer on the local mic track (reuses the call's open mic — no 2nd AVAudioEngine). Finalized segments -> 'transcript' event -> web sends meeting-transcript (same server assembly as desktop). startSR/stopSR use the native recognizer on native calls; Web Speech API path unchanged elsewhere. - NSSpeechRecognitionUsageDescription added to the iOS Info.plist. Native pieces (#12 reconnect, #5 transcript) need a Codemagic build; the web+server half is verified and deploys now (already fixes the reported laptop+phone case). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-14 16:36:18 +05:30
parent c322448774
commit 68ff3a878b
5 changed files with 260 additions and 35 deletions
@@ -5,6 +5,7 @@ import Capacitor
 import PushKit
 import CallKit
 import AVFoundation
+import Speech // #5 native transcript: SFSpeechRecognizer (iOS on-device speech-to-text; WKWebView has no Web Speech API)
 import LiveKit // SPM product name is 'LiveKit' (Package.swift). (The old CocoaPods module was 'LiveKitClient'.)

 // Native calling for Biz Connect (iOS). Registered by `cap sync` as window.Capacitor.Plugins.NativeCall.
@@ -27,6 +28,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        CAPPluginMethod(name: "getToken", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "reportOutgoingCall", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise),
+        CAPPluginMethod(name: "reconnectRoom", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise),
@@ -35,6 +37,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise),
+        CAPPluginMethod(name: "startTranscription", returnType: CAPPluginReturnPromise),
+        CAPPluginMethod(name: "stopTranscription", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise)
    ]

@@ -56,6 +60,9 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
    // on top. Saved so we can restore the WebView when the call ends.
    private var holePunchOn = false
    private var savedVCBg: UIColor?   // the WebView parent's original background, restored when the call ends
+    // #5 native transcript: transcribes THIS device's mic (WKWebView has no Web Speech API). Fed by a LiveKit
+    // AudioRenderer on the local mic track, so it reuses the call's already-open mic (no 2nd audio engine).
+    private let transcriber = SpeechTranscriber()

    override public func load() {
        let config = CXProviderConfiguration(localizedName: "Biz Connect")
@@ -89,6 +96,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        // (with shouldPublishTrack=true, the default) auto-publishes/unpublishes the screen-share track.
        BroadcastManager.shared.delegate = self

+        // #5 native transcript: each finalized speech segment → the web (meeting-transcript over the WS), which
+        // merges it into the shared meeting transcript (same path as desktop's Web Speech API).
+        transcriber.onFinal = { [weak self] text in self?.notifyListeners("transcript", data: ["text": text]) }
+
        // Make the WebView transparent at STARTUP. WKWebView can IGNORE isOpaque=false when it's flipped after
        // the page has already rendered — the likely reason the hole-punch showed no video. Doing it once, up
        // front, makes the transparent-meeting areas actually reveal the native video behind the WebView. The
@@ -141,6 +152,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
    private func disconnectRoom() {
        let r = room
        room = nil
+        transcriber.stop() // #5: end any live transcription with the call
        Task { await r?.disconnect() }
        removeAllTileViews()
        DispatchQueue.main.async { [weak self] in guard let self = self, let web = self.bridge?.webView else { return }; self.applyHolePunchRestore(web) }
@@ -303,6 +315,22 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        call.resolve()
    }

+    // #12 Multi-device: swap the media connection to a token whose identity = this device's mesh peerId (unique
+    // per connection). The web calls this once the native WebView has joined the mesh and has a peerId. On
+    // answer we connected INSTANTLY with the push token (identity=userId) for zero-latency audio; this re-homes
+    // the media onto the unique peerId identity so two devices of the same user are distinct LiveKit
+    // participants (LiveKit allows one connection per identity — otherwise the older device is kicked and
+    // "audio jumps to whichever joined last"). connectRoom disconnects the old room first; the CallKit call and
+    // its audio session stay active, so audio just re-attaches. callConnected fires on success → web re-applies
+    // mic/cam. Guarded to only run during an active call.
+    @objc func reconnectRoom(_ call: CAPPluginCall) {
+        let url = call.getString("url") ?? ""
+        let token = call.getString("token") ?? ""
+        guard room != nil, !url.isEmpty, !token.isEmpty else { call.resolve(); return }
+        connectRoom(url: url, token: token)
+        call.resolve()
+    }
+
    @objc func setMuted(_ call: CAPPluginCall) {
        let muted = call.getBool("muted") ?? false
        // Drive mute THROUGH CallKit so the system call screen and the in-app meeting UI stay in sync (the
@@ -362,6 +390,28 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        call.resolve()
    }

+    // MARK: - #5 Live transcript (native SFSpeechRecognizer)
+
+    // The current published local mic track — the source we tap for transcription. nil until the user has
+    // unmuted at least once (the mic is published on unmute), or between reconnects.
+    private func localAudioTrack() -> LocalAudioTrack? {
+        return room?.localParticipant.audioTracks.first?.track as? LocalAudioTrack
+    }
+
+    // Start transcribing this device's mic. The web calls this when the room's transcript is active and we're
+    // in a native call (WKWebView has no Web Speech API). Attaches a LiveKit AudioRenderer to the local mic
+    // track and runs SFSpeechRecognizer; finalized segments fire the "transcript" event. Safe to call when the
+    // mic track doesn't exist yet — we re-attach on unmute (setMicrophone-enable) too.
+    @objc func startTranscription(_ call: CAPPluginCall) {
+        transcriber.start(track: localAudioTrack())
+        call.resolve()
+    }
+
+    @objc func stopTranscription(_ call: CAPPluginCall) {
+        transcriber.stop()
+        call.resolve()
+    }
+
    // MARK: - BroadcastManagerDelegate

    public func broadcastManager(didChangeState isBroadcasting: Bool) {
@@ -529,6 +579,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        Task { [weak self] in
            try? await r?.localParticipant.setMicrophone(enabled: !action.isMuted)
            self?.preferSpeaker() // toggling the mic can flip the route back to the earpiece — re-assert speaker
+            // #5: unmuting publishes the mic track — (re)attach the transcriber's renderer to it if transcript is on.
+            if !action.isMuted { self?.transcriber.refresh(track: self?.localAudioTrack()) }
        }
        notifyListeners("setMuted", data: ["callUUID": action.callUUID.uuidString, "muted": action.isMuted])
        action.fulfill()
@@ -592,3 +644,111 @@ final class TileVideoView: UIView {
        else { video.transform = CGAffineTransform(translationX: tx, y: ty).scaledBy(x: scale, y: scale) }
    }
 }
+
+// #5 Live transcript on iOS. WKWebView has no Web Speech API, so an iOS participant's speech was never captured
+// into the meeting transcript (desktop Chrome/Edge already works). This transcribes THIS device's mic with
+// SFSpeechRecognizer, fed by a LiveKit `AudioRenderer` attached to the local mic track — so it reuses the call's
+// already-open capture (no second AVAudioEngine fighting WebRTC for the audio session). Each finished utterance
+// fires `onFinal`; the plugin relays it to the web, which sends it over the meeting WS exactly like the desktop
+// path. Segments are cut on a short silence gap (and on the recognizer's own isFinal), and the request is
+// restarted per segment so text flows continuously and stays within the recognizer's limits. On-device
+// recognition is used when available (offline, continuous, no ~1-minute cap). All state is main-confined except
+// the locked `request` that the audio-thread renderer appends to.
+final class SpeechTranscriber: NSObject, AudioRenderer, @unchecked Sendable {
+    var onFinal: ((String) -> Void)?
+
+    private let recognizer = SFSpeechRecognizer(locale: Locale(identifier: "en-US"))
+    private let lock = NSLock()
+    private var request: SFSpeechAudioBufferRecognitionRequest?
+    private var task: SFSpeechRecognitionTask?
+    private weak var track: LocalAudioTrack?
+    private var running = false
+    private var latest = ""
+    private var lastEmitted = ""
+    private var silenceTimer: DispatchWorkItem?
+
+    // Begin transcription (idempotent). Asks for Speech authorization once, then starts recognition and taps the
+    // current mic track. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute).
+    func start(track: LocalAudioTrack?) {
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { return }
+            if self.running { self.attach(track); return }
+            SFSpeechRecognizer.requestAuthorization { status in
+                DispatchQueue.main.async {
+                    guard status == .authorized, !self.running else { return }
+                    self.running = true
+                    self.beginRequest()
+                    self.attach(track)
+                }
+            }
+        }
+    }
+
+    // (Re)attach to the current local mic track — called on start and whenever the mic (re)publishes on unmute.
+    func refresh(track: LocalAudioTrack?) { DispatchQueue.main.async { [weak self] in self?.attach(track) } }
+
+    func stop() {
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { return }
+            self.running = false
+            self.silenceTimer?.cancel(); self.silenceTimer = nil
+            self.track?.remove(audioRenderer: self); self.track = nil
+            self.lock.lock(); let r = self.request; self.request = nil; self.lock.unlock()
+            r?.endAudio(); self.task?.cancel(); self.task = nil
+        }
+    }
+
+    private func attach(_ t: LocalAudioTrack?) {
+        guard running, let t = t, track !== t else { return }
+        track?.remove(audioRenderer: self)
+        t.add(audioRenderer: self)
+        track = t
+    }
+
+    private func beginRequest() {
+        guard let recognizer = recognizer, recognizer.isAvailable else { return }
+        let req = SFSpeechAudioBufferRecognitionRequest()
+        req.shouldReportPartialResults = true // stream results; we only EMIT a segment on silence / isFinal
+        if recognizer.supportsOnDeviceRecognition { req.requiresOnDeviceRecognition = true }
+        lock.lock(); request = req; lock.unlock()
+        latest = ""; lastEmitted = ""
+        task = recognizer.recognitionTask(with: req) { [weak self] result, error in
+            guard let self = self else { return }
+            if let result = result {
+                let text = result.bestTranscription.formattedString
+                let isFinal = result.isFinal
+                DispatchQueue.main.async {
+                    guard self.running else { return }
+                    self.latest = text
+                    if isFinal { self.flushAndRestart() } else { self.armSilenceTimer() }
+                }
+            } else if error != nil {
+                DispatchQueue.main.async { if self.running { self.flushAndRestart() } }
+            }
+        }
+    }
+
+    // A short pause = end of an utterance → emit it and start a fresh request for the next one.
+    private func armSilenceTimer() {
+        silenceTimer?.cancel()
+        let work = DispatchWorkItem { [weak self] in guard let self = self, self.running else { return }; self.flushAndRestart() }
+        silenceTimer = work
+        DispatchQueue.main.asyncAfter(deadline: .now() + 1.4, execute: work)
+    }
+
+    private func flushAndRestart() {
+        silenceTimer?.cancel(); silenceTimer = nil
+        let text = latest.trimmingCharacters(in: .whitespacesAndNewlines)
+        if !text.isEmpty && text != lastEmitted { lastEmitted = text; onFinal?(text) }
+        guard running else { return }
+        lock.lock(); let r = request; request = nil; lock.unlock()
+        r?.endAudio(); task?.cancel(); task = nil
+        beginRequest()
+    }
+
+    // MARK: AudioRenderer — receives the local mic PCM from LiveKit (audio thread).
+    func render(pcmBuffer: AVAudioPCMBuffer) {
+        lock.lock(); let r = request; lock.unlock()
+        r?.append(pcmBuffer)
+    }
+}