From e46ac1e7cc8855cb44a58c928a9f2708f71620a2 Mon Sep 17 00:00:00 2001 From: sravan Date: Fri, 14 Aug 2026 17:05:56 +0530 Subject: [PATCH] #5 transcript: cover scheduled/web meetings on iOS too (not just native calls) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a scheduled meeting the WebView owns the mic (the plugin has no LiveKit room), so the native-call AudioRenderer tap didn't apply. Now the WebView reads its OWN local mic PCM via Web Audio (the same non-intrusive tap the app already uses for active-speaker metering — NOT a 2nd getUserMedia, which would fight the call's mic on iOS and yield silence), downsamples to 16 kHz mono Int16, and forwards it to the plugin's SFSpeechRecognizer via feedAudio(). Native calls keep the LiveKit tap. Muted -> the local track carries silence -> nothing transcribed; the feed re-inits when the mic goes live on unmute. - Plugin: startTranscription({external:true}) runs the recognizer without a track; feedAudio({pcm,rate}) decodes base64 LE Int16 -> Float32 buffer -> recognizer. - Web: startSR now uses the native recognizer for ALL iOS meetings (native call = LiveKit tap, scheduled = PCM feed); desktop/browser unchanged (Web Speech API). Web deploys now; the plugin's feedAudio/startExternal ride the next Codemagic build. Co-Authored-By: Claude Opus 4.8 --- .../NativeCallPlugin/NativeCallPlugin.swift | 58 +++++++++++++++---- server/public/home.html | 55 +++++++++++++++--- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index 46fc41a..a7e821a 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -39,6 +39,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "startTranscription", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "stopTranscription", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "feedAudio", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] @@ -398,12 +399,18 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega return room?.localParticipant.audioTracks.first?.track as? LocalAudioTrack } - // Start transcribing this device's mic. The web calls this when the room's transcript is active and we're - // in a native call (WKWebView has no Web Speech API). Attaches a LiveKit AudioRenderer to the local mic - // track and runs SFSpeechRecognizer; finalized segments fire the "transcript" event. Safe to call when the - // mic track doesn't exist yet — we re-attach on unmute (setMicrophone-enable) too. + // Start transcribing this device's mic (WKWebView has no Web Speech API). Two audio sources: + // * NATIVE CALL (default): the plugin owns the LiveKit room, so we tap the local mic track with a LiveKit + // AudioRenderer (reuses the call's open mic — reliable, no 2nd capturer). Re-attaches on unmute. + // * SCHEDULED/WEB MEETING ({external:true}): the WebView owns the mic (the plugin has no room), so the web + // reads its own mic PCM via Web Audio and pushes it here with feedAudio() — no second mic capture on iOS + // (two input units would fight and yield silence). @objc func startTranscription(_ call: CAPPluginCall) { - transcriber.start(track: localAudioTrack()) + if call.getBool("external") == true { + transcriber.startExternal() + } else { + transcriber.start(track: localAudioTrack()) + } call.resolve() } @@ -412,6 +419,15 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } + // Web-forwarded mic PCM for the {external:true} path (scheduled/web meetings). `pcm` = base64 little-endian + // Int16 mono at `rate` Hz (the web downsamples to 16 kHz). Fed straight into the recognizer. + @objc func feedAudio(_ call: CAPPluginCall) { + guard let b64 = call.getString("pcm"), let data = Data(base64Encoded: b64) else { call.resolve(); return } + let rate = call.getDouble("rate") ?? 16000 + transcriber.appendPCM(int16: data, sampleRate: rate) + call.resolve() + } + // MARK: - BroadcastManagerDelegate public func broadcastManager(didChangeState isBroadcasting: Bool) { @@ -667,23 +683,45 @@ final class SpeechTranscriber: NSObject, AudioRenderer, @unchecked Sendable { private var lastEmitted = "" private var silenceTimer: DispatchWorkItem? - // Begin transcription (idempotent). Asks for Speech authorization once, then starts recognition and taps the - // current mic track. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute). - func start(track: LocalAudioTrack?) { + // Begin transcription tapping a LiveKit local mic track (native calls). Idempotent; asks for Speech + // authorization once. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute). + func start(track: LocalAudioTrack?) { begin { self.attach(track) } } + + // Begin transcription in EXTERNAL mode (scheduled/web meetings): no LiveKit track to tap — PCM arrives via + // appendPCM() from the web. Same recognition pipeline. + func startExternal() { begin { } } + + private func begin(_ afterStart: @escaping () -> Void) { DispatchQueue.main.async { [weak self] in guard let self = self else { return } - if self.running { self.attach(track); return } + if self.running { afterStart(); return } SFSpeechRecognizer.requestAuthorization { status in DispatchQueue.main.async { guard status == .authorized, !self.running else { return } self.running = true self.beginRequest() - self.attach(track) + afterStart() } } } } + // Web-forwarded PCM (external mode): little-endian Int16 mono → Float32 buffer → recognizer. + func appendPCM(int16 data: Data, sampleRate: Double) { + let count = data.count / 2 + guard count > 0, + let fmt = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false), + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: AVAudioFrameCount(count)) else { return } + buf.frameLength = AVAudioFrameCount(count) + guard let dst = buf.floatChannelData?[0] else { return } + data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in + let src = raw.bindMemory(to: Int16.self) + for i in 0..{ const text=((e&&e.text)||'').trim(); if(text) meetSend({type:'meeting-transcript', text}); }); }catch(_){} - try{ const p=NC.startTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} + try{ const p=NC.startTranscription(meetNative?{}:{external:true}); if(p&&p.catch) p.catch(()=>{}); }catch(_){} + if(!meetNative) _startWebPcmFeed(); // scheduled meeting → stream the WebView's own mic to the recognizer return; - }} + } const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR){ if(meetTranscribe) toast('Live transcript needs Chrome or Edge'); return; } try{ meetSR=new SR(); }catch(_){ return; } meetSR.continuous=true; meetSR.interimResults=false; meetSR.lang='en-US'; @@ -5795,9 +5800,45 @@ function startSR(){ if(meetSR) return; } function _removeTxListener(){ const h=_meetTxListener; _meetTxListener=null; if(!h) return; try{ if(h.remove) h.remove(); else if(h.then) h.then(x=>{ try{ x&&x.remove&&x.remove(); }catch(_){} }); }catch(_){} } function stopSR(){ - if(meetSR==='native'){ meetSR=null; _removeTxListener(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; } + if(meetSR==='native'){ meetSR=null; _removeTxListener(); _stopWebPcmFeed(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; } if(meetSR){ try{ meetSR.onend=null; meetSR.stop(); }catch(_){} meetSR=null; } } +// #5 (scheduled/web meetings on iOS): the WebView owns the mic, so read its PCM via Web Audio — the same +// non-intrusive tap the app already uses for active-speaker metering (NOT a 2nd getUserMedia, which would fight +// the call's mic) — downsample to 16 kHz mono Int16, and push it to the plugin's recognizer (feedAudio). While +// muted the local track carries silence, so nothing is transcribed. Re-inited on unmute (the mic track appears). +let _txCtx=null,_txProc=null,_txSrc=null,_txGain=null; +function _txMicTrack(){ try{ const ts=meetLocalStream&&meetLocalStream.getAudioTracks&&meetLocalStream.getAudioTracks(); return (ts&&ts[0])||null; }catch(_){ return null; } } +function _startWebPcmFeed(){ + _stopWebPcmFeed(); + const NC=nativeCallPlugin(); if(!NC||!NC.feedAudio) return; + const track=_txMicTrack(); if(!track) return; // mic not live yet — retried on unmute + try{ + const Ctx=window.AudioContext||window.webkitAudioContext; if(!Ctx) return; + _txCtx=new Ctx(); + _txSrc=_txCtx.createMediaStreamSource(new MediaStream([track])); + _txProc=_txCtx.createScriptProcessor(4096,1,1); + const inRate=_txCtx.sampleRate||48000, outRate=16000, ratio=inRate/outRate; + _txProc.onaudioprocess=(ev)=>{ + try{ + const inp=ev.inputBuffer.getChannelData(0); const outLen=Math.max(0,Math.floor(inp.length/ratio)); + if(!outLen) return; const i16=new Int16Array(outLen); + for(let i=0;i1?1:s); i16[i]=s<0?s*0x8000:s*0x7fff; } + const bytes=new Uint8Array(i16.buffer); let bin=''; for(let i=0;i0; if(!hasTrack){ let astream; try{ astream=await navigator.mediaDevices.getUserMedia({ audio:true }); }