diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index 46fc41a..a7e821a 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -39,6 +39,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "startTranscription", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "stopTranscription", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "feedAudio", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] @@ -398,12 +399,18 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega return room?.localParticipant.audioTracks.first?.track as? LocalAudioTrack } - // Start transcribing this device's mic. The web calls this when the room's transcript is active and we're - // in a native call (WKWebView has no Web Speech API). Attaches a LiveKit AudioRenderer to the local mic - // track and runs SFSpeechRecognizer; finalized segments fire the "transcript" event. Safe to call when the - // mic track doesn't exist yet — we re-attach on unmute (setMicrophone-enable) too. + // Start transcribing this device's mic (WKWebView has no Web Speech API). Two audio sources: + // * NATIVE CALL (default): the plugin owns the LiveKit room, so we tap the local mic track with a LiveKit + // AudioRenderer (reuses the call's open mic — reliable, no 2nd capturer). Re-attaches on unmute. + // * SCHEDULED/WEB MEETING ({external:true}): the WebView owns the mic (the plugin has no room), so the web + // reads its own mic PCM via Web Audio and pushes it here with feedAudio() — no second mic capture on iOS + // (two input units would fight and yield silence). @objc func startTranscription(_ call: CAPPluginCall) { - transcriber.start(track: localAudioTrack()) + if call.getBool("external") == true { + transcriber.startExternal() + } else { + transcriber.start(track: localAudioTrack()) + } call.resolve() } @@ -412,6 +419,15 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } + // Web-forwarded mic PCM for the {external:true} path (scheduled/web meetings). `pcm` = base64 little-endian + // Int16 mono at `rate` Hz (the web downsamples to 16 kHz). Fed straight into the recognizer. + @objc func feedAudio(_ call: CAPPluginCall) { + guard let b64 = call.getString("pcm"), let data = Data(base64Encoded: b64) else { call.resolve(); return } + let rate = call.getDouble("rate") ?? 16000 + transcriber.appendPCM(int16: data, sampleRate: rate) + call.resolve() + } + // MARK: - BroadcastManagerDelegate public func broadcastManager(didChangeState isBroadcasting: Bool) { @@ -667,23 +683,45 @@ final class SpeechTranscriber: NSObject, AudioRenderer, @unchecked Sendable { private var lastEmitted = "" private var silenceTimer: DispatchWorkItem? - // Begin transcription (idempotent). Asks for Speech authorization once, then starts recognition and taps the - // current mic track. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute). - func start(track: LocalAudioTrack?) { + // Begin transcription tapping a LiveKit local mic track (native calls). Idempotent; asks for Speech + // authorization once. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute). + func start(track: LocalAudioTrack?) { begin { self.attach(track) } } + + // Begin transcription in EXTERNAL mode (scheduled/web meetings): no LiveKit track to tap — PCM arrives via + // appendPCM() from the web. Same recognition pipeline. + func startExternal() { begin { } } + + private func begin(_ afterStart: @escaping () -> Void) { DispatchQueue.main.async { [weak self] in guard let self = self else { return } - if self.running { self.attach(track); return } + if self.running { afterStart(); return } SFSpeechRecognizer.requestAuthorization { status in DispatchQueue.main.async { guard status == .authorized, !self.running else { return } self.running = true self.beginRequest() - self.attach(track) + afterStart() } } } } + // Web-forwarded PCM (external mode): little-endian Int16 mono → Float32 buffer → recognizer. + func appendPCM(int16 data: Data, sampleRate: Double) { + let count = data.count / 2 + guard count > 0, + let fmt = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false), + let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: AVAudioFrameCount(count)) else { return } + buf.frameLength = AVAudioFrameCount(count) + guard let dst = buf.floatChannelData?[0] else { return } + data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in + let src = raw.bindMemory(to: Int16.self) + for i in 0..{ const text=((e&&e.text)||'').trim(); if(text) meetSend({type:'meeting-transcript', text}); }); }catch(_){} - try{ const p=NC.startTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} + try{ const p=NC.startTranscription(meetNative?{}:{external:true}); if(p&&p.catch) p.catch(()=>{}); }catch(_){} + if(!meetNative) _startWebPcmFeed(); // scheduled meeting → stream the WebView's own mic to the recognizer return; - }} + } const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR){ if(meetTranscribe) toast('Live transcript needs Chrome or Edge'); return; } try{ meetSR=new SR(); }catch(_){ return; } meetSR.continuous=true; meetSR.interimResults=false; meetSR.lang='en-US'; @@ -5795,9 +5800,45 @@ function startSR(){ if(meetSR) return; } function _removeTxListener(){ const h=_meetTxListener; _meetTxListener=null; if(!h) return; try{ if(h.remove) h.remove(); else if(h.then) h.then(x=>{ try{ x&&x.remove&&x.remove(); }catch(_){} }); }catch(_){} } function stopSR(){ - if(meetSR==='native'){ meetSR=null; _removeTxListener(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; } + if(meetSR==='native'){ meetSR=null; _removeTxListener(); _stopWebPcmFeed(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; } if(meetSR){ try{ meetSR.onend=null; meetSR.stop(); }catch(_){} meetSR=null; } } +// #5 (scheduled/web meetings on iOS): the WebView owns the mic, so read its PCM via Web Audio — the same +// non-intrusive tap the app already uses for active-speaker metering (NOT a 2nd getUserMedia, which would fight +// the call's mic) — downsample to 16 kHz mono Int16, and push it to the plugin's recognizer (feedAudio). While +// muted the local track carries silence, so nothing is transcribed. Re-inited on unmute (the mic track appears). +let _txCtx=null,_txProc=null,_txSrc=null,_txGain=null; +function _txMicTrack(){ try{ const ts=meetLocalStream&&meetLocalStream.getAudioTracks&&meetLocalStream.getAudioTracks(); return (ts&&ts[0])||null; }catch(_){ return null; } } +function _startWebPcmFeed(){ + _stopWebPcmFeed(); + const NC=nativeCallPlugin(); if(!NC||!NC.feedAudio) return; + const track=_txMicTrack(); if(!track) return; // mic not live yet — retried on unmute + try{ + const Ctx=window.AudioContext||window.webkitAudioContext; if(!Ctx) return; + _txCtx=new Ctx(); + _txSrc=_txCtx.createMediaStreamSource(new MediaStream([track])); + _txProc=_txCtx.createScriptProcessor(4096,1,1); + const inRate=_txCtx.sampleRate||48000, outRate=16000, ratio=inRate/outRate; + _txProc.onaudioprocess=(ev)=>{ + try{ + const inp=ev.inputBuffer.getChannelData(0); const outLen=Math.max(0,Math.floor(inp.length/ratio)); + if(!outLen) return; const i16=new Int16Array(outLen); + for(let i=0;i1?1:s); i16[i]=s<0?s*0x8000:s*0x7fff; } + const bytes=new Uint8Array(i16.buffer); let bin=''; for(let i=0;i0; if(!hasTrack){ let astream; try{ astream=await navigator.mediaDevices.getUserMedia({ audio:true }); }