Sravan/BizGaze_Remote
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

#5 transcript: cover scheduled/web meetings on iOS too (not just native calls)

Browse Source 
In a scheduled meeting the WebView owns the mic (the plugin has no LiveKit room), so
the native-call AudioRenderer tap didn't apply. Now the WebView reads its OWN local
mic PCM via Web Audio (the same non-intrusive tap the app already uses for
active-speaker metering — NOT a 2nd getUserMedia, which would fight the call's mic on
iOS and yield silence), downsamples to 16 kHz mono Int16, and forwards it to the
plugin's SFSpeechRecognizer via feedAudio(). Native calls keep the LiveKit tap.
Muted -> the local track carries silence -> nothing transcribed; the feed re-inits
when the mic goes live on unmute.

- Plugin: startTranscription({external:true}) runs the recognizer without a track;
  feedAudio({pcm,rate}) decodes base64 LE Int16 -> Float32 buffer -> recognizer.
- Web: startSR now uses the native recognizer for ALL iOS meetings (native call =
  LiveKit tap, scheduled = PCM feed); desktop/browser unchanged (Web Speech API).

Web deploys now; the plugin's feedAudio/startExternal ride the next Codemagic build.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Sravan
2026-08-14 17:05:56 +05:30
parent 68ff3a878b
commit e46ac1e7cc
2 changed files with 96 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/public/home.html
+48 -7
View File
@@ -5778,14 +5778,19 @@ async function uploadRecording(blob, durMs){
function toggleTranscribe(){ meetTranscribe=!meetTranscribe; meetSend({type:'meeting-transcribe', on:meetTranscribe}); updateTransBtn(); toast(meetTranscribe?'Transcript on — your private copy is saved to Past meetings after the call':'You left the transcript — your copy is being saved'); }
function applyRoomTx(active){ if(active===meetRoomTx) return; meetRoomTx=active; if(active) startSR(); else stopSR(); transcribeNotice(active); }
function startSR(){ if(meetSR) return;
// #5: on an iOS NATIVE call the WKWebView has no Web Speech API — transcribe via the native plugin
// (SFSpeechRecognizer tapping the call's mic). Its 'transcript' events feed the same meeting-transcript path.
if(meetNative){ const NC=nativeCallPlugin(); if(NC && NC.startTranscription){
// #5: the iOS WKWebView has no Web Speech API, so transcribe via the native plugin (SFSpeechRecognizer).
// Its 'transcript' events feed the same meeting-transcript path as the desktop Web Speech API. Two mic sources:
// * NATIVE call (meetNative): the plugin owns the LiveKit mic → it taps its own track.
// * SCHEDULED/web meeting: the WebView owns the mic → we read its PCM (Web Audio, no 2nd capture) and push it
// to the plugin via feedAudio(). On desktop/browser NC is null → falls through to the Web Speech API below.
const NC=nativeCallPlugin();
if(NC && NC.startTranscription){
meetSR='native';
try{ _meetTxListener=NC.addListener('transcript', (e)=>{ const text=((e&&e.text)||'').trim(); if(text) meetSend({type:'meeting-transcript', text}); }); }catch(_){}
try{ const p=NC.startTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){}
try{ const p=NC.startTranscription(meetNative?{}:{external:true}); if(p&&p.catch) p.catch(()=>{}); }catch(_){}
if(!meetNative) _startWebPcmFeed(); // scheduled meeting → stream the WebView's own mic to the recognizer
return;
}}
}
const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR){ if(meetTranscribe) toast('Live transcript needs Chrome or Edge'); return; }
try{ meetSR=new SR(); }catch(_){ return; }
meetSR.continuous=true; meetSR.interimResults=false; meetSR.lang='en-US';
@@ -5795,9 +5800,45 @@ function startSR(){ if(meetSR) return;
}
function _removeTxListener(){ const h=_meetTxListener; _meetTxListener=null; if(!h) return; try{ if(h.remove) h.remove(); else if(h.then) h.then(x=>{ try{ x&&x.remove&&x.remove(); }catch(_){} }); }catch(_){} }
function stopSR(){
if(meetSR==='native'){ meetSR=null; _removeTxListener(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; }
if(meetSR==='native'){ meetSR=null; _removeTxListener(); _stopWebPcmFeed(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; }
if(meetSR){ try{ meetSR.onend=null; meetSR.stop(); }catch(_){} meetSR=null; }
}
// #5 (scheduled/web meetings on iOS): the WebView owns the mic, so read its PCM via Web Audio — the same
// non-intrusive tap the app already uses for active-speaker metering (NOT a 2nd getUserMedia, which would fight
// the call's mic) — downsample to 16 kHz mono Int16, and push it to the plugin's recognizer (feedAudio). While
// muted the local track carries silence, so nothing is transcribed. Re-inited on unmute (the mic track appears).
let _txCtx=null,_txProc=null,_txSrc=null,_txGain=null;
function _txMicTrack(){ try{ const ts=meetLocalStream&&meetLocalStream.getAudioTracks&&meetLocalStream.getAudioTracks(); return (ts&&ts[0])||null; }catch(_){ return null; } }
function _startWebPcmFeed(){
_stopWebPcmFeed();
const NC=nativeCallPlugin(); if(!NC||!NC.feedAudio) return;
const track=_txMicTrack(); if(!track) return; // mic not live yet — retried on unmute
try{
const Ctx=window.AudioContext||window.webkitAudioContext; if(!Ctx) return;
_txCtx=new Ctx();
_txSrc=_txCtx.createMediaStreamSource(new MediaStream([track]));
_txProc=_txCtx.createScriptProcessor(4096,1,1);
const inRate=_txCtx.sampleRate||48000, outRate=16000, ratio=inRate/outRate;
_txProc.onaudioprocess=(ev)=>{
try{
const inp=ev.inputBuffer.getChannelData(0); const outLen=Math.max(0,Math.floor(inp.length/ratio));
if(!outLen) return; const i16=new Int16Array(outLen);
for(let i=0;i<outLen;i++){ let s=inp[Math.floor(i*ratio)]; s=s<-1?-1:(s>1?1:s); i16[i]=s<0?s*0x8000:s*0x7fff; }
const bytes=new Uint8Array(i16.buffer); let bin=''; for(let i=0;i<bytes.length;i++) bin+=String.fromCharCode(bytes[i]);
const NC2=nativeCallPlugin(); if(NC2&&NC2.feedAudio) NC2.feedAudio({ pcm:btoa(bin), rate:outRate });
}catch(_){}
};
_txGain=_txCtx.createGain(); _txGain.gain.value=0; // silent sink so the processor runs without echoing the mic
_txSrc.connect(_txProc); _txProc.connect(_txGain); _txGain.connect(_txCtx.destination);
}catch(_){ _stopWebPcmFeed(); }
}
function _stopWebPcmFeed(){
try{ if(_txProc){ _txProc.onaudioprocess=null; _txProc.disconnect(); } }catch(_){}
try{ if(_txSrc) _txSrc.disconnect(); }catch(_){}
try{ if(_txGain) _txGain.disconnect(); }catch(_){}
try{ if(_txCtx) _txCtx.close(); }catch(_){}
_txProc=null; _txSrc=null; _txGain=null; _txCtx=null;
}
function updateTransBtn(){ const b=document.getElementById('meetTransBtn'); if(!b) return; b.classList.toggle('on', meetTranscribe); b.title=meetTranscribe?'Stop my transcript':'Transcribe (your private copy)'; }
function transcribeNotice(on){ let el=document.getElementById('txNotice'); if(on){ if(!el){ el=document.createElement('div'); el.id='txNotice'; el.className='tx-notice'; el.innerHTML=ic('fileText',12)+' Transcribing'; document.body.appendChild(el); } } else if(el){ el.remove(); } }
// iOS blocks media/WebRTC audio playback until a user gesture, so remote call audio stays SILENT until you
@@ -6005,7 +6046,7 @@ function updateFlipBtn(){ const b=document.getElementById('meetFlipBtn'); if(b)
async function toggleMic(){
if(!meetLocalStream) return;
if(meetNative){ const next=!meetMic; const NC=nativeCallPlugin(); if(NC){ try{ NC.setMuted({ muted:!next }); }catch(_){} } meetMic=next; updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); return; } // native: mute the plugin's mic
if(SFU.on){ const next=!meetMic; try{ await sfuSetMic(next); meetMic=next; }catch(e){ toast(mediaErrMsg(e,'microphone')); return; } updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); return; }
if(SFU.on){ const next=!meetMic; try{ await sfuSetMic(next); meetMic=next; }catch(e){ toast(mediaErrMsg(e,'microphone')); return; } updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); if(meetSR==='native' && next) _startWebPcmFeed(); /* #5: mic just went live → (re)connect the PCM feed to it */ return; }
const hasTrack=meetLocalStream.getAudioTracks().length>0;
if(!hasTrack){
let astream; try{ astream=await navigator.mediaDevices.getUserMedia({ audio:true }); }