iOS remote support: Share Screen -> Connect Screen over LiveKit (core)
Rebuild the iOS remote-support screen share the RIGHT way: keep the exact Share/Connect UX + session (consent + symmetric session-ended), swap only the media to LiveKit since WKWebView can't getDisplayMedia. (Replaces the earlier "route to a meeting" detour, which was reverted.) Flow: customer taps Share Screen -> gets a 6-digit code (unchanged UI) -> helper enters it in Connect Screen -> on the customer's "Allow", the app publishes the screen natively (ReplayKit -> LiveKit) into a per-session room and tells the agent it's a LiveKit session -> connect.html joins that room and shows the screen in its existing viewer (recording/controls intact). Chat runs over the session socket (no P2P data channel in this mode). Either side ending fires the existing session-ended -> both tear down (symmetric disconnect). - signaling.js: relay 'rs-livekit' + 'rs-chat' between the two ends. - home.html: parent bridge so the /share iframe can drive startMeetingScreenShare/ stop on the native plugin (+ a capability handshake). - share.html (iOS): publish via native LiveKit instead of getDisplayMedia; chat over WS; hide mic (voice = next iteration) + remote-control (impossible on iOS). - connect.html: LiveKit viewer for iOS-shared sessions, reusing the P2P viewer. Web-only, no new build (reuses the shipped startMeetingScreenShare). Desktop Share/Connect P2P unchanged. Two-way voice is the planned follow-up. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -106,6 +106,26 @@ const card=document.getElementById('card'), wrap=document.getElementById('wrap')
|
||||
agentChip=document.getElementById('agentChip'), bar=document.getElementById('bar'),
|
||||
topbar=document.getElementById('topbar'), video=document.getElementById('video'), barStatus=document.getElementById('barStatus');
|
||||
let ws,pc,inputChannel,chatChannel,sessionId,me=null;
|
||||
let RS_LK=false, lkRoom=null; // iOS-shared session: view the screen over LiveKit instead of P2P
|
||||
// Load the LiveKit browser SDK on demand (same vendored build the meeting UI uses).
|
||||
function sfuLoadLib(){ return new Promise((res,rej)=>{ if(window.LivekitClient) return res(window.LivekitClient); const s=document.createElement('script'); s.src='/vendor/livekit-client.umd.min.js'; s.onload=()=>res(window.LivekitClient); s.onerror=()=>rej(new Error('livekit sdk failed to load')); document.head.appendChild(s); }); }
|
||||
// The customer is on iPhone (WKWebView can't getDisplayMedia), so they publish their screen over LiveKit.
|
||||
// Join that room and show the screen in the SAME viewer we use for P2P (recording/chat/controls unchanged).
|
||||
async function startLiveKitView(room){
|
||||
const statusEl=document.getElementById('status');
|
||||
if(statusEl){ statusEl.className='status'; statusEl.innerHTML='<img src="/loaders/loader-orbit.svg" width="20" height="20" style="vertical-align:-5px;margin-right:7px" alt="">Connecting to the shared screen…'; }
|
||||
try{
|
||||
const LK=await sfuLoadLib();
|
||||
const tk=await fetch('/api/meetings/token',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({room})}).then(r=>r.json());
|
||||
if(!tk||!tk.token) throw new Error('no token');
|
||||
const r=new LK.Room({adaptiveStream:false,dynacast:false}); lkRoom=r;
|
||||
const showVideo=(mst)=>{ video.srcObject=new MediaStream([mst]); if(typeof wrap!=='undefined'&&wrap) wrap.style.display='none'; if(typeof topbar!=='undefined'&&topbar) topbar.style.display='none'; video.style.display='block'; try{ video.play(); }catch(_){}; try{ video.focus(); }catch(_){}; buildBar(); };
|
||||
const attach=(track)=>{ if(!track) return; const mst=track.mediaStreamTrack; if(track.kind==='video'){ showVideo(mst); } else { let a=document.getElementById('remoteAudio'); if(!a){ a=document.createElement('audio'); a.id='remoteAudio'; a.autoplay=true; document.body.appendChild(a); } a.srcObject=new MediaStream([mst]); } };
|
||||
r.on(LK.RoomEvent.TrackSubscribed,(track)=>attach(track));
|
||||
await r.connect(tk.url, tk.token);
|
||||
try{ r.remoteParticipants.forEach((p)=>{ p.trackPublications.forEach((pub)=>{ if(pub.track) attach(pub.track); }); }); }catch(_){}
|
||||
}catch(e){ if(statusEl){ statusEl.className='status err'; statusEl.textContent='Could not connect to the shared screen.'; } }
|
||||
}
|
||||
|
||||
async function api(path,body,method='POST'){
|
||||
const opt={method,headers:{'Content-Type':'application/json'}};
|
||||
@@ -191,6 +211,8 @@ function connectWS(){
|
||||
const ans=await pc.createAnswer(); await pc.setLocalDescription(ans);
|
||||
ws.send(JSON.stringify({type:'answer',sessionId,sdp:pc.localDescription})); break;
|
||||
case 'ice-candidate': if(m.candidate&&pc) await pc.addIceCandidate(new RTCIceCandidate(m.candidate)); break;
|
||||
case 'rs-livekit': RS_LK=true; startLiveKitView(m.room); break; // iOS customer shares over LiveKit — view it there
|
||||
case 'rs-chat': if(m.msg) addChat({from:'other',name:m.msg.name||'Customer',text:m.msg.text}); break; // chat over the socket in LiveKit mode
|
||||
case 'transcript': if(recogActive&&m.text) addLine('customer', m.name||'Customer', m.text, !!m.chat); break;
|
||||
case 'session-denied': renderEnded('The customer declined the request.'); break;
|
||||
case 'session-ended': {
|
||||
@@ -213,6 +235,7 @@ function renderWaiting(){
|
||||
|
||||
function renderEnded(msg){
|
||||
bzcSession(false);
|
||||
try{ if(lkRoom){ lkRoom.disconnect(); lkRoom=null; } }catch(_){} // tear down the LiveKit view (symmetric disconnect)
|
||||
try{ stopRecording(); }catch(_){}
|
||||
removeSessionUI();
|
||||
document.body.classList.remove('has-bar');
|
||||
@@ -385,7 +408,10 @@ let __ac=null;
|
||||
function ensureAudio(){try{__ac=__ac||new (window.AudioContext||window.webkitAudioContext)();if(__ac.state==='suspended')__ac.resume();}catch(_){}}
|
||||
function beep(){ensureAudio();if(!__ac)return;try{const o=__ac.createOscillator(),g=__ac.createGain();o.type='sine';o.connect(g);g.connect(__ac.destination);const t0=__ac.currentTime;o.frequency.setValueAtTime(880,t0);o.frequency.setValueAtTime(660,t0+0.09);g.gain.setValueAtTime(0.0001,t0);g.gain.exponentialRampToValueAtTime(0.12,t0+0.02);g.gain.exponentialRampToValueAtTime(0.0001,t0+0.22);o.start(t0);o.stop(t0+0.24);}catch(_){}}
|
||||
try{['pointerdown','keydown','touchstart','click'].forEach(ev=>document.addEventListener(ev,ensureAudio,{passive:true}));}catch(_){}
|
||||
function sendChat(){const i=document.getElementById('chatInput');if(!i)return;const t=i.value.trim();if(!t)return;if(chatChannel&&chatChannel.readyState==='open'){chatChannel.send(JSON.stringify({name:(me&&(me.name||me.email))||'Support agent',text:t}));}addChat({from:'__self',name:'You',text:t});if(recogActive)addLine('agent',(me&&(me.name||me.email))||'Agent',t,true);i.value='';}
|
||||
function sendChat(){const i=document.getElementById('chatInput');if(!i)return;const t=i.value.trim();if(!t)return;
|
||||
if(RS_LK){ try{ ws.send(JSON.stringify({type:'rs-chat',sessionId,msg:{name:(me&&(me.name||me.email))||'Support agent',text:t}})); }catch(_){} }
|
||||
else if(chatChannel&&chatChannel.readyState==='open'){chatChannel.send(JSON.stringify({name:(me&&(me.name||me.email))||'Support agent',text:t}));}
|
||||
addChat({from:'__self',name:'You',text:t});if(recogActive)addLine('agent',(me&&(me.name||me.email))||'Agent',t,true);i.value='';}
|
||||
function removeSessionUI(){['sessionBar','chatPanel','remoteAudio','muteBtn','msgToast'].forEach(id=>{const e=document.getElementById(id);if(e)e.remove();});}
|
||||
|
||||
async function setupPeer(){
|
||||
|
||||
@@ -6587,6 +6587,12 @@ async function doRegister(){
|
||||
window.addEventListener('message',(e)=>{
|
||||
if(e.origin!==location.origin) return; const d=e.data||{}; const n=window.bizConnectNative;
|
||||
if(d.type==='rc-ping'){ const desktop=!!(n&&n.rcAvailable&&n.rcAvailable()); try{ e.source&&e.source.postMessage({type:'rc-pong', desktop}, location.origin); }catch(_){} return; }
|
||||
// iOS remote-support: the /share iframe can't reach the native plugin, so it asks THIS top frame to
|
||||
// publish the screen over LiveKit (WKWebView can't getDisplayMedia). Answer the capability handshake and
|
||||
// drive startMeetingScreenShare/stopMeetingScreenShare (the same native method meetings use).
|
||||
if(d.type==='bzc-native-ping'){ const ok=bzIsIOS() && !!(nativeCallPlugin() && nativeCallPlugin().startMeetingScreenShare); try{ e.source&&e.source.postMessage({type:'bzc-native', ok}, location.origin); }catch(_){} return; }
|
||||
if(d.type==='rs-native-share'){ (async()=>{ try{ const NC=nativeCallPlugin(); if(!NC||!NC.startMeetingScreenShare||!d.room) return; const tk=await postJSON('/api/meetings/token',{ room:d.room, screen:true }); await NC.startMeetingScreenShare({ url:(tk.url||SFU.url), token:tk.token }); }catch(_){} })(); return; }
|
||||
if(d.type==='rs-native-stop'){ try{ const NC=nativeCallPlugin(); if(NC&&NC.stopMeetingScreenShare) NC.stopMeetingScreenShare(); }catch(_){} return; }
|
||||
if(!n) return;
|
||||
if(d.type==='rc-arm'){ try{ n.rcArm&&n.rcArm(!!d.on); }catch(_){} return; }
|
||||
if(d.type==='rc-input'){ try{ n.rcInput&&n.rcInput(d.evt); }catch(_){} return; }
|
||||
|
||||
@@ -106,6 +106,10 @@ let ICE={iceServers:[{urls:'stun:stun.l.google.com:19302'}]};
|
||||
let SHARER_NAME='Customer';
|
||||
try{fetch('/api/me').then(r=>r.ok?r.json():null).then(m=>{if(m&&(m.name||m.email))SHARER_NAME=m.name||m.email;}).catch(()=>{});}catch(_){}
|
||||
const IS_MOBILE=/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile/i.test(navigator.userAgent||'');
|
||||
// iOS app: WKWebView can't getDisplayMedia. Ask the top frame (home.html) if it can publish the screen
|
||||
// natively (ReplayKit -> LiveKit). If so, we run this session over LiveKit instead of P2P.
|
||||
let NATIVE_IOS=false, RS_ROOM=null;
|
||||
try{ if(window.parent && window.parent!==window){ window.addEventListener('message',(e)=>{ if(e.origin!==location.origin) return; const d=e.data||{}; if(d.type==='bzc-native'){ NATIVE_IOS=!!d.ok; } }); window.parent.postMessage({type:'bzc-native-ping'}, location.origin); } }catch(_){}
|
||||
let __icePromise=Promise.resolve();try{__icePromise=fetch('/api/ice').then(r=>r.ok?r.json():null).then(c=>{if(c&&c.iceServers)ICE=c;}).catch(()=>{});}catch(_){}
|
||||
async function ensureIce(){try{await __icePromise;}catch(_){}return ICE;}
|
||||
function pEsc(s){return String(s==null?'':s).replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));}
|
||||
@@ -138,6 +142,7 @@ ws.onmessage=async(e)=>{const m=JSON.parse(e.data);switch(m.type){
|
||||
case 'answer': if(pc) await pc.setRemoteDescription(new RTCSessionDescription(m.sdp)); break;
|
||||
case 'ice-candidate': if(m.candidate&&pc) await pc.addIceCandidate(new RTCIceCandidate(m.candidate)); break;
|
||||
case 'recording': recNotice(m.on); if(m.on) startCustTranscription(); else stopCustTranscription(); break;
|
||||
case 'rs-chat': if(m.msg) addChat({from:'other', name:m.msg.name||'Agent', text:m.msg.text}); break; // chat over the socket in LiveKit mode (no P2P data channel)
|
||||
case 'session-ended': endShareSession('Your support agent ended the session. Tap below for a new code if you still need help.'); break;
|
||||
case 'error': setStatus(m.message,''); break;
|
||||
}};
|
||||
@@ -169,6 +174,7 @@ function showConsent(m){
|
||||
// getDisplayMedia unless it is called from a user gesture, so this must not run
|
||||
// after a server round-trip. getDisplayMedia is called first to keep the gesture.
|
||||
async function beginCapture(){
|
||||
if(NATIVE_IOS){ return true; } // iOS: the app captures via ReplayKit on start-stream (no getDisplayMedia in WKWebView)
|
||||
try{ localStream=await navigator.mediaDevices.getDisplayMedia({video:{displaySurface:'monitor',frameRate:{ideal:30}},audio:false,monitorTypeSurfaces:'include'}); }
|
||||
catch(err){ return false; }
|
||||
// Mic is OFF by default — we do NOT prompt for it here. Asking for the screen and the
|
||||
@@ -178,6 +184,18 @@ async function beginCapture(){
|
||||
return true;
|
||||
}
|
||||
async function startStreaming(){
|
||||
// iOS: publish the screen NATIVELY over LiveKit (WKWebView can't getDisplayMedia). Tell the app to start the
|
||||
// ReplayKit broadcast into a room derived from this session, and tell the agent to view over LiveKit.
|
||||
if(NATIVE_IOS){
|
||||
RS_ROOM='rs'+String(sessionId||'').replace(/[^A-Za-z0-9]/g,'').slice(0,60);
|
||||
try{ window.parent.postMessage({type:'rs-native-share', room:RS_ROOM}, location.origin); }catch(_){}
|
||||
try{ ws.send(JSON.stringify({type:'rs-livekit', sessionId, room:RS_ROOM})); }catch(_){}
|
||||
indicator.classList.add('show'); setStatus('You are now sharing your screen with your agent.','on'); bzcSession(true);
|
||||
{ const hl=document.getElementById('homeLink'); if(hl) hl.style.display='none'; }
|
||||
window.onbeforeunload=function(){ if(!sessionOver){ return 'Leaving this page will end your screen sharing session.'; } };
|
||||
buildBar();
|
||||
return;
|
||||
}
|
||||
// If the Allow tap already captured the screen (mobile path), reuse it.
|
||||
if(!localStream){
|
||||
await ensureIce();
|
||||
@@ -320,6 +338,7 @@ function recNotice(on){
|
||||
} else { clearInterval(recTimerInt); recTimerInt=null; if(n) n.remove(); }
|
||||
}
|
||||
function endShareSession(msgText){
|
||||
if(NATIVE_IOS){ try{ window.parent.postMessage({type:'rs-native-stop'}, location.origin); }catch(_){} } // stop the native ReplayKit broadcast + LiveKit
|
||||
try{ rcStopControl(); }catch(_){} // release remote control when the session ends
|
||||
sessionOver=true; window.onbeforeunload=null; bzcSession(false); { const hl=document.getElementById('homeLink'); if(hl) hl.style.display=''; } try{recNotice(false);stopCustTranscription();}catch(_){}
|
||||
removeSessionUI();
|
||||
@@ -330,7 +349,7 @@ function endShareSession(msgText){
|
||||
var card=document.querySelector('.panelside .card');
|
||||
if(card){ card.innerHTML='<h1 style="color:var(--blue)">Session ended</h1><div class="sub">'+esc(msgText||'The session has ended.')+'</div><button onclick="location.reload()" style="width:100%;margin-top:.4rem">Get a new code</button>'; }
|
||||
}
|
||||
function teardown(){try{rcStopControl();}catch(_){}sessionOver=true;window.onbeforeunload=null;bzcSession(false);{const hl=document.getElementById('homeLink');if(hl)hl.style.display='';}try{recNotice(false);stopCustTranscription();}catch(_){}indicator.classList.remove('show');removeSessionUI();if(window.__mic){window.__mic.getTracks().forEach(t=>t.stop());window.__mic=null;}if(localStream){localStream.getTracks().forEach(t=>t.stop());localStream=null;}if(pc){pc.close();pc=null;}consentBox.innerHTML='';setStatus('Session ended. Refresh this page to get a new code.');}
|
||||
function teardown(){if(NATIVE_IOS){try{window.parent.postMessage({type:'rs-native-stop'},location.origin);}catch(_){}}try{rcStopControl();}catch(_){}sessionOver=true;window.onbeforeunload=null;bzcSession(false);{const hl=document.getElementById('homeLink');if(hl)hl.style.display='';}try{recNotice(false);stopCustTranscription();}catch(_){}indicator.classList.remove('show');removeSessionUI();if(window.__mic){window.__mic.getTracks().forEach(t=>t.stop());window.__mic=null;}if(localStream){localStream.getTracks().forEach(t=>t.stop());localStream=null;}if(pc){pc.close();pc=null;}consentBox.innerHTML='';setStatus('Session ended. Refresh this page to get a new code.');}
|
||||
|
||||
let chatOpen=false;
|
||||
const SVG_MIC='<svg viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="2" width="6" height="11" rx="3"/><path d="M5 10a7 7 0 0 0 14 0"/><line x1="12" y1="19" x2="12" y2="22"/></svg>';
|
||||
@@ -349,10 +368,10 @@ function buildBar(){
|
||||
const rcb=_btn('rcBtn',I('monitor'),'Remote control is OFF','#6b7280');
|
||||
const chat=_btn('chatBtn',I('chat'),'Chat','#475569');
|
||||
const end=_btn('endBtn2',I('callEnd'),'End','#dc2626');
|
||||
bar.appendChild(mic);bar.appendChild(rcb);bar.appendChild(chat);bar.appendChild(end);
|
||||
if(!NATIVE_IOS){ bar.appendChild(mic); bar.appendChild(rcb); } // iOS: two-way voice is a follow-up; remote control is impossible on iOS
|
||||
bar.appendChild(chat);bar.appendChild(end);
|
||||
document.body.appendChild(bar);
|
||||
rcb.onclick=()=>{ if(rcAllowed) rcStopControl(); else rcGrantControl(); };
|
||||
updateRcBtn();
|
||||
if(!NATIVE_IOS){ rcb.onclick=()=>{ if(rcAllowed) rcStopControl(); else rcGrantControl(); }; updateRcBtn(); }
|
||||
makeBarDraggable(bar,'bzc_sharebar_pos'); // new #4: let the customer move the bar off their content
|
||||
const setMic=(on)=>{mic.title=on?'Mute':'Unmute';mic.innerHTML='<span style="display:inline-flex">'+I(on?'mic':'micOff')+'</span>';mic.style.background=on?'#2563eb':'#6b7280';};
|
||||
mic.onclick=async()=>{
|
||||
@@ -391,7 +410,10 @@ let __ac=null;
|
||||
function ensureAudio(){try{__ac=__ac||new (window.AudioContext||window.webkitAudioContext)();if(__ac.state==='suspended')__ac.resume();}catch(_){}}
|
||||
function beep(){ensureAudio();if(!__ac)return;try{const o=__ac.createOscillator(),g=__ac.createGain();o.type='sine';o.connect(g);g.connect(__ac.destination);const t0=__ac.currentTime;o.frequency.setValueAtTime(880,t0);o.frequency.setValueAtTime(660,t0+0.09);g.gain.setValueAtTime(0.0001,t0);g.gain.exponentialRampToValueAtTime(0.12,t0+0.02);g.gain.exponentialRampToValueAtTime(0.0001,t0+0.22);o.start(t0);o.stop(t0+0.24);}catch(_){}}
|
||||
try{['pointerdown','keydown','touchstart','click'].forEach(ev=>document.addEventListener(ev,ensureAudio,{passive:true}));}catch(_){}
|
||||
function sendChat(){const i=document.getElementById('chatInput');if(!i)return;const t=i.value.trim();if(!t)return;if(chatChannel&&chatChannel.readyState==='open'){chatChannel.send(JSON.stringify({name:SHARER_NAME,text:t}));}addChat({from:'__self',name:'You',text:t});i.value='';}
|
||||
function sendChat(){const i=document.getElementById('chatInput');if(!i)return;const t=i.value.trim();if(!t)return;
|
||||
if(NATIVE_IOS){ try{ ws.send(JSON.stringify({type:'rs-chat',sessionId,msg:{name:SHARER_NAME,text:t}})); }catch(_){} }
|
||||
else if(chatChannel&&chatChannel.readyState==='open'){chatChannel.send(JSON.stringify({name:SHARER_NAME,text:t}));}
|
||||
addChat({from:'__self',name:'You',text:t});i.value='';}
|
||||
function removeSessionUI(){['sessionBar','chatPanel','remoteAudio','muteBtn','msgToast'].forEach(id=>{const e=document.getElementById(id);if(e)e.remove();});}
|
||||
|
||||
function esc(s){return String(s).replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));}
|
||||
|
||||
Reference in New Issue
Block a user