iOS meeting screen share: publish ReplayKit broadcast into the SFU room

Gap: on iOS, screen share only worked in native CallKit calls. In a scheduled /
code-joined SFU meeting the webview called setScreenShareEnabled() ->
getDisplayMedia(), which WKWebView does not implement, so it failed silently.
Production meetings are real LiveKit rooms, so we can publish natively instead.

Fix (native ReplayKit -> same LiveKit room as a dedicated screen participant):
- server: /api/meetings/token accepts screen:true -> mints a distinct
  <peerId>-screen identity (LiveKit allows one connection per identity), so the
  native publisher doesn't collide with the webview's own connection.
- native plugin: startMeetingScreenShare/stopMeetingScreenShare + connectScreenRoom
  (a screen-only LiveKit connection: mic off, no camera, no callConnected) that
  reuses the existing broadcast-extension publishing path.
- webview: toggleScreen routes to the native method on iOS SFU meetings; the
  screenShareState listener now drives the SFU case too (and tears the screen
  connection down on the system "Stop Broadcast"); sfuAttach/sfuDetach map the
  '<peerId>-screen' participant's screen track onto the sharer's tile and suppress
  the phantom person tile + the sharer's own self-view.

Web/server deploy now; the native method needs the next Codemagic build to test
on device. Verified: db-smoke 22/22; Swift braces balanced.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-08-22 01:01:42 +05:30
parent 14eb99edaf
commit 467a8b9c6e
3 changed files with 82 additions and 2 deletions
@@ -34,6 +34,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "startScreenShare", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "startScreenShare", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "stopScreenShare", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "stopScreenShare", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "startMeetingScreenShare", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "stopMeetingScreenShare", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise),
CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise),
@@ -391,6 +393,41 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
call.resolve() call.resolve()
} }
// Screen share into a SCHEDULED / code-joined SFU meeting. The WebView already holds the meeting
// connection (identity = peerId) and WKWebView has no getDisplayMedia so we open a SECOND, screen-ONLY
// native LiveKit connection under a distinct `<peerId>-screen` identity and publish the ReplayKit
// broadcast into the SAME room. The WebView maps that identity back onto the sharer's tile.
@objc func startMeetingScreenShare(_ call: CAPPluginCall) {
let url = call.getString("url") ?? ""
let token = call.getString("token") ?? ""
guard !url.isEmpty, !token.isEmpty else { call.reject("url/token required"); return }
connectScreenRoom(url: url, token: token)
DispatchQueue.main.async { BroadcastManager.shared.requestActivation() } // system broadcast picker
call.resolve()
}
@objc func stopMeetingScreenShare(_ call: CAPPluginCall) {
BroadcastManager.shared.requestStop()
let r = room; room = nil
Task { await r?.disconnect() } // drop the screen-only connection (no call/tile/transcriber side effects)
call.resolve()
}
// A screen-ONLY LiveKit connection (mic off, no camera) used purely to publish the ReplayKit broadcast
// into an SFU meeting. Unlike connectRoom() it enables no mic and fires no callConnected this is not a call.
private func connectScreenRoom(url: String, token: String) {
guard !url.isEmpty, !token.isEmpty else { return }
let old = room
let opts = RoomOptions(defaultScreenShareCaptureOptions: ScreenShareCaptureOptions(useBroadcastExtension: true))
let r = Room(roomOptions: opts)
room = r
Task { [weak self] in
await old?.disconnect() // never leave a duplicate connection
do { try await r.connect(url: url, token: token) }
catch { self?.notifyListeners("screenShareState", data: ["sharing": false, "error": String(describing: error)]) }
}
}
// MARK: - #5 Live transcript (native SFSpeechRecognizer) // MARK: - #5 Live transcript (native SFSpeechRecognizer)
// The current published local mic track the source we tap for transcription. nil until the user has // The current published local mic track the source we tap for transcription. nil until the user has
+38 -1
View File
@@ -3153,6 +3153,7 @@ function bzB64(u8){ let s=''; const CH=0x8000; for(let i=0;i<u8.length;i+=CH) s+
const BZ_LIB_KEY='bzc.lib.v1'; const BZ_LIB_KEY='bzc.lib.v1';
const BZ_DIR='DOCUMENTS'; const BZ_DIR='DOCUMENTS';
function bzLibOn(){ const C=window.Capacitor; if(!C) return false; const p=C.getPlatform?C.getPlatform():''; return (p==='ios'||p==='android') && !!(C.Plugins&&C.Plugins.Filesystem); } function bzLibOn(){ const C=window.Capacitor; if(!C) return false; const p=C.getPlatform?C.getPlatform():''; return (p==='ios'||p==='android') && !!(C.Plugins&&C.Plugins.Filesystem); }
function bzIsIOS(){ try{ const C=window.Capacitor; return !!(C && C.getPlatform && C.getPlatform()==='ios'); }catch(_){ return false; } }
function bzLibAll(){ try{ return JSON.parse(localStorage.getItem(BZ_LIB_KEY)||'{}')||{}; }catch(_){ return {}; } } function bzLibAll(){ try{ return JSON.parse(localStorage.getItem(BZ_LIB_KEY)||'{}')||{}; }catch(_){ return {}; } }
function bzLibWrite(m){ try{ localStorage.setItem(BZ_LIB_KEY, JSON.stringify(m)); }catch(_){} } function bzLibWrite(m){ try{ localStorage.setItem(BZ_LIB_KEY, JSON.stringify(m)); }catch(_){} }
function bzLibGet(id){ return bzLibAll()[id]||null; } function bzLibGet(id){ return bzLibAll()[id]||null; }
@@ -4301,7 +4302,15 @@ async function setupNativeCall(){
// Muted from the CallKit system UI → reflect it in the meeting window's mic button. // Muted from the CallKit system UI → reflect it in the meeting window's mic button.
NC.addListener('setMuted', (e)=>{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} }); NC.addListener('setMuted', (e)=>{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} });
// Screen share (ReplayKit) started/stopped from the system — reflect it and tell peers so their stage shows it. // Screen share (ReplayKit) started/stopped from the system — reflect it and tell peers so their stage shows it.
NC.addListener('screenShareState', (e)=>{ if(meetState!=='call'||!meetNative) return; const on=!!(e&&e.sharing); meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); meetSend({type:'meeting-screen', on}); }catch(_){} }); NC.addListener('screenShareState', (e)=>{
if(meetState!=='call') return;
const on=!!(e&&e.sharing);
if(meetNative){ meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); meetSend({type:'meeting-screen', on}); }catch(_){} return; } // native CALL: peers learn via the mesh signal
// iOS SFU meeting: peers receive the screen as a LiveKit track (no meetSend). On stop from the system
// broadcast bar, tear down the dedicated <peerId>-screen connection.
meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); }catch(_){}
if(!on){ try{ const NC2=nativeCallPlugin(); if(NC2&&NC2.stopMeetingScreenShare) NC2.stopMeetingScreenShare(); }catch(_){} }
});
// Ended on the CallKit system screen / plugin ended on remote hang-up. Leave the meeting window too (guarded // Ended on the CallKit system screen / plugin ended on remote hang-up. Leave the meeting window too (guarded
// so our own in-app hang-up, which already called the plugin, doesn't loop). Then tell the server. // so our own in-app hang-up, which already called the plugin, doesn't loop). Then tell the server.
NC.addListener('endCall', (d)=>{ try{ NC.addListener('endCall', (d)=>{ try{
@@ -5019,6 +5028,16 @@ function sfuRebuild(pid){
addTile(pid, st, meetNames.get(pid)||'Guest', false); setTileScreen(pid, !!p.screen); meetWatchStream(pid, st); addTile(pid, st, meetNames.get(pid)||'Guest', false); setTileScreen(pid, !!p.screen); meetWatchStream(pid, st);
} }
function sfuAttach(pub, track, participant, _try){ function sfuAttach(pub, track, participant, _try){
// iOS screen-share: WKWebView has no getDisplayMedia, so the native ReplayKit publisher joins the room as a
// SEPARATE '<peerId>-screen' participant. Map its screen track onto the base peer's tile; drop anything else.
if(typeof participant.identity==='string' && participant.identity.endsWith('-screen')){
if(pub.source!==SFU.lib.Track.Source.ScreenShare) return; // a screen publisher carries only a screen track
const base=participant.identity.slice(0,-7);
if(base===meetMyId){ meetScreen=true; try{ updateScreenBtn(); setLocalSharing(true); }catch(_){} return; } // my own screen → don't mirror it back to me
const bpid = meetPeers.has(base) ? base : (peerIdForUid(base) || ('lk:'+base));
const p=SFU.peers.get(bpid)||{}; SFU.peers.set(bpid,p); p.screen=track.mediaStreamTrack; meetSharers.add(bpid);
sfuRebuild(bpid); updateShareMode(); return;
}
// #12: LiveKit identity is now the mesh peerId (unique per connection), so a track maps straight to its // #12: LiveKit identity is now the mesh peerId (unique per connection), so a track maps straight to its
// tile. peerIdForUid is kept as a fallback for any participant still on the old identity=userId scheme // tile. peerIdForUid is kept as a fallback for any participant still on the old identity=userId scheme
// (e.g. a native device on an older plugin build, before it reconnects with its peerId token). // (e.g. a native device on an older plugin build, before it reconnects with its peerId token).
@@ -5044,6 +5063,12 @@ function sfuAttach(pub, track, participant, _try){
sfuRebuild(pid); updateShareMode(); sfuRebuild(pid); updateShareMode();
} }
function sfuDetach(pub, track, participant){ function sfuDetach(pub, track, participant){
if(typeof participant.identity==='string' && participant.identity.endsWith('-screen')){ // native screen publisher left / stopped
const base=participant.identity.slice(0,-7);
if(base===meetMyId){ meetScreen=false; try{ updateScreenBtn(); setLocalSharing(false); }catch(_){} return; }
const bpid = meetPeers.has(base) ? base : (peerIdForUid(base) || ('lk:'+base));
const p2=SFU.peers.get(bpid); if(p2) p2.screen=null; meetSharers.delete(bpid); setTileScreen(bpid,false); sfuRebuild(bpid); updateShareMode(); return;
}
let pid = meetPeers.has(participant.identity) ? participant.identity : (peerIdForUid(participant.identity) || ('lk:'+participant.identity)); const p=SFU.peers.get(pid); if(!p) return; const mt=track.mediaStreamTrack; let pid = meetPeers.has(participant.identity) ? participant.identity : (peerIdForUid(participant.identity) || ('lk:'+participant.identity)); const p=SFU.peers.get(pid); if(!p) return; const mt=track.mediaStreamTrack;
if(p.audio===mt) p.audio=null; else if(p.screen===mt){ p.screen=null; meetSharers.delete(pid); setTileScreen(pid,false); } else if(p.cam===mt) p.cam=null; if(p.audio===mt) p.audio=null; else if(p.screen===mt){ p.screen=null; meetSharers.delete(pid); setTileScreen(pid,false); } else if(p.cam===mt) p.cam=null;
sfuRebuild(pid); updateShareMode(); sfuRebuild(pid); updateShareMode();
@@ -5768,6 +5793,18 @@ async function toggleScreen(){
try{ if(meetScreen) await NC.stopScreenShare(); else await NC.startScreenShare(); }catch(_){} try{ if(meetScreen) await NC.stopScreenShare(); else await NC.startScreenShare(); }catch(_){}
return; // meetScreen + the button flip happen on the authoritative screenShareState event return; // meetScreen + the button flip happen on the authoritative screenShareState event
} }
// iOS SFU meeting (scheduled / code-joined): WKWebView has no getDisplayMedia, so publish the screen
// NATIVELY (ReplayKit) into the SAME LiveKit room via a dedicated <peerId>-screen connection.
if(SFU.on && !meetNative && bzIsIOS()){
const NC=nativeCallPlugin();
if(NC && typeof NC.startMeetingScreenShare==='function'){
if(meetScreen){ try{ await NC.stopMeetingScreenShare(); }catch(_){} return; }
let tk={}; try{ tk=await postJSON('/api/meetings/token',{ room:meetRoom, peerId:meetMyId, screen:true }); }catch(_){ toast('Could not start screen share'); return; }
if(!tk||!tk.token){ toast('Could not start screen share'); return; }
try{ await NC.startMeetingScreenShare({ url:(SFU.url||tk.url), token:tk.token }); }catch(_){ toast('Could not start screen share'); }
return; // the screenShareState event flips the button + state
}
}
if(meetScreen){ stopScreen(); return; } if(meetScreen){ stopScreen(); return; }
if(!meetMultiShare && meetSharers.size>0){ toast('Someone is already sharing their screen'); return; } if(!meetMultiShare && meetSharers.size>0){ toast('Someone is already sharing their screen'); return; }
if(SFU.on){ if(SFU.on){
+7 -1
View File
@@ -1055,7 +1055,13 @@ route('POST', '/api/meetings/token', async (req, res) => {
try { const peers = require('./presence').meetingRooms.get(rm); const pr = peers && peers.get(pid); if (pr && pr.uid && pr.uid !== u.id) ok = false; } catch (_) {} try { const peers = require('./presence').meetingRooms.get(rm); const pr = peers && peers.get(pid); if (pr && pr.uid && pr.uid !== u.id) ok = false; } catch (_) {}
if (ok) identity = pid; if (ok) identity = pid;
} }
const metadata = JSON.stringify({ avatarUrl: u.avatar_url || '' }); // Screen-share publisher (iOS): the WebView already holds the meeting connection under `identity`, and
// LiveKit allows one connection per identity — so the native ReplayKit publisher joins the SAME room under
// a DISTINCT `<identity>-screen` id. The client maps that suffix back onto the sharer's tile. WKWebView has
// no getDisplayMedia, so this native second connection is the only way to screen-share in an SFU meeting.
const screen = body.screen === true || body.screen === 1 || body.screen === '1';
if (screen) identity = identity + '-screen';
const metadata = JSON.stringify({ avatarUrl: u.avatar_url || '', screen });
const token = livekitToken(identity, u.name || u.email, rm, metadata); const token = livekitToken(identity, u.name || u.email, rm, metadata);
json(res, 200, { token, url: LIVEKIT_URL, identity, name: u.name || u.email }); json(res, 200, { token, url: LIVEKIT_URL, identity, name: u.name || u.email });
}); });