From 467a8b9c6efae462ee19b52dc28c8361b436b9b8 Mon Sep 17 00:00:00 2001 From: sravan Date: Sat, 22 Aug 2026 01:01:42 +0530 Subject: [PATCH] iOS meeting screen share: publish ReplayKit broadcast into the SFU room Gap: on iOS, screen share only worked in native CallKit calls. In a scheduled / code-joined SFU meeting the webview called setScreenShareEnabled() -> getDisplayMedia(), which WKWebView does not implement, so it failed silently. Production meetings are real LiveKit rooms, so we can publish natively instead. Fix (native ReplayKit -> same LiveKit room as a dedicated screen participant): - server: /api/meetings/token accepts screen:true -> mints a distinct -screen identity (LiveKit allows one connection per identity), so the native publisher doesn't collide with the webview's own connection. - native plugin: startMeetingScreenShare/stopMeetingScreenShare + connectScreenRoom (a screen-only LiveKit connection: mic off, no camera, no callConnected) that reuses the existing broadcast-extension publishing path. - webview: toggleScreen routes to the native method on iOS SFU meetings; the screenShareState listener now drives the SFU case too (and tears the screen connection down on the system "Stop Broadcast"); sfuAttach/sfuDetach map the '-screen' participant's screen track onto the sharer's tile and suppress the phantom person tile + the sharer's own self-view. Web/server deploy now; the native method needs the next Codemagic build to test on device. Verified: db-smoke 22/22; Swift braces balanced. Co-Authored-By: Claude Opus 4.8 --- .../NativeCallPlugin/NativeCallPlugin.swift | 37 ++++++++++++++++++ server/public/home.html | 39 ++++++++++++++++++- server/routes.js | 8 +++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index a7e821a..840a46f 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -34,6 +34,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "startScreenShare", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "stopScreenShare", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "startMeetingScreenShare", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "stopMeetingScreenShare", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), @@ -391,6 +393,41 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } + // Screen share into a SCHEDULED / code-joined SFU meeting. The WebView already holds the meeting + // connection (identity = peerId) and WKWebView has no getDisplayMedia — so we open a SECOND, screen-ONLY + // native LiveKit connection under a distinct `-screen` identity and publish the ReplayKit + // broadcast into the SAME room. The WebView maps that identity back onto the sharer's tile. + @objc func startMeetingScreenShare(_ call: CAPPluginCall) { + let url = call.getString("url") ?? "" + let token = call.getString("token") ?? "" + guard !url.isEmpty, !token.isEmpty else { call.reject("url/token required"); return } + connectScreenRoom(url: url, token: token) + DispatchQueue.main.async { BroadcastManager.shared.requestActivation() } // system broadcast picker + call.resolve() + } + + @objc func stopMeetingScreenShare(_ call: CAPPluginCall) { + BroadcastManager.shared.requestStop() + let r = room; room = nil + Task { await r?.disconnect() } // drop the screen-only connection (no call/tile/transcriber side effects) + call.resolve() + } + + // A screen-ONLY LiveKit connection (mic off, no camera) used purely to publish the ReplayKit broadcast + // into an SFU meeting. Unlike connectRoom() it enables no mic and fires no callConnected — this is not a call. + private func connectScreenRoom(url: String, token: String) { + guard !url.isEmpty, !token.isEmpty else { return } + let old = room + let opts = RoomOptions(defaultScreenShareCaptureOptions: ScreenShareCaptureOptions(useBroadcastExtension: true)) + let r = Room(roomOptions: opts) + room = r + Task { [weak self] in + await old?.disconnect() // never leave a duplicate connection + do { try await r.connect(url: url, token: token) } + catch { self?.notifyListeners("screenShareState", data: ["sharing": false, "error": String(describing: error)]) } + } + } + // MARK: - #5 Live transcript (native SFSpeechRecognizer) // The current published local mic track — the source we tap for transcription. nil until the user has diff --git a/server/public/home.html b/server/public/home.html index bd9dc6e..bd00d87 100644 --- a/server/public/home.html +++ b/server/public/home.html @@ -3153,6 +3153,7 @@ function bzB64(u8){ let s=''; const CH=0x8000; for(let i=0;i{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} }); // Screen share (ReplayKit) started/stopped from the system — reflect it and tell peers so their stage shows it. - NC.addListener('screenShareState', (e)=>{ if(meetState!=='call'||!meetNative) return; const on=!!(e&&e.sharing); meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); meetSend({type:'meeting-screen', on}); }catch(_){} }); + NC.addListener('screenShareState', (e)=>{ + if(meetState!=='call') return; + const on=!!(e&&e.sharing); + if(meetNative){ meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); meetSend({type:'meeting-screen', on}); }catch(_){} return; } // native CALL: peers learn via the mesh signal + // iOS SFU meeting: peers receive the screen as a LiveKit track (no meetSend). On stop from the system + // broadcast bar, tear down the dedicated -screen connection. + meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); }catch(_){} + if(!on){ try{ const NC2=nativeCallPlugin(); if(NC2&&NC2.stopMeetingScreenShare) NC2.stopMeetingScreenShare(); }catch(_){} } + }); // Ended on the CallKit system screen / plugin ended on remote hang-up. Leave the meeting window too (guarded // so our own in-app hang-up, which already called the plugin, doesn't loop). Then tell the server. NC.addListener('endCall', (d)=>{ try{ @@ -5019,6 +5028,16 @@ function sfuRebuild(pid){ addTile(pid, st, meetNames.get(pid)||'Guest', false); setTileScreen(pid, !!p.screen); meetWatchStream(pid, st); } function sfuAttach(pub, track, participant, _try){ + // iOS screen-share: WKWebView has no getDisplayMedia, so the native ReplayKit publisher joins the room as a + // SEPARATE '-screen' participant. Map its screen track onto the base peer's tile; drop anything else. + if(typeof participant.identity==='string' && participant.identity.endsWith('-screen')){ + if(pub.source!==SFU.lib.Track.Source.ScreenShare) return; // a screen publisher carries only a screen track + const base=participant.identity.slice(0,-7); + if(base===meetMyId){ meetScreen=true; try{ updateScreenBtn(); setLocalSharing(true); }catch(_){} return; } // my own screen → don't mirror it back to me + const bpid = meetPeers.has(base) ? base : (peerIdForUid(base) || ('lk:'+base)); + const p=SFU.peers.get(bpid)||{}; SFU.peers.set(bpid,p); p.screen=track.mediaStreamTrack; meetSharers.add(bpid); + sfuRebuild(bpid); updateShareMode(); return; + } // #12: LiveKit identity is now the mesh peerId (unique per connection), so a track maps straight to its // tile. peerIdForUid is kept as a fallback for any participant still on the old identity=userId scheme // (e.g. a native device on an older plugin build, before it reconnects with its peerId token). @@ -5044,6 +5063,12 @@ function sfuAttach(pub, track, participant, _try){ sfuRebuild(pid); updateShareMode(); } function sfuDetach(pub, track, participant){ + if(typeof participant.identity==='string' && participant.identity.endsWith('-screen')){ // native screen publisher left / stopped + const base=participant.identity.slice(0,-7); + if(base===meetMyId){ meetScreen=false; try{ updateScreenBtn(); setLocalSharing(false); }catch(_){} return; } + const bpid = meetPeers.has(base) ? base : (peerIdForUid(base) || ('lk:'+base)); + const p2=SFU.peers.get(bpid); if(p2) p2.screen=null; meetSharers.delete(bpid); setTileScreen(bpid,false); sfuRebuild(bpid); updateShareMode(); return; + } let pid = meetPeers.has(participant.identity) ? participant.identity : (peerIdForUid(participant.identity) || ('lk:'+participant.identity)); const p=SFU.peers.get(pid); if(!p) return; const mt=track.mediaStreamTrack; if(p.audio===mt) p.audio=null; else if(p.screen===mt){ p.screen=null; meetSharers.delete(pid); setTileScreen(pid,false); } else if(p.cam===mt) p.cam=null; sfuRebuild(pid); updateShareMode(); @@ -5768,6 +5793,18 @@ async function toggleScreen(){ try{ if(meetScreen) await NC.stopScreenShare(); else await NC.startScreenShare(); }catch(_){} return; // meetScreen + the button flip happen on the authoritative screenShareState event } + // iOS SFU meeting (scheduled / code-joined): WKWebView has no getDisplayMedia, so publish the screen + // NATIVELY (ReplayKit) into the SAME LiveKit room via a dedicated -screen connection. + if(SFU.on && !meetNative && bzIsIOS()){ + const NC=nativeCallPlugin(); + if(NC && typeof NC.startMeetingScreenShare==='function'){ + if(meetScreen){ try{ await NC.stopMeetingScreenShare(); }catch(_){} return; } + let tk={}; try{ tk=await postJSON('/api/meetings/token',{ room:meetRoom, peerId:meetMyId, screen:true }); }catch(_){ toast('Could not start screen share'); return; } + if(!tk||!tk.token){ toast('Could not start screen share'); return; } + try{ await NC.startMeetingScreenShare({ url:(SFU.url||tk.url), token:tk.token }); }catch(_){ toast('Could not start screen share'); } + return; // the screenShareState event flips the button + state + } + } if(meetScreen){ stopScreen(); return; } if(!meetMultiShare && meetSharers.size>0){ toast('Someone is already sharing their screen'); return; } if(SFU.on){ diff --git a/server/routes.js b/server/routes.js index 00ed15c..eee0132 100644 --- a/server/routes.js +++ b/server/routes.js @@ -1055,7 +1055,13 @@ route('POST', '/api/meetings/token', async (req, res) => { try { const peers = require('./presence').meetingRooms.get(rm); const pr = peers && peers.get(pid); if (pr && pr.uid && pr.uid !== u.id) ok = false; } catch (_) {} if (ok) identity = pid; } - const metadata = JSON.stringify({ avatarUrl: u.avatar_url || '' }); + // Screen-share publisher (iOS): the WebView already holds the meeting connection under `identity`, and + // LiveKit allows one connection per identity — so the native ReplayKit publisher joins the SAME room under + // a DISTINCT `-screen` id. The client maps that suffix back onto the sharer's tile. WKWebView has + // no getDisplayMedia, so this native second connection is the only way to screen-share in an SFU meeting. + const screen = body.screen === true || body.screen === 1 || body.screen === '1'; + if (screen) identity = identity + '-screen'; + const metadata = JSON.stringify({ avatarUrl: u.avatar_url || '', screen }); const token = livekitToken(identity, u.name || u.email, rm, metadata); json(res, 200, { token, url: LIVEKIT_URL, identity, name: u.name || u.email }); });