From 441522540710d85608743a7fe5f6d4730567075a Mon Sep 17 00:00:00 2001 From: sravan Date: Thu, 6 Aug 2026 11:59:52 +0530 Subject: [PATCH] Native video Increment 2b: remote tiles + self-view on tile; speaker fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Video (2b): render every participant's camera natively, positioned to match the web meeting tiles. The web has no LiveKit connection on a native call, so the plugin draws native VideoViews over the WebView. New NativeCall.syncVideoTiles({tiles:[{uid,local,x,y,w,h}]}) — the web polls each tile's getBoundingClientRect + user id (400ms + on camera toggle) and the plugin places a VideoView (subview of the WKWebView, so CSS-px rects == points) for whoever has a live, unmuted camera track; camera-off keeps the web avatar. Replaces the 2a fixed-corner self-view: your own camera now renders on the __local tile. Remote video correlated by LiveKit identity == meetPeerUids user id. APIs verified vs client-sdk-swift 2.15.3 source: Room.remoteParticipants[Participant.Identity(from:)], Participant.videoTracks, TrackPublication.source/.track/.isMuted, Track.Source.camera. Audio: fix "sound starts on the earpiece until I tap something" — the LiveKit audio engine starting after CallKit activates the session flips the route to the receiver. Added an AVAudioSession routeChange observer that re-asserts the loudspeaker (via preferSpeaker) whenever we land on the built-in receiver mid-call (headset/BT still win). Web change is safe on the current (2a) build: syncVideoTiles is absent so the poll no-ops. Needs a Codemagic build to take effect. Co-Authored-By: Claude Opus 4.8 --- .../NativeCallPlugin/NativeCallPlugin.swift | 131 ++++++++++++------ server/public/home.html | 29 +++- 2 files changed, 120 insertions(+), 40 deletions(-) diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index c5564f5..7944e55 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -27,6 +27,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] @@ -40,11 +41,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega private var endedCalls = Set() private var room: Room? private var activeUUID: UUID? - // Native video (Increment 2a): the local camera is published to the room (so everyone else sees this - // user) and mirrored into a small self-view (PiP) drawn over the WebView. Rendering the OTHER - // participants as native tiles synced to the web meeting grid is Increment 2b. VideoView is a UIKit - // view — only ever touched on the main thread (helpers below dispatch there). - private var localVideoView: VideoView? + // Native video: one native VideoView per visible participant (key "__local" or the remote user id), + // drawn over the WebView and positioned to match the web meeting tiles (see syncVideoTiles). VideoView is + // a UIKit view — only ever touched on the main thread. + private var tileViews: [String: VideoView] = [:] override public func load() { let config = CXProviderConfiguration(localizedName: "Biz Connect") @@ -66,6 +66,19 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega // engine OFF; we configure the session and enable the engine ONLY in didActivate. AudioManager.shared.audioSession.isAutomaticConfigurationEnabled = false try? AudioManager.shared.setEngineAvailability(.none) + + // Re-assert the LOUDSPEAKER whenever iOS routes call audio back to the quiet earpiece. The LiveKit + // audio engine starting up right after CallKit activates the session flips the route to the built-in + // receiver — that's the "sound is on the earpiece until I tap something" bug (tapping mic/cam re-ran + // preferSpeaker and fixed it). Listening for route changes makes that self-healing. + NotificationCenter.default.addObserver(self, selector: #selector(audioRouteChanged(_:)), + name: AVAudioSession.routeChangeNotification, object: nil) + } + + @objc private func audioRouteChanged(_ note: Notification) { + guard room != nil else { return } // only steer the route during an active native call + // Let the engine's own route change settle first, then override if we landed on the earpiece. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.15) { [weak self] in self?.preferSpeaker() } } // MARK: - LiveKit media @@ -97,40 +110,41 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega let r = room room = nil Task { await r?.disconnect() } - hideSelfView() + removeAllTileViews() } - // MARK: - Native video (local self-view) + // MARK: - Native video (tile rendering, Increment 2b) + // + // The WebView owns the meeting UI (grid, roster, controls) but — for a native call — has NO LiveKit + // connection, so it can't render any video. The video lives only in the plugin's LiveKit connection. + // So we draw native VideoViews on top of the WebView, positioned to match each web tile: the web reports + // each tile's on-screen rect + the participant's user id (syncVideoTiles), and we place/size a VideoView + // for whichever participants currently have a live camera track. Views are subviews of the WKWebView, so + // their frames use the SAME coordinate space as getBoundingClientRect (CSS px == points, both + // viewport-relative) and stay aligned as the page scrolls. - // Show the local camera as a small rounded self-view pinned to the top-right safe area, above the WebView. - // Increment 2b will render the REMOTE participants as native views synced to the web meeting tiles. - private func showSelfView(track: VideoTrack?) { - DispatchQueue.main.async { [weak self] in - guard let self = self, let track = track, let host = self.bridge?.viewController?.view else { return } - let vv = self.localVideoView ?? VideoView() - vv.layoutMode = .fill // fill the PiP box (crop) instead of letterboxing - vv.track = track - if vv.superview == nil { - vv.backgroundColor = .black - vv.clipsToBounds = true - vv.layer.cornerRadius = 10 - vv.translatesAutoresizingMaskIntoConstraints = false - host.addSubview(vv) - NSLayoutConstraint.activate([ - vv.widthAnchor.constraint(equalToConstant: 96), - vv.heightAnchor.constraint(equalToConstant: 132), - vv.topAnchor.constraint(equalTo: host.safeAreaLayoutGuide.topAnchor, constant: 16), - vv.trailingAnchor.constraint(equalTo: host.safeAreaLayoutGuide.trailingAnchor, constant: -12), - ]) - } - self.localVideoView = vv + // Find the live (unmuted, subscribed) camera track for a user id — nil when the camera is off, so the web + // tile's avatar shows through instead. + private func cameraTrack(forUid uid: String, isLocal: Bool) -> VideoTrack? { + guard let room = room else { return nil } + let pubs: [TrackPublication] + if isLocal { + pubs = room.localParticipant.videoTracks + } else { + guard let p = room.remoteParticipants[Participant.Identity(from: uid)] else { return nil } + pubs = p.videoTracks } + guard let pub = pubs.first(where: { $0.source == .camera && !$0.isMuted && $0.track != nil }) else { return nil } + return pub.track as? VideoTrack } - private func hideSelfView() { + private func tileKey(uid: String, isLocal: Bool) -> String { isLocal ? "__local" : uid } + + private func removeAllTileViews() { DispatchQueue.main.async { [weak self] in - self?.localVideoView?.removeFromSuperview() - self?.localVideoView = nil + guard let self = self else { return } + for (_, vv) in self.tileViews { vv.removeFromSuperview() } + self.tileViews.removeAll() } } @@ -212,19 +226,17 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } - // Native video (Increment 2a): enable/disable the local camera. Publishing it makes this user's video - // appear for everyone else (their web/desktop clients render it via their own SFU subscription); locally - // we mirror it into a small self-view. Seeing the OTHER participants' video (native tiles) is Increment 2b. - // Front camera only for now. + // Enable/disable the local camera. Publishing it makes this user's video appear for everyone else (their + // web/desktop clients render it via their own SFU subscription); locally it's drawn on the __local tile by + // syncVideoTiles (the web triggers a sync right after this resolves). Front camera only for now. @objc func setCamera(_ call: CAPPluginCall) { guard let r = room else { call.reject("no active call"); return } let on = call.getBool("on") ?? false - Task { [weak self] in + Task { do { - let pub = try await r.localParticipant.setCamera( + try await r.localParticipant.setCamera( enabled: on, captureOptions: CameraCaptureOptions(position: .front)) - if on { self?.showSelfView(track: pub?.track as? VideoTrack) } else { self?.hideSelfView() } call.resolve(["on": on]) } catch { call.reject("camera failed: \(String(describing: error))") @@ -232,6 +244,47 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega } } + // Position native video views to match the web meeting tiles. `tiles` = [{uid, local, x, y, w, h}] in + // CSS px (== points; getBoundingClientRect coords). We create/move a VideoView for each participant that + // has a live camera track, and remove views for tiles that are gone or whose camera is off (so the web + // avatar shows). Called on a short poll by the web while a native call is on screen, plus on demand. + @objc func syncVideoTiles(_ call: CAPPluginCall) { + let tiles = (call.getArray("tiles") as? [[String: Any]]) ?? [] + DispatchQueue.main.async { [weak self] in + guard let self = self else { call.resolve(); return } + guard let host = self.bridge?.webView else { call.resolve(); return } + var wanted = Set() + for t in tiles { + guard let uid = t["uid"] as? String, !uid.isEmpty else { continue } + let isLocal = (t["local"] as? Bool) ?? false + func num(_ k: String) -> CGFloat { CGFloat((t[k] as? NSNumber)?.doubleValue ?? 0) } + let x = num("x"), y = num("y"), w = num("w"), h = num("h") + if w < 2 || h < 2 { continue } + let key = self.tileKey(uid: uid, isLocal: isLocal) + guard let track = self.cameraTrack(forUid: uid, isLocal: isLocal) else { continue } // camera off → leave the web avatar + wanted.insert(key) + let vv = self.tileViews[key] ?? { + let v = VideoView() + v.layoutMode = .fill + v.backgroundColor = .black + v.clipsToBounds = true + v.layer.cornerRadius = 8 + host.addSubview(v) + self.tileViews[key] = v + return v + }() + if vv.superview !== host { host.addSubview(vv) } + if vv.track !== track { vv.track = track } + vv.frame = CGRect(x: x, y: y, width: w, height: h) + } + // Drop views for participants no longer present / camera turned off. + for (key, vv) in self.tileViews where !wanted.contains(key) { + vv.removeFromSuperview(); self.tileViews.removeValue(forKey: key) + } + call.resolve() + } + } + // End the CallKit call (remote hung up / user ended from the web UI). No callUUID -> end all. @objc func endCall(_ call: CAPPluginCall) { if let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) { diff --git a/server/public/home.html b/server/public/home.html index d20215a..dd8d462 100644 --- a/server/public/home.html +++ b/server/public/home.html @@ -5361,10 +5361,35 @@ function bzUnlockAudio(){ document.addEventListener('touchend', function(){ if(meetState==='call') bzUnlockAudio(); }, {passive:true}); // fallback: any tap during a call restarts silent remote audio document.addEventListener('click', function(){ if(meetState==='call') bzUnlockAudio(); }, {passive:true}); let meetNative=false, meetNativeUuid=null, _ncEnding=false; // native call: WebView joins the mesh for UI; the plugin owns media/CallKit +// ---- Native call video (Increment 2b) ---------------------------------------------------------------- +// On a native call the WebView owns the meeting grid but has NO LiveKit connection, so it can't render any +// video — the media lives only in the native plugin. So we hand the plugin each tile's on-screen rect + the +// participant's user id, and it draws a native VideoView over each tile for whoever has a live camera +// (camera-off participants keep showing the web avatar). Polled so it follows layout / rotation / scroll / +// join-leave / camera-toggle changes; a no-op on web calls. getBoundingClientRect coords are CSS px, which +// equal points in the WKWebView, so the plugin can use them directly. +let _ncTileTimer=null; +function bzNativeSyncTiles(){ + if(!meetNative) return; + const NC=nativeCallPlugin(); if(!NC||!NC.syncVideoTiles) return; + const tiles=[]; + document.querySelectorAll('#meetGrid .meet-tile').forEach(el=>{ + const id=el.id ? el.id.replace('meet-tile-','') : ''; if(!id || id==='__waiting') return; + const r=el.getBoundingClientRect(); if(r.width<2||r.height<2) return; + let uid, local=false; + if(id==='__local'){ uid=(ME&&ME.id)||''; local=true; } else { uid=meetPeerUids.get(id)||''; } + if(!uid) return; + tiles.push({ uid, local, x:r.left, y:r.top, w:r.width, h:r.height }); + }); + try{ NC.syncVideoTiles({ tiles }); }catch(_){} +} +function bzNativeStartTiles(){ if(_ncTileTimer||!meetNative) return; _ncTileTimer=setInterval(bzNativeSyncTiles, 400); bzNativeSyncTiles(); } +function bzNativeStopTiles(){ if(_ncTileTimer){ clearInterval(_ncTileTimer); _ncTileTimer=null; } const NC=nativeCallPlugin(); if(NC&&NC.syncVideoTiles){ try{ NC.syncVideoTiles({ tiles:[] }); }catch(_){} } } async function enterMeeting(code, audioOnly, opts){ bzUnlockAudio(); // runs inside the Join tap → unlock playback so remote audio isn't silent until you tap if(meetState==='call'){ switchTab('meeting'); return; } // already in a call — ignore double-join meetNative=!!(opts&&opts.native); meetNativeUuid=(opts&&opts.uuid)||null; // native → skip our own SFU media; mute/end bridge to the plugin + if(meetNative) bzNativeStartTiles(); // native call → drive the plugin's native video overlays from the web tiles // Joining this room → clear any lingering incoming-call invite popup for it (and stop its ring). // Fixes: joining via the header "Join" button left the Join/Decline popup on screen. Belt-and-braces // we clear ALL open invites, since you can only be in one call at a time. @@ -5473,7 +5498,8 @@ async function toggleCam(){ const next=!meetCam; const NC=nativeCallPlugin(); if(!NC||!NC.setCamera){ toast('Update the app to use video on calls'); return; } try{ await NC.setCamera({ on:next }); }catch(e){ toast('Could not turn the camera '+(next?'on':'off')); return; } - meetCam=next; meetAudioOnly=false; updateCamBtn(); + meetCam=next; meetAudioOnly=false; updateCamBtn(); bzNativeSyncTiles(); // reflect the new camera state on the tiles now + bzNativeStartTiles(); // ensure the overlay poll is running (first camera-on) // The web __local tile stays an avatar — the WebView has no camera stream in a native call (the plugin // owns media and draws its own self-view). Tell peers our camera state so their SFU tile shows the video. meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); @@ -5505,6 +5531,7 @@ function leaveMeeting(forced){ // Native call: also end the CallKit/plugin call. _ncEnding tells the plugin's endCall listener this leave // originated here, so it doesn't call leaveMeeting again (the plugin ending the call re-fires endCall). if(meetNative && meetNativeUuid){ _ncEnding=true; callkitEnd(meetNativeUuid); } + bzNativeStopTiles(); // stop the native video overlay poll + clear any native tiles meetNative=false; meetNativeUuid=null; stopRingback(); removeWaitingTile(); _dmCallWaiting=null; // #17/#7: any call exit stops the ring + ringing tile const isDm=!!(meetReturn && meetReturn.kind==='dm');