Native video Increment 2b: remote tiles + self-view on tile; speaker fix

Video (2b): render every participant's camera natively, positioned to match the web meeting tiles. The web has no LiveKit connection on a native call, so the plugin draws native VideoViews over the WebView. New NativeCall.syncVideoTiles({tiles:[{uid,local,x,y,w,h}]}) — the web polls each tile's getBoundingClientRect + user id (400ms + on camera toggle) and the plugin places a VideoView (subview of the WKWebView, so CSS-px rects == points) for whoever has a live, unmuted camera track; camera-off keeps the web avatar. Replaces the 2a fixed-corner self-view: your own camera now renders on the __local tile. Remote video correlated by LiveKit identity == meetPeerUids user id. APIs verified vs client-sdk-swift 2.15.3 source: Room.remoteParticipants[Participant.Identity(from:)], Participant.videoTracks, TrackPublication.source/.track/.isMuted, Track.Source.camera. Audio: fix "sound starts on the earpiece until I tap something" — the LiveKit audio engine starting after CallKit activates the session flips the route to the receiver. Added an AVAudioSession routeChange observer that re-asserts the loudspeaker (via preferSpeaker) whenever we land on the built-in receiver mid-call (headset/BT still win). Web change is safe on the current (2a) build: syncVideoTiles is absent so the poll no-ops. Needs a Codemagic build to take effect. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-06 11:59:52 +05:30
parent 149e32b8d8
commit 4415225407
2 changed files with 120 additions and 40 deletions
@@ -27,6 +27,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise),
+        CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise),
        CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise)
    ]

@@ -40,11 +41,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
    private var endedCalls = Set<UUID>()
    private var room: Room?
    private var activeUUID: UUID?
-    // Native video (Increment 2a): the local camera is published to the room (so everyone else sees this
-    // user) and mirrored into a small self-view (PiP) drawn over the WebView. Rendering the OTHER
-    // participants as native tiles synced to the web meeting grid is Increment 2b. VideoView is a UIKit
-    // view — only ever touched on the main thread (helpers below dispatch there).
-    private var localVideoView: VideoView?
+    // Native video: one native VideoView per visible participant (key "__local" or the remote user id),
+    // drawn over the WebView and positioned to match the web meeting tiles (see syncVideoTiles). VideoView is
+    // a UIKit view — only ever touched on the main thread.
+    private var tileViews: [String: VideoView] = [:]

    override public func load() {
        let config = CXProviderConfiguration(localizedName: "Biz Connect")
@@ -66,6 +66,19 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        // engine OFF; we configure the session and enable the engine ONLY in didActivate.
        AudioManager.shared.audioSession.isAutomaticConfigurationEnabled = false
        try? AudioManager.shared.setEngineAvailability(.none)
+
+        // Re-assert the LOUDSPEAKER whenever iOS routes call audio back to the quiet earpiece. The LiveKit
+        // audio engine starting up right after CallKit activates the session flips the route to the built-in
+        // receiver — that's the "sound is on the earpiece until I tap something" bug (tapping mic/cam re-ran
+        // preferSpeaker and fixed it). Listening for route changes makes that self-healing.
+        NotificationCenter.default.addObserver(self, selector: #selector(audioRouteChanged(_:)),
+                                               name: AVAudioSession.routeChangeNotification, object: nil)
+    }
+
+    @objc private func audioRouteChanged(_ note: Notification) {
+        guard room != nil else { return } // only steer the route during an active native call
+        // Let the engine's own route change settle first, then override if we landed on the earpiece.
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.15) { [weak self] in self?.preferSpeaker() }
    }

    // MARK: - LiveKit media
@@ -97,40 +110,41 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        let r = room
        room = nil
        Task { await r?.disconnect() }
-        hideSelfView()
+        removeAllTileViews()
    }

-    // MARK: - Native video (local self-view)
+    // MARK: - Native video (tile rendering, Increment 2b)
+    //
+    // The WebView owns the meeting UI (grid, roster, controls) but — for a native call — has NO LiveKit
+    // connection, so it can't render any video. The video lives only in the plugin's LiveKit connection.
+    // So we draw native VideoViews on top of the WebView, positioned to match each web tile: the web reports
+    // each tile's on-screen rect + the participant's user id (syncVideoTiles), and we place/size a VideoView
+    // for whichever participants currently have a live camera track. Views are subviews of the WKWebView, so
+    // their frames use the SAME coordinate space as getBoundingClientRect (CSS px == points, both
+    // viewport-relative) and stay aligned as the page scrolls.

-    // Show the local camera as a small rounded self-view pinned to the top-right safe area, above the WebView.
-    // Increment 2b will render the REMOTE participants as native views synced to the web meeting tiles.
-    private func showSelfView(track: VideoTrack?) {
+    // Find the live (unmuted, subscribed) camera track for a user id — nil when the camera is off, so the web
+    // tile's avatar shows through instead.
+    private func cameraTrack(forUid uid: String, isLocal: Bool) -> VideoTrack? {
+        guard let room = room else { return nil }
+        let pubs: [TrackPublication]
+        if isLocal {
+            pubs = room.localParticipant.videoTracks
+        } else {
+            guard let p = room.remoteParticipants[Participant.Identity(from: uid)] else { return nil }
+            pubs = p.videoTracks
+        }
+        guard let pub = pubs.first(where: { $0.source == .camera && !$0.isMuted && $0.track != nil }) else { return nil }
+        return pub.track as? VideoTrack
+    }
+
+    private func tileKey(uid: String, isLocal: Bool) -> String { isLocal ? "__local" : uid }
+
+    private func removeAllTileViews() {
        DispatchQueue.main.async { [weak self] in
-            guard let self = self, let track = track, let host = self.bridge?.viewController?.view else { return }
-            let vv = self.localVideoView ?? VideoView()
-            vv.layoutMode = .fill            // fill the PiP box (crop) instead of letterboxing
-            vv.track = track
-            if vv.superview == nil {
-                vv.backgroundColor = .black
-                vv.clipsToBounds = true
-                vv.layer.cornerRadius = 10
-                vv.translatesAutoresizingMaskIntoConstraints = false
-                host.addSubview(vv)
-                NSLayoutConstraint.activate([
-                    vv.widthAnchor.constraint(equalToConstant: 96),
-                    vv.heightAnchor.constraint(equalToConstant: 132),
-                    vv.topAnchor.constraint(equalTo: host.safeAreaLayoutGuide.topAnchor, constant: 16),
-                    vv.trailingAnchor.constraint(equalTo: host.safeAreaLayoutGuide.trailingAnchor, constant: -12),
-                ])
-            }
-            self.localVideoView = vv
-        }
-    }
-
-    private func hideSelfView() {
-        DispatchQueue.main.async { [weak self] in
-            self?.localVideoView?.removeFromSuperview()
-            self?.localVideoView = nil
+            guard let self = self else { return }
+            for (_, vv) in self.tileViews { vv.removeFromSuperview() }
+            self.tileViews.removeAll()
        }
    }

@@ -212,19 +226,17 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        call.resolve()
    }

-    // Native video (Increment 2a): enable/disable the local camera. Publishing it makes this user's video
-    // appear for everyone else (their web/desktop clients render it via their own SFU subscription); locally
-    // we mirror it into a small self-view. Seeing the OTHER participants' video (native tiles) is Increment 2b.
-    // Front camera only for now.
+    // Enable/disable the local camera. Publishing it makes this user's video appear for everyone else (their
+    // web/desktop clients render it via their own SFU subscription); locally it's drawn on the __local tile by
+    // syncVideoTiles (the web triggers a sync right after this resolves). Front camera only for now.
    @objc func setCamera(_ call: CAPPluginCall) {
        guard let r = room else { call.reject("no active call"); return }
        let on = call.getBool("on") ?? false
-        Task { [weak self] in
+        Task {
            do {
-                let pub = try await r.localParticipant.setCamera(
+                try await r.localParticipant.setCamera(
                    enabled: on,
                    captureOptions: CameraCaptureOptions(position: .front))
-                if on { self?.showSelfView(track: pub?.track as? VideoTrack) } else { self?.hideSelfView() }
                call.resolve(["on": on])
            } catch {
                call.reject("camera failed: \(String(describing: error))")
@@ -232,6 +244,47 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega
        }
    }

+    // Position native video views to match the web meeting tiles. `tiles` = [{uid, local, x, y, w, h}] in
+    // CSS px (== points; getBoundingClientRect coords). We create/move a VideoView for each participant that
+    // has a live camera track, and remove views for tiles that are gone or whose camera is off (so the web
+    // avatar shows). Called on a short poll by the web while a native call is on screen, plus on demand.
+    @objc func syncVideoTiles(_ call: CAPPluginCall) {
+        let tiles = (call.getArray("tiles") as? [[String: Any]]) ?? []
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { call.resolve(); return }
+            guard let host = self.bridge?.webView else { call.resolve(); return }
+            var wanted = Set<String>()
+            for t in tiles {
+                guard let uid = t["uid"] as? String, !uid.isEmpty else { continue }
+                let isLocal = (t["local"] as? Bool) ?? false
+                func num(_ k: String) -> CGFloat { CGFloat((t[k] as? NSNumber)?.doubleValue ?? 0) }
+                let x = num("x"), y = num("y"), w = num("w"), h = num("h")
+                if w < 2 || h < 2 { continue }
+                let key = self.tileKey(uid: uid, isLocal: isLocal)
+                guard let track = self.cameraTrack(forUid: uid, isLocal: isLocal) else { continue } // camera off → leave the web avatar
+                wanted.insert(key)
+                let vv = self.tileViews[key] ?? {
+                    let v = VideoView()
+                    v.layoutMode = .fill
+                    v.backgroundColor = .black
+                    v.clipsToBounds = true
+                    v.layer.cornerRadius = 8
+                    host.addSubview(v)
+                    self.tileViews[key] = v
+                    return v
+                }()
+                if vv.superview !== host { host.addSubview(vv) }
+                if vv.track !== track { vv.track = track }
+                vv.frame = CGRect(x: x, y: y, width: w, height: h)
+            }
+            // Drop views for participants no longer present / camera turned off.
+            for (key, vv) in self.tileViews where !wanted.contains(key) {
+                vv.removeFromSuperview(); self.tileViews.removeValue(forKey: key)
+            }
+            call.resolve()
+        }
+    }
+
    // End the CallKit call (remote hung up / user ended from the web UI). No callUUID -> end all.
    @objc func endCall(_ call: CAPPluginCall) {
        if let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) {
@@ -5361,10 +5361,35 @@ function bzUnlockAudio(){
 document.addEventListener('touchend', function(){ if(meetState==='call') bzUnlockAudio(); }, {passive:true}); // fallback: any tap during a call restarts silent remote audio
 document.addEventListener('click', function(){ if(meetState==='call') bzUnlockAudio(); }, {passive:true});
 let meetNative=false, meetNativeUuid=null, _ncEnding=false; // native call: WebView joins the mesh for UI; the plugin owns media/CallKit
+// ---- Native call video (Increment 2b) ----------------------------------------------------------------
+// On a native call the WebView owns the meeting grid but has NO LiveKit connection, so it can't render any
+// video — the media lives only in the native plugin. So we hand the plugin each tile's on-screen rect + the
+// participant's user id, and it draws a native VideoView over each tile for whoever has a live camera
+// (camera-off participants keep showing the web avatar). Polled so it follows layout / rotation / scroll /
+// join-leave / camera-toggle changes; a no-op on web calls. getBoundingClientRect coords are CSS px, which
+// equal points in the WKWebView, so the plugin can use them directly.
+let _ncTileTimer=null;
+function bzNativeSyncTiles(){
+  if(!meetNative) return;
+  const NC=nativeCallPlugin(); if(!NC||!NC.syncVideoTiles) return;
+  const tiles=[];
+  document.querySelectorAll('#meetGrid .meet-tile').forEach(el=>{
+    const id=el.id ? el.id.replace('meet-tile-','') : ''; if(!id || id==='__waiting') return;
+    const r=el.getBoundingClientRect(); if(r.width<2||r.height<2) return;
+    let uid, local=false;
+    if(id==='__local'){ uid=(ME&&ME.id)||''; local=true; } else { uid=meetPeerUids.get(id)||''; }
+    if(!uid) return;
+    tiles.push({ uid, local, x:r.left, y:r.top, w:r.width, h:r.height });
+  });
+  try{ NC.syncVideoTiles({ tiles }); }catch(_){}
+}
+function bzNativeStartTiles(){ if(_ncTileTimer||!meetNative) return; _ncTileTimer=setInterval(bzNativeSyncTiles, 400); bzNativeSyncTiles(); }
+function bzNativeStopTiles(){ if(_ncTileTimer){ clearInterval(_ncTileTimer); _ncTileTimer=null; } const NC=nativeCallPlugin(); if(NC&&NC.syncVideoTiles){ try{ NC.syncVideoTiles({ tiles:[] }); }catch(_){} } }
 async function enterMeeting(code, audioOnly, opts){
  bzUnlockAudio(); // runs inside the Join tap → unlock playback so remote audio isn't silent until you tap
  if(meetState==='call'){ switchTab('meeting'); return; } // already in a call — ignore double-join
  meetNative=!!(opts&&opts.native); meetNativeUuid=(opts&&opts.uuid)||null; // native → skip our own SFU media; mute/end bridge to the plugin
+  if(meetNative) bzNativeStartTiles(); // native call → drive the plugin's native video overlays from the web tiles
  // Joining this room → clear any lingering incoming-call invite popup for it (and stop its ring).
  // Fixes: joining via the header "Join" button left the Join/Decline popup on screen. Belt-and-braces
  // we clear ALL open invites, since you can only be in one call at a time.
@@ -5473,7 +5498,8 @@ async function toggleCam(){
    const next=!meetCam; const NC=nativeCallPlugin();
    if(!NC||!NC.setCamera){ toast('Update the app to use video on calls'); return; }
    try{ await NC.setCamera({ on:next }); }catch(e){ toast('Could not turn the camera '+(next?'on':'off')); return; }
-    meetCam=next; meetAudioOnly=false; updateCamBtn();
+    meetCam=next; meetAudioOnly=false; updateCamBtn(); bzNativeSyncTiles(); // reflect the new camera state on the tiles now
+    bzNativeStartTiles(); // ensure the overlay poll is running (first camera-on)
    // The web __local tile stays an avatar — the WebView has no camera stream in a native call (the plugin
    // owns media and draws its own self-view). Tell peers our camera state so their SFU tile shows the video.
    meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam});
@@ -5505,6 +5531,7 @@ function leaveMeeting(forced){
  // Native call: also end the CallKit/plugin call. _ncEnding tells the plugin's endCall listener this leave
  // originated here, so it doesn't call leaveMeeting again (the plugin ending the call re-fires endCall).
  if(meetNative && meetNativeUuid){ _ncEnding=true; callkitEnd(meetNativeUuid); }
+  bzNativeStopTiles(); // stop the native video overlay poll + clear any native tiles
  meetNative=false; meetNativeUuid=null;
  stopRingback(); removeWaitingTile(); _dmCallWaiting=null; // #17/#7: any call exit stops the ring + ringing tile
  const isDm=!!(meetReturn && meetReturn.kind==='dm');