import Foundation import UIKit import Capacitor import PushKit import CallKit import AVFoundation import LiveKit // SPM product name is 'LiveKit' (Package.swift). (The old CocoaPods module was 'LiveKitClient'.) // Native calling for Biz Connect (iOS). Registered by `cap sync` as window.Capacitor.Plugins.NativeCall. // // ARCHITECTURE (native media): // * PushKit: registers for VoIP pushes, reports the VoIP token to JS (-> /api/v1/devices 'ios-voip'). // * CallKit: incoming VoIP push -> full-screen system ring (works when force-killed). The CallKit call // stays ACTIVE for the whole call (that's what foregrounds/unlocks the app on answer and keeps the call // alive in the background). // * LiveKit: the call MEDIA runs NATIVELY via the LiveKit iOS SDK — so the mic works with an active CallKit // call (WebKit's WebRTC could not) and audio survives backgrounding. The VoIP payload carries the // LiveKit url+token so we can connect immediately on answer, even from a killed state; outgoing calls get // the token from the WebView via reportOutgoingCall(). The WebView is UI only for native calls (it must // NOT also join the room — LiveKit allows one connection per identity). @objc(NativeCallPlugin) public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelegate, CXProviderDelegate { public let identifier = "NativeCallPlugin" public let jsName = "NativeCall" public let pluginMethods: [CAPPluginMethod] = [ CAPPluginMethod(name: "getToken", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "reportOutgoingCall", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] private var pushRegistry: PKPushRegistry? private var provider: CXProvider? private let callController = CXCallController() private var voipToken: String = "" // callUUID -> the call's data (room, kind, callerId, livekitUrl, livekitToken, …). private var calls: [UUID: [String: Any]] = [:] // Calls we've already ended locally — so a LATE cancel push doesn't re-report (which briefly re-rang). private var endedCalls = Set() private var room: Room? private var activeUUID: UUID? // Native video: one native VideoView per visible participant (key "__local" or the remote user id), // drawn over the WebView and positioned to match the web meeting tiles (see syncVideoTiles). VideoView is // a UIKit view — only ever touched on the main thread. private var tileViews: [String: VideoView] = [:] override public func load() { let config = CXProviderConfiguration(localizedName: "Biz Connect") config.supportsVideo = true config.maximumCallGroups = 1 config.maximumCallsPerCallGroup = 1 config.supportedHandleTypes = [.generic] let p = CXProvider(configuration: config) p.setDelegate(self, queue: nil) provider = p let registry = PKPushRegistry(queue: .main) registry.delegate = self registry.desiredPushTypes = [.voIP] pushRegistry = registry // CallKit audio coordination (LiveKit 2.15+, pinned to the git tag in ios-patch.sh). LiveKit's automatic // AVAudioSession config RACES CallKit's own activation → intermittent dead mic / no audio + the output // route only settling once audio flows ("speaker turns on late"). Fix: turn auto-config OFF and keep the // engine OFF; we configure the session and enable the engine ONLY in didActivate. AudioManager.shared.audioSession.isAutomaticConfigurationEnabled = false try? AudioManager.shared.setEngineAvailability(.none) // Re-assert the LOUDSPEAKER whenever iOS routes call audio back to the quiet earpiece. The LiveKit // audio engine starting up right after CallKit activates the session flips the route to the built-in // receiver — that's the "sound is on the earpiece until I tap something" bug (tapping mic/cam re-ran // preferSpeaker and fixed it). Listening for route changes makes that self-healing. NotificationCenter.default.addObserver(self, selector: #selector(audioRouteChanged(_:)), name: AVAudioSession.routeChangeNotification, object: nil) } @objc private func audioRouteChanged(_ note: Notification) { guard room != nil else { return } // only steer the route during an active native call // Let the engine's own route change settle first, then override if we landed on the earpiece. DispatchQueue.main.asyncAfter(deadline: .now() + 0.15) { [weak self] in self?.preferSpeaker() } } // MARK: - LiveKit media private func connectRoom(url: String, token: String) { guard !url.isEmpty, !token.isEmpty else { return } // Ask for mic permission now (we still connect MUTED). If it stays "undetermined", enabling the audio // engine in didActivate can block on first use — determining it up front avoids that. AVAudioSession.sharedInstance().requestRecordPermission { _ in } let old = room let r = Room() room = r Task { [weak self] in await old?.disconnect() // drop any previous/stale connection so we never leave DUPLICATE participants do { try await r.connect(url: url, token: token) // House rule: answer MUTED. Not enabling the mic here means nothing is captured/published // until the user taps Mic in the meeting UI (→ setMuted(false) → setMicrophone(true)), which // is also when iOS asks for mic permission. Playback (hearing others) is unaffected. try await r.localParticipant.setMicrophone(enabled: false) self?.notifyListeners("callConnected", data: ["ok": true]) } catch { self?.notifyListeners("callError", data: ["error": String(describing: error)]) } } } private func disconnectRoom() { let r = room room = nil Task { await r?.disconnect() } removeAllTileViews() } // MARK: - Native video (tile rendering, Increment 2b) // // The WebView owns the meeting UI (grid, roster, controls) but — for a native call — has NO LiveKit // connection, so it can't render any video. The video lives only in the plugin's LiveKit connection. // So we draw native VideoViews on top of the WebView, positioned to match each web tile: the web reports // each tile's on-screen rect + the participant's user id (syncVideoTiles), and we place/size a VideoView // for whichever participants currently have a live camera track. Views are subviews of the WKWebView, so // their frames use the SAME coordinate space as getBoundingClientRect (CSS px == points, both // viewport-relative) and stay aligned as the page scrolls. // Find the live (unmuted, subscribed) camera track for a user id — nil when the camera is off, so the web // tile's avatar shows through instead. private func cameraTrack(forUid uid: String, isLocal: Bool) -> VideoTrack? { guard let room = room else { return nil } let pubs: [TrackPublication] if isLocal { pubs = room.localParticipant.videoTracks } else { guard let p = room.remoteParticipants[Participant.Identity(from: uid)] else { return nil } pubs = p.videoTracks } guard let pub = pubs.first(where: { $0.source == .camera && !$0.isMuted && $0.track != nil }) else { return nil } return pub.track as? VideoTrack } private func tileKey(uid: String, isLocal: Bool) -> String { isLocal ? "__local" : uid } private func removeAllTileViews() { DispatchQueue.main.async { [weak self] in guard let self = self else { return } for (_, vv) in self.tileViews { vv.removeFromSuperview() } self.tileViews.removeAll() } } // Tags for the overlay subviews we add to each tile's VideoView (name label + mute badge). private static let nameTag = 9001 private static let muteTag = 9002 // Build a tile VideoView with its native name label (bottom-left) + mute badge (bottom-right) — the video // covers the web tile, so these redraw the essentials the web can no longer show through. private func makeTileView(host: UIView, key: String) -> VideoView { let v = VideoView() v.layoutMode = .fill v.backgroundColor = .black v.clipsToBounds = true v.layer.cornerRadius = 8 host.addSubview(v) let label = UILabel() label.tag = NativeCallPlugin.nameTag label.font = .systemFont(ofSize: 12, weight: .semibold) label.textColor = .white label.shadowColor = UIColor.black.withAlphaComponent(0.9) // legible over any video label.shadowOffset = CGSize(width: 0, height: 1) label.translatesAutoresizingMaskIntoConstraints = false v.addSubview(label) let badge = UIImageView(image: UIImage(systemName: "mic.slash.fill")) badge.tag = NativeCallPlugin.muteTag badge.tintColor = .white badge.contentMode = .center badge.backgroundColor = UIColor.black.withAlphaComponent(0.55) badge.layer.cornerRadius = 9 badge.clipsToBounds = true badge.translatesAutoresizingMaskIntoConstraints = false v.addSubview(badge) NSLayoutConstraint.activate([ label.leadingAnchor.constraint(equalTo: v.leadingAnchor, constant: 6), label.bottomAnchor.constraint(equalTo: v.bottomAnchor, constant: -5), label.trailingAnchor.constraint(lessThanOrEqualTo: badge.leadingAnchor, constant: -4), badge.trailingAnchor.constraint(equalTo: v.trailingAnchor, constant: -6), badge.bottomAnchor.constraint(equalTo: v.bottomAnchor, constant: -5), badge.widthAnchor.constraint(equalToConstant: 18), badge.heightAnchor.constraint(equalToConstant: 18), ]) tileViews[key] = v return v } private func updateTileOverlay(_ vv: VideoView, name: String, muted: Bool) { // Keep the overlays above the video renderer (VideoView adds its renderer when the track is set). if let label = vv.viewWithTag(NativeCallPlugin.nameTag) as? UILabel { label.text = name; label.isHidden = name.isEmpty; vv.bringSubviewToFront(label) } if let badge = vv.viewWithTag(NativeCallPlugin.muteTag) { badge.isHidden = !muted; vv.bringSubviewToFront(badge) } } // Route call audio to the LOUDSPEAKER when we'd otherwise be on the quiet earpiece. Guarded so a connected // wired/Bluetooth headset (any non-receiver route) still wins. Re-asserted on mute toggles because enabling // the mic can flip the route back to the earpiece. private func preferSpeaker() { let s = AVAudioSession.sharedInstance() if s.currentRoute.outputs.contains(where: { $0.portType == .builtInReceiver }) { try? s.overrideOutputAudioPort(.speaker) } } // MARK: - JS-callable methods @objc func getToken(_ call: CAPPluginCall) { call.resolve(["token": voipToken]) } // Outgoing call from the web app: start a CallKit outgoing call AND connect the LiveKit room natively. @objc func reportOutgoingCall(_ call: CAPPluginCall) { guard let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) else { call.reject("callUUID required"); return } let url = call.getString("url") ?? "" let token = call.getString("token") ?? "" let video = call.getBool("hasVideo") ?? false calls[uuid] = ["room": call.getString("room") ?? "", "kind": call.getString("kind") ?? "dm", "livekitUrl": url, "livekitToken": token] activeUUID = uuid let handle = CXHandle(type: .generic, value: call.getString("peerName") ?? "Call") let start = CXStartCallAction(call: uuid, handle: handle) start.isVideo = video callController.request(CXTransaction(action: start)) { [weak self] error in if let error = error { call.reject(error.localizedDescription); return } self?.connectRoom(url: url, token: token) // Start muted (house rule) — also reflect it on the CallKit system call screen. self?.callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: true))) { _ in } self?.provider?.reportOutgoingCall(with: uuid, connectedAt: Date()) call.resolve() } } // Ring CallKit from a WEBSOCKET call event (a 2nd path alongside the VoIP push). When the app is open the // WS is connected and reliable, but the VoIP push can be delayed/dropped → "sometimes it doesn't ring". // Deduped by UUID: if the VoIP push already reported this call, this is a no-op (and vice-versa). @objc func reportIncomingCall(_ call: CAPPluginCall) { guard let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) else { call.reject("callUUID required"); return } // Already handled (ended, ringing, or active) → don't re-report (CallKit would reject a dup anyway). if endedCalls.contains(uuid) || calls[uuid] != nil || activeUUID == uuid { call.resolve(); return } let callerName = call.getString("callerName") ?? call.getString("groupName") ?? "Incoming call" let hasVideo = call.getBool("hasVideo") ?? false calls[uuid] = [ "callUUID": uuidStr, "room": call.getString("room") ?? "", "kind": call.getString("kind") ?? "dm", "callerId": call.getString("callerId") ?? "", "callerName": callerName, "groupId": call.getString("groupId") ?? "", "groupName": call.getString("groupName") ?? "", "hasVideo": hasVideo, "livekitUrl": call.getString("url") ?? "", "livekitToken": call.getString("token") ?? "", ] let update = CXCallUpdate() update.remoteHandle = CXHandle(type: .generic, value: callerName) update.localizedCallerName = callerName update.hasVideo = hasVideo update.supportsHolding = false; update.supportsGrouping = false; update.supportsUngrouping = false provider?.reportNewIncomingCall(with: uuid, update: update) { _ in } call.resolve() } @objc func setMuted(_ call: CAPPluginCall) { let muted = call.getBool("muted") ?? false // Drive mute THROUGH CallKit so the system call screen and the in-app meeting UI stay in sync (the // CXSetMutedCallAction handler does the actual setMicrophone). Fall back to a direct call if somehow // there's no active CallKit call. if let uuid = activeUUID { callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: muted))) { _ in } } else { let r = room Task { try? await r?.localParticipant.setMicrophone(enabled: !muted) } } call.resolve() } // Enable/disable the local camera. Publishing it makes this user's video appear for everyone else (their // web/desktop clients render it via their own SFU subscription); locally it's drawn on the __local tile by // syncVideoTiles (the web triggers a sync right after this resolves). Front camera only for now. @objc func setCamera(_ call: CAPPluginCall) { guard let r = room else { call.reject("no active call"); return } let on = call.getBool("on") ?? false Task { do { try await r.localParticipant.setCamera( enabled: on, captureOptions: CameraCaptureOptions(position: .front)) call.resolve(["on": on]) } catch { call.reject("camera failed: \(String(describing: error))") } } } // Position native video views to match the web meeting tiles. `tiles` = [{uid, local, x, y, w, h}] in // CSS px (== points; getBoundingClientRect coords). We create/move a VideoView for each participant that // has a live camera track, and remove views for tiles that are gone or whose camera is off (so the web // avatar shows). Called on a short poll by the web while a native call is on screen, plus on demand. @objc func syncVideoTiles(_ call: CAPPluginCall) { let tiles = (call.getArray("tiles") as? [[String: Any]]) ?? [] DispatchQueue.main.async { [weak self] in guard let self = self else { call.resolve(); return } guard let host = self.bridge?.webView else { call.resolve(); return } var wanted = Set() for t in tiles { guard let uid = t["uid"] as? String, !uid.isEmpty else { continue } let isLocal = (t["local"] as? Bool) ?? false func num(_ k: String) -> CGFloat { CGFloat((t[k] as? NSNumber)?.doubleValue ?? 0) } let x = num("x"), y = num("y"), w = num("w"), h = num("h") if w < 2 || h < 2 { continue } let key = self.tileKey(uid: uid, isLocal: isLocal) guard let track = self.cameraTrack(forUid: uid, isLocal: isLocal) else { continue } // camera off → leave the web avatar wanted.insert(key) let vv = self.tileViews[key] ?? self.makeTileView(host: host, key: key) if vv.superview !== host { host.addSubview(vv) } if vv.track !== track { vv.track = track } vv.frame = CGRect(x: x, y: y, width: w, height: h) // Native video covers the web tile, so redraw the essentials (name + muted) natively. self.updateTileOverlay(vv, name: (t["name"] as? String) ?? "", muted: (t["muted"] as? Bool) ?? false) } // Drop views for participants no longer present / camera turned off. for (key, vv) in self.tileViews where !wanted.contains(key) { vv.removeFromSuperview(); self.tileViews.removeValue(forKey: key) } call.resolve() } } // End the CallKit call (remote hung up / user ended from the web UI). No callUUID -> end all. @objc func endCall(_ call: CAPPluginCall) { if let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) { requestEnd(uuid) } else { for uuid in calls.keys { requestEnd(uuid) } if let a = activeUUID { requestEnd(a) } } call.resolve() } private func requestEnd(_ uuid: UUID) { callController.request(CXTransaction(action: CXEndCallAction(call: uuid))) { _ in } calls.removeValue(forKey: uuid) endedCalls.insert(uuid) } // MARK: - PushKit (VoIP) public func pushRegistry(_ registry: PKPushRegistry, didUpdate pushCredentials: PKPushCredentials, for type: PKPushType) { let token = pushCredentials.token.map { String(format: "%02x", $0) }.joined() voipToken = token notifyListeners("voipToken", data: ["token": token]) } public func pushRegistry(_ registry: PKPushRegistry, didInvalidatePushTokenFor type: PKPushType) { voipToken = "" } // Incoming VoIP push. iOS 13+: we MUST reportNewIncomingCall for EVERY push before completion(), or the // app is terminated. public func pushRegistry(_ registry: PKPushRegistry, didReceiveIncomingPushWith payload: PKPushPayload, for type: PKPushType, completion: @escaping () -> Void) { var dict: [String: Any] = [:] for (k, v) in payload.dictionaryPayload { if let ks = k as? String { dict[ks] = v } } let uuid = UUID(uuidString: (dict["callUUID"] as? String) ?? "") ?? UUID() // CANCEL: caller hung up / declined / timed out. iOS requires a reported call per push, but blindly // reporting a NEW incoming call is what caused the "rings back for a second" blip on a call we've // already handled. So: if (dict["type"] as? String) == "cancel" { // iOS 13+ TERMINATES the app if a VoIP push doesn't result in reportNewIncomingCall before // completion() — that's the crash after several calls (my earlier "no-blip" optimization SKIPPED // the report for known/ended calls, which iOS kills for). So ALWAYS report, then immediately end: // * uuid already ringing/active → the report errors harmlessly (NO second ring), reportCall ends it. // * late/duplicate cancel (done) → a brief, unavoidable blip (acceptable; a crash is not). let u = CXCallUpdate() u.remoteHandle = CXHandle(type: .generic, value: (dict["callerName"] as? String) ?? "Call") let wasActive = (activeUUID == uuid) var ev = calls[uuid] ?? [:]; ev["callUUID"] = uuid.uuidString calls.removeValue(forKey: uuid) endedCalls.insert(uuid) if wasActive { disconnectRoom(); activeUUID = nil } provider?.reportNewIncomingCall(with: uuid, update: u) { [weak self] _ in self?.provider?.reportCall(with: uuid, endedAt: Date(), reason: .remoteEnded) if wasActive { self?.notifyListeners("endCall", data: ev) } // active call cancelled → leave the meeting completion() } return } let callerName = (dict["callerName"] as? String) ?? (dict["groupName"] as? String) ?? "Incoming call" let hasVideo = (dict["hasVideo"] as? Bool) ?? false calls[uuid] = dict let update = CXCallUpdate() update.remoteHandle = CXHandle(type: .generic, value: callerName) update.localizedCallerName = callerName update.hasVideo = hasVideo update.supportsHolding = false update.supportsGrouping = false update.supportsUngrouping = false provider?.reportNewIncomingCall(with: uuid, update: update) { _ in completion() } } // MARK: - CXProviderDelegate public func providerDidReset(_ provider: CXProvider) { disconnectRoom() calls.removeAll(); endedCalls.removeAll(); activeUUID = nil } public func provider(_ provider: CXProvider, perform action: CXAnswerCallAction) { let uuid = action.callUUID let data = calls[uuid] ?? [:] activeUUID = uuid action.fulfill() // Keep the CallKit call ACTIVE (foregrounds the app + keeps the call alive). Connect the LiveKit room // NATIVELY so the mic works with the active CallKit call and audio survives backgrounding. connectRoom(url: (data["livekitUrl"] as? String) ?? "", token: (data["livekitToken"] as? String) ?? "") // We answer muted (house rule) — reflect that on the CallKit system call screen (the mic button there // showed unmuted before, even though we were functionally muted). callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: true))) { _ in } var ev = data; ev["callUUID"] = uuid.uuidString // Answering from a KILLED/locked state launches the app — the WebView's JS listener may not be attached // yet, so retain the event until it is. Without this the "answered" signal is lost, the server's // unanswered timer fires, and the call is cancelled ~40s after pickup. notifyListeners("answerCall", data: ev, retainUntilConsumed: true) } public func provider(_ provider: CXProvider, perform action: CXEndCallAction) { var data: [String: Any] = calls[action.callUUID] ?? [:] data["callUUID"] = action.callUUID.uuidString disconnectRoom() notifyListeners("endCall", data: data) calls.removeValue(forKey: action.callUUID) endedCalls.insert(action.callUUID) if activeUUID == action.callUUID { activeUUID = nil } action.fulfill() } public func provider(_ provider: CXProvider, perform action: CXStartCallAction) { provider.reportOutgoingCall(with: action.callUUID, startedConnectingAt: Date()) action.fulfill() } public func provider(_ provider: CXProvider, perform action: CXSetMutedCallAction) { let r = room Task { [weak self] in try? await r?.localParticipant.setMicrophone(enabled: !action.isMuted) self?.preferSpeaker() // toggling the mic can flip the route back to the earpiece — re-assert speaker } notifyListeners("setMuted", data: ["callUUID": action.callUUID.uuidString, "muted": action.isMuted]) action.fulfill() } // CallKit owns the AVAudioSession lifecycle. Since LiveKit auto-config is OFF, configure the session HERE — // when CallKit activates it — and enable LiveKit's audio engine. This is the fix for the intermittent // dead mic / no-audio and late speaker routing. Don't call setActive(true): CallKit already activated it. public func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) { do { // Route to the LOUDSPEAKER by default (earpiece was the .voiceChat default). .videoChat + // .defaultToSpeaker prefers the speaker while still letting wired/Bluetooth headsets win. try audioSession.setCategory(.playAndRecord, mode: .videoChat, options: [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP]) try AudioManager.shared.setEngineAvailability(.default) preferSpeaker() // belt-and-braces: if we still landed on the earpiece, force the speaker let route = audioSession.currentRoute.outputs.first?.portType.rawValue ?? "none" notifyListeners("audioActivated", data: ["ok": true, "route": route]) } catch { notifyListeners("audioActivated", data: ["ok": false, "error": String(describing: error)]) } } public func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) { try? AudioManager.shared.setEngineAvailability(.none) notifyListeners("audioDeactivated", data: ["ok": true]) } }