Sravan
e46ac1e7cc
In a scheduled meeting the WebView owns the mic (the plugin has no LiveKit room), so
the native-call AudioRenderer tap didn't apply. Now the WebView reads its OWN local
mic PCM via Web Audio (the same non-intrusive tap the app already uses for
active-speaker metering — NOT a 2nd getUserMedia, which would fight the call's mic on
iOS and yield silence), downsamples to 16 kHz mono Int16, and forwards it to the
plugin's SFSpeechRecognizer via feedAudio(). Native calls keep the LiveKit tap.
Muted -> the local track carries silence -> nothing transcribed; the feed re-inits
when the mic goes live on unmute.
- Plugin: startTranscription({external:true}) runs the recognizer without a track;
feedAudio({pcm,rate}) decodes base64 LE Int16 -> Float32 buffer -> recognizer.
- Web: startSR now uses the native recognizer for ALL iOS meetings (native call =
LiveKit tap, scheduled = PCM feed); desktop/browser unchanged (Web Speech API).
Web deploys now; the plugin's feedAudio/startExternal ride the next Codemagic build.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
793 lines
44 KiB
Swift
793 lines
44 KiB
Swift
import Foundation
|
|
import UIKit
|
|
import WebKit
|
|
import Capacitor
|
|
import PushKit
|
|
import CallKit
|
|
import AVFoundation
|
|
import Speech // #5 native transcript: SFSpeechRecognizer (iOS on-device speech-to-text; WKWebView has no Web Speech API)
|
|
import LiveKit // SPM product name is 'LiveKit' (Package.swift). (The old CocoaPods module was 'LiveKitClient'.)
|
|
|
|
// Native calling for Biz Connect (iOS). Registered by `cap sync` as window.Capacitor.Plugins.NativeCall.
|
|
//
|
|
// ARCHITECTURE (native media):
|
|
// * PushKit: registers for VoIP pushes, reports the VoIP token to JS (-> /api/v1/devices 'ios-voip').
|
|
// * CallKit: incoming VoIP push -> full-screen system ring (works when force-killed). The CallKit call
|
|
// stays ACTIVE for the whole call (that's what foregrounds/unlocks the app on answer and keeps the call
|
|
// alive in the background).
|
|
// * LiveKit: the call MEDIA runs NATIVELY via the LiveKit iOS SDK — so the mic works with an active CallKit
|
|
// call (WebKit's WebRTC could not) and audio survives backgrounding. The VoIP payload carries the
|
|
// LiveKit url+token so we can connect immediately on answer, even from a killed state; outgoing calls get
|
|
// the token from the WebView via reportOutgoingCall(). The WebView is UI only for native calls (it must
|
|
// NOT also join the room — LiveKit allows one connection per identity).
|
|
@objc(NativeCallPlugin)
|
|
public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelegate, CXProviderDelegate, BroadcastManagerDelegate {
|
|
public let identifier = "NativeCallPlugin"
|
|
public let jsName = "NativeCall"
|
|
public let pluginMethods: [CAPPluginMethod] = [
|
|
CAPPluginMethod(name: "getToken", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "reportOutgoingCall", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "reconnectRoom", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "startScreenShare", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "stopScreenShare", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "startTranscription", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "stopTranscription", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "feedAudio", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise)
|
|
]
|
|
|
|
private var pushRegistry: PKPushRegistry?
|
|
private var provider: CXProvider?
|
|
private let callController = CXCallController()
|
|
private var voipToken: String = ""
|
|
// callUUID -> the call's data (room, kind, callerId, livekitUrl, livekitToken, …).
|
|
private var calls: [UUID: [String: Any]] = [:]
|
|
// Calls we've already ended locally — so a LATE cancel push doesn't re-report (which briefly re-rang).
|
|
private var endedCalls = Set<UUID>()
|
|
private var room: Room?
|
|
private var activeUUID: UUID?
|
|
// Native video: one native video view per visible participant (key "__local" or the remote user id),
|
|
// drawn over the WebView and positioned to match the web meeting tiles (see syncVideoTiles). UIKit views —
|
|
// only ever touched on the main thread. TileVideoView adds pinch-zoom/pan for shared screens.
|
|
private var tileViews: [String: TileVideoView] = [:]
|
|
// Hole-punch: draw the native video BEHIND a transparent WebView so all web UI (bar, menus, panels) floats
|
|
// on top. Saved so we can restore the WebView when the call ends.
|
|
private var holePunchOn = false
|
|
private var savedVCBg: UIColor? // the WebView parent's original background, restored when the call ends
|
|
// #5 native transcript: transcribes THIS device's mic (WKWebView has no Web Speech API). Fed by a LiveKit
|
|
// AudioRenderer on the local mic track, so it reuses the call's already-open mic (no 2nd audio engine).
|
|
private let transcriber = SpeechTranscriber()
|
|
|
|
override public func load() {
|
|
let config = CXProviderConfiguration(localizedName: "Biz Connect")
|
|
config.supportsVideo = true
|
|
config.maximumCallGroups = 1
|
|
config.maximumCallsPerCallGroup = 1
|
|
config.supportedHandleTypes = [.generic]
|
|
let p = CXProvider(configuration: config)
|
|
p.setDelegate(self, queue: nil)
|
|
provider = p
|
|
|
|
let registry = PKPushRegistry(queue: .main)
|
|
registry.delegate = self
|
|
registry.desiredPushTypes = [.voIP]
|
|
pushRegistry = registry
|
|
// CallKit audio coordination (LiveKit 2.15+, pinned to the git tag in ios-patch.sh). LiveKit's automatic
|
|
// AVAudioSession config RACES CallKit's own activation → intermittent dead mic / no audio + the output
|
|
// route only settling once audio flows ("speaker turns on late"). Fix: turn auto-config OFF and keep the
|
|
// engine OFF; we configure the session and enable the engine ONLY in didActivate.
|
|
AudioManager.shared.audioSession.isAutomaticConfigurationEnabled = false
|
|
try? AudioManager.shared.setEngineAvailability(.none)
|
|
|
|
// Re-assert the LOUDSPEAKER whenever iOS routes call audio back to the quiet earpiece. The LiveKit
|
|
// audio engine starting up right after CallKit activates the session flips the route to the built-in
|
|
// receiver — that's the "sound is on the earpiece until I tap something" bug (tapping mic/cam re-ran
|
|
// preferSpeaker and fixed it). Listening for route changes makes that self-healing.
|
|
NotificationCenter.default.addObserver(self, selector: #selector(audioRouteChanged(_:)),
|
|
name: AVAudioSession.routeChangeNotification, object: nil)
|
|
|
|
// Screen sharing (ReplayKit broadcast extension). LiveKit tells us when a broadcast starts/stops and
|
|
// (with shouldPublishTrack=true, the default) auto-publishes/unpublishes the screen-share track.
|
|
BroadcastManager.shared.delegate = self
|
|
|
|
// #5 native transcript: each finalized speech segment → the web (meeting-transcript over the WS), which
|
|
// merges it into the shared meeting transcript (same path as desktop's Web Speech API).
|
|
transcriber.onFinal = { [weak self] text in self?.notifyListeners("transcript", data: ["text": text]) }
|
|
|
|
// Make the WebView transparent at STARTUP. WKWebView can IGNORE isOpaque=false when it's flipped after
|
|
// the page has already rendered — the likely reason the hole-punch showed no video. Doing it once, up
|
|
// front, makes the transparent-meeting areas actually reveal the native video behind the WebView. The
|
|
// web body is opaque, so the app looks normal outside a call.
|
|
DispatchQueue.main.async { [weak self] in self?.makeWebViewTransparent() }
|
|
}
|
|
|
|
private func makeWebViewTransparent() {
|
|
guard let web = bridge?.webView else { return }
|
|
web.isOpaque = false
|
|
web.backgroundColor = .clear
|
|
web.scrollView.isOpaque = false // the scrollView being opaque can occlude native content behind the WebView
|
|
web.scrollView.backgroundColor = .clear
|
|
}
|
|
|
|
@objc private func audioRouteChanged(_ note: Notification) {
|
|
guard room != nil else { return } // only steer the route during an active native call
|
|
// Let the engine's own route change settle first, then override if we landed on the earpiece.
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 0.15) { [weak self] in self?.preferSpeaker() }
|
|
}
|
|
|
|
// MARK: - LiveKit media
|
|
|
|
private func connectRoom(url: String, token: String) {
|
|
guard !url.isEmpty, !token.isEmpty else { return }
|
|
// Ask for mic permission now (we still connect MUTED). If it stays "undetermined", enabling the audio
|
|
// engine in didActivate can block on first use — determining it up front avoids that.
|
|
AVAudioSession.sharedInstance().requestRecordPermission { _ in }
|
|
let old = room
|
|
// Route screen-share through the ReplayKit broadcast extension (so the user can share their screen
|
|
// even when the app is backgrounded, and it captures the whole phone, not just the WebView).
|
|
let opts = RoomOptions(defaultScreenShareCaptureOptions: ScreenShareCaptureOptions(useBroadcastExtension: true))
|
|
let r = Room(roomOptions: opts)
|
|
room = r
|
|
Task { [weak self] in
|
|
await old?.disconnect() // drop any previous/stale connection so we never leave DUPLICATE participants
|
|
do {
|
|
try await r.connect(url: url, token: token)
|
|
// House rule: answer MUTED. Not enabling the mic here means nothing is captured/published
|
|
// until the user taps Mic in the meeting UI (→ setMuted(false) → setMicrophone(true)), which
|
|
// is also when iOS asks for mic permission. Playback (hearing others) is unaffected.
|
|
try await r.localParticipant.setMicrophone(enabled: false)
|
|
self?.notifyListeners("callConnected", data: ["ok": true])
|
|
} catch {
|
|
self?.notifyListeners("callError", data: ["error": String(describing: error)])
|
|
}
|
|
}
|
|
}
|
|
|
|
private func disconnectRoom() {
|
|
let r = room
|
|
room = nil
|
|
transcriber.stop() // #5: end any live transcription with the call
|
|
Task { await r?.disconnect() }
|
|
removeAllTileViews()
|
|
DispatchQueue.main.async { [weak self] in guard let self = self, let web = self.bridge?.webView else { return }; self.applyHolePunchRestore(web) }
|
|
}
|
|
|
|
// MARK: - Native video (tile rendering, Increment 2b)
|
|
//
|
|
// The WebView owns the meeting UI (grid, roster, controls) but — for a native call — has NO LiveKit
|
|
// connection, so it can't render any video. The video lives only in the plugin's LiveKit connection.
|
|
// So we draw native VideoViews on top of the WebView, positioned to match each web tile: the web reports
|
|
// each tile's on-screen rect + the participant's user id (syncVideoTiles), and we place/size a VideoView
|
|
// for whichever participants currently have a live camera track. Views are subviews of the WKWebView, so
|
|
// their frames use the SAME coordinate space as getBoundingClientRect (CSS px == points, both
|
|
// viewport-relative) and stay aligned as the page scrolls.
|
|
|
|
// Find the live (unmuted, subscribed) camera track for a user id — nil when the camera is off, so the web
|
|
// tile's avatar shows through instead.
|
|
private func cameraTrack(forUid uid: String, isLocal: Bool) -> VideoTrack? {
|
|
guard let room = room else { return nil }
|
|
let pubs: [TrackPublication]
|
|
if isLocal {
|
|
pubs = room.localParticipant.videoTracks
|
|
} else {
|
|
guard let p = room.remoteParticipants[Participant.Identity(from: uid)] else { return nil }
|
|
pubs = p.videoTracks
|
|
}
|
|
guard let pub = pubs.first(where: { $0.source == .camera && !$0.isMuted && $0.track != nil }) else { return nil }
|
|
return pub.track as? VideoTrack
|
|
}
|
|
|
|
// The remote screen-share track for a user id — nil if they aren't sharing (or it isn't subscribed yet).
|
|
// (Local screen-share isn't supported on iOS — no ReplayKit broadcast extension — so this is remote-only.)
|
|
private func screenTrack(forUid uid: String) -> VideoTrack? {
|
|
guard let room = room, let p = room.remoteParticipants[Participant.Identity(from: uid)] else { return nil }
|
|
guard let pub = p.videoTracks.first(where: { $0.source == .screenShareVideo && !$0.isMuted && $0.track != nil }) else { return nil }
|
|
return pub.track as? VideoTrack
|
|
}
|
|
|
|
private func tileKey(uid: String, isLocal: Bool) -> String { isLocal ? "__local" : uid }
|
|
|
|
private func removeAllTileViews() {
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self else { return }
|
|
for (_, vv) in self.tileViews { vv.removeFromSuperview() }
|
|
self.tileViews.removeAll()
|
|
}
|
|
}
|
|
|
|
// Create a tile view BEHIND the whole WebView (in its superview). WKWebView does NOT composite native
|
|
// subviews placed under its scrollView through transparent web content, so the video must sit behind the
|
|
// (transparent) WebView itself; the web UI then paints on top. No native overlays — the web tile draws them.
|
|
private func makeTileView(host: WKWebView, key: String) -> TileVideoView {
|
|
let v = TileVideoView(frame: .zero)
|
|
if let sup = host.superview { sup.insertSubview(v, belowSubview: host) } else { host.addSubview(v) }
|
|
tileViews[key] = v
|
|
return v
|
|
}
|
|
|
|
// Turn hole-punch on/off. Transparency is already set at startup (makeWebViewTransparent); here we just add
|
|
// a BLACK backing to the WebView's parent (the layer directly behind the video tiles) during the call, and
|
|
// remove it + the tiles afterwards.
|
|
@objc func setHolePunch(_ call: CAPPluginCall) {
|
|
let on = call.getBool("on") ?? false
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self, let web = self.bridge?.webView else { call.resolve(); return }
|
|
self.makeWebViewTransparent() // belt-and-braces
|
|
let parent = web.superview
|
|
if on {
|
|
if !self.holePunchOn { self.savedVCBg = parent?.backgroundColor }
|
|
parent?.backgroundColor = .black
|
|
self.holePunchOn = true
|
|
} else {
|
|
self.applyHolePunchRestore(web)
|
|
for (_, vv) in self.tileViews { vv.removeFromSuperview() }
|
|
self.tileViews.removeAll()
|
|
}
|
|
call.resolve()
|
|
}
|
|
}
|
|
|
|
// Remove the black backing (leave the WebView transparent — the web body is opaque, so it looks normal).
|
|
private func applyHolePunchRestore(_ web: WKWebView) {
|
|
guard holePunchOn else { return }
|
|
web.superview?.backgroundColor = savedVCBg
|
|
holePunchOn = false
|
|
}
|
|
|
|
// Web-forwarded zoom: touches land on the WebView (on top), so the web captures pinch/pan on the shared
|
|
// screen and forwards the transform here; we apply it to that tile's inner video.
|
|
@objc func setTileZoom(_ call: CAPPluginCall) {
|
|
let key = tileKey(uid: call.getString("uid") ?? "", isLocal: call.getBool("local") ?? false)
|
|
let scale = CGFloat(call.getDouble("scale") ?? 1)
|
|
let tx = CGFloat(call.getDouble("tx") ?? 0)
|
|
let ty = CGFloat(call.getDouble("ty") ?? 0)
|
|
DispatchQueue.main.async { [weak self] in self?.tileViews[key]?.applyZoom(scale: scale, tx: tx, ty: ty) }
|
|
call.resolve()
|
|
}
|
|
|
|
// Route call audio to the LOUDSPEAKER when we'd otherwise be on the quiet earpiece. Guarded so a connected
|
|
// wired/Bluetooth headset (any non-receiver route) still wins. Re-asserted on mute toggles because enabling
|
|
// the mic can flip the route back to the earpiece.
|
|
private func preferSpeaker() {
|
|
let s = AVAudioSession.sharedInstance()
|
|
if s.currentRoute.outputs.contains(where: { $0.portType == .builtInReceiver }) {
|
|
try? s.overrideOutputAudioPort(.speaker)
|
|
}
|
|
}
|
|
|
|
// MARK: - JS-callable methods
|
|
|
|
@objc func getToken(_ call: CAPPluginCall) { call.resolve(["token": voipToken]) }
|
|
|
|
// Outgoing call from the web app: start a CallKit outgoing call AND connect the LiveKit room natively.
|
|
@objc func reportOutgoingCall(_ call: CAPPluginCall) {
|
|
guard let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) else {
|
|
call.reject("callUUID required"); return
|
|
}
|
|
let url = call.getString("url") ?? ""
|
|
let token = call.getString("token") ?? ""
|
|
let video = call.getBool("hasVideo") ?? false
|
|
calls[uuid] = ["room": call.getString("room") ?? "", "kind": call.getString("kind") ?? "dm",
|
|
"livekitUrl": url, "livekitToken": token]
|
|
activeUUID = uuid
|
|
let handle = CXHandle(type: .generic, value: call.getString("peerName") ?? "Call")
|
|
let start = CXStartCallAction(call: uuid, handle: handle)
|
|
start.isVideo = video
|
|
callController.request(CXTransaction(action: start)) { [weak self] error in
|
|
if let error = error { call.reject(error.localizedDescription); return }
|
|
self?.connectRoom(url: url, token: token)
|
|
// Start muted (house rule) — also reflect it on the CallKit system call screen.
|
|
self?.callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: true))) { _ in }
|
|
self?.provider?.reportOutgoingCall(with: uuid, connectedAt: Date())
|
|
call.resolve()
|
|
}
|
|
}
|
|
|
|
// Ring CallKit from a WEBSOCKET call event (a 2nd path alongside the VoIP push). When the app is open the
|
|
// WS is connected and reliable, but the VoIP push can be delayed/dropped → "sometimes it doesn't ring".
|
|
// Deduped by UUID: if the VoIP push already reported this call, this is a no-op (and vice-versa).
|
|
@objc func reportIncomingCall(_ call: CAPPluginCall) {
|
|
guard let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) else {
|
|
call.reject("callUUID required"); return
|
|
}
|
|
// Already handled (ended, ringing, or active) → don't re-report (CallKit would reject a dup anyway).
|
|
if endedCalls.contains(uuid) || calls[uuid] != nil || activeUUID == uuid { call.resolve(); return }
|
|
let callerName = call.getString("callerName") ?? call.getString("groupName") ?? "Incoming call"
|
|
let hasVideo = call.getBool("hasVideo") ?? false
|
|
calls[uuid] = [
|
|
"callUUID": uuidStr, "room": call.getString("room") ?? "", "kind": call.getString("kind") ?? "dm",
|
|
"callerId": call.getString("callerId") ?? "", "callerName": callerName,
|
|
"groupId": call.getString("groupId") ?? "", "groupName": call.getString("groupName") ?? "",
|
|
"hasVideo": hasVideo, "livekitUrl": call.getString("url") ?? "", "livekitToken": call.getString("token") ?? "",
|
|
]
|
|
let update = CXCallUpdate()
|
|
update.remoteHandle = CXHandle(type: .generic, value: callerName)
|
|
update.localizedCallerName = callerName
|
|
update.hasVideo = hasVideo
|
|
update.supportsHolding = false; update.supportsGrouping = false; update.supportsUngrouping = false
|
|
provider?.reportNewIncomingCall(with: uuid, update: update) { _ in }
|
|
call.resolve()
|
|
}
|
|
|
|
// #12 Multi-device: swap the media connection to a token whose identity = this device's mesh peerId (unique
|
|
// per connection). The web calls this once the native WebView has joined the mesh and has a peerId. On
|
|
// answer we connected INSTANTLY with the push token (identity=userId) for zero-latency audio; this re-homes
|
|
// the media onto the unique peerId identity so two devices of the same user are distinct LiveKit
|
|
// participants (LiveKit allows one connection per identity — otherwise the older device is kicked and
|
|
// "audio jumps to whichever joined last"). connectRoom disconnects the old room first; the CallKit call and
|
|
// its audio session stay active, so audio just re-attaches. callConnected fires on success → web re-applies
|
|
// mic/cam. Guarded to only run during an active call.
|
|
@objc func reconnectRoom(_ call: CAPPluginCall) {
|
|
let url = call.getString("url") ?? ""
|
|
let token = call.getString("token") ?? ""
|
|
guard room != nil, !url.isEmpty, !token.isEmpty else { call.resolve(); return }
|
|
connectRoom(url: url, token: token)
|
|
call.resolve()
|
|
}
|
|
|
|
@objc func setMuted(_ call: CAPPluginCall) {
|
|
let muted = call.getBool("muted") ?? false
|
|
// Drive mute THROUGH CallKit so the system call screen and the in-app meeting UI stay in sync (the
|
|
// CXSetMutedCallAction handler does the actual setMicrophone). Fall back to a direct call if somehow
|
|
// there's no active CallKit call.
|
|
if let uuid = activeUUID {
|
|
callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: muted))) { _ in }
|
|
} else {
|
|
let r = room
|
|
Task { try? await r?.localParticipant.setMicrophone(enabled: !muted) }
|
|
}
|
|
call.resolve()
|
|
}
|
|
|
|
// Enable/disable the local camera. Publishing it makes this user's video appear for everyone else (their
|
|
// web/desktop clients render it via their own SFU subscription); locally it's drawn on the __local tile by
|
|
// syncVideoTiles (the web triggers a sync right after this resolves). Front camera only for now.
|
|
@objc func setCamera(_ call: CAPPluginCall) {
|
|
guard let r = room else { call.reject("no active call"); return }
|
|
let on = call.getBool("on") ?? false
|
|
Task {
|
|
do {
|
|
try await r.localParticipant.setCamera(
|
|
enabled: on,
|
|
captureOptions: CameraCaptureOptions(position: .front))
|
|
call.resolve(["on": on])
|
|
} catch {
|
|
call.reject("camera failed: \(String(describing: error))")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Flip the local camera between front and back.
|
|
@objc func switchCamera(_ call: CAPPluginCall) {
|
|
guard let r = room else { call.reject("no active call"); return }
|
|
let pub = r.localParticipant.videoTracks.first(where: { $0.source == .camera })
|
|
guard let track = pub?.track as? LocalVideoTrack, let cam = track.capturer as? CameraCapturer else {
|
|
call.reject("camera not active"); return
|
|
}
|
|
Task {
|
|
do { _ = try await cam.switchCameraPosition(); call.resolve() }
|
|
catch { call.reject("switch failed: \(String(describing: error))") }
|
|
}
|
|
}
|
|
|
|
// Screen sharing from iOS: show the system broadcast picker. When the user starts the broadcast, the
|
|
// extension streams the screen to us over IPC and LiveKit publishes it (BroadcastManager.shouldPublishTrack
|
|
// defaults true). broadcastManager(didChangeState:) fires screenShareState back to the web either way.
|
|
@objc func startScreenShare(_ call: CAPPluginCall) {
|
|
guard room != nil else { call.reject("no active call"); return }
|
|
DispatchQueue.main.async { BroadcastManager.shared.requestActivation() } // presents RPSystemBroadcastPickerView
|
|
call.resolve()
|
|
}
|
|
|
|
@objc func stopScreenShare(_ call: CAPPluginCall) {
|
|
BroadcastManager.shared.requestStop()
|
|
call.resolve()
|
|
}
|
|
|
|
// MARK: - #5 Live transcript (native SFSpeechRecognizer)
|
|
|
|
// The current published local mic track — the source we tap for transcription. nil until the user has
|
|
// unmuted at least once (the mic is published on unmute), or between reconnects.
|
|
private func localAudioTrack() -> LocalAudioTrack? {
|
|
return room?.localParticipant.audioTracks.first?.track as? LocalAudioTrack
|
|
}
|
|
|
|
// Start transcribing this device's mic (WKWebView has no Web Speech API). Two audio sources:
|
|
// * NATIVE CALL (default): the plugin owns the LiveKit room, so we tap the local mic track with a LiveKit
|
|
// AudioRenderer (reuses the call's open mic — reliable, no 2nd capturer). Re-attaches on unmute.
|
|
// * SCHEDULED/WEB MEETING ({external:true}): the WebView owns the mic (the plugin has no room), so the web
|
|
// reads its own mic PCM via Web Audio and pushes it here with feedAudio() — no second mic capture on iOS
|
|
// (two input units would fight and yield silence).
|
|
@objc func startTranscription(_ call: CAPPluginCall) {
|
|
if call.getBool("external") == true {
|
|
transcriber.startExternal()
|
|
} else {
|
|
transcriber.start(track: localAudioTrack())
|
|
}
|
|
call.resolve()
|
|
}
|
|
|
|
@objc func stopTranscription(_ call: CAPPluginCall) {
|
|
transcriber.stop()
|
|
call.resolve()
|
|
}
|
|
|
|
// Web-forwarded mic PCM for the {external:true} path (scheduled/web meetings). `pcm` = base64 little-endian
|
|
// Int16 mono at `rate` Hz (the web downsamples to 16 kHz). Fed straight into the recognizer.
|
|
@objc func feedAudio(_ call: CAPPluginCall) {
|
|
guard let b64 = call.getString("pcm"), let data = Data(base64Encoded: b64) else { call.resolve(); return }
|
|
let rate = call.getDouble("rate") ?? 16000
|
|
transcriber.appendPCM(int16: data, sampleRate: rate)
|
|
call.resolve()
|
|
}
|
|
|
|
// MARK: - BroadcastManagerDelegate
|
|
|
|
public func broadcastManager(didChangeState isBroadcasting: Bool) {
|
|
notifyListeners("screenShareState", data: ["sharing": isBroadcasting])
|
|
}
|
|
|
|
// Position native video views to match the web meeting tiles. `tiles` = [{uid, local, x, y, w, h}] in
|
|
// CSS px (== points; getBoundingClientRect coords). We create/move a VideoView for each participant that
|
|
// has a live camera track, and remove views for tiles that are gone or whose camera is off (so the web
|
|
// avatar shows). Called on a short poll by the web while a native call is on screen, plus on demand.
|
|
@objc func syncVideoTiles(_ call: CAPPluginCall) {
|
|
let tiles = (call.getArray("tiles") as? [[String: Any]]) ?? []
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self else { call.resolve(); return }
|
|
guard let host = self.bridge?.webView else { call.resolve(); return }
|
|
var wanted = Set<String>()
|
|
for t in tiles {
|
|
guard let uid = t["uid"] as? String, !uid.isEmpty else { continue }
|
|
let isLocal = (t["local"] as? Bool) ?? false
|
|
func num(_ k: String) -> CGFloat { CGFloat((t[k] as? NSNumber)?.doubleValue ?? 0) }
|
|
let x = num("x"), y = num("y"), w = num("w"), h = num("h")
|
|
if w < 2 || h < 2 { continue }
|
|
let key = self.tileKey(uid: uid, isLocal: isLocal)
|
|
// A tile flagged `screen` is a sharer's stage tile → show their screen-share track (fit, so it
|
|
// isn't cropped); otherwise the camera (fill). Either is nil when off → web avatar shows.
|
|
let wantScreen = (t["screen"] as? Bool) ?? false
|
|
guard let track = wantScreen ? self.screenTrack(forUid: uid)
|
|
: self.cameraTrack(forUid: uid, isLocal: isLocal) else { continue }
|
|
wanted.insert(key)
|
|
let vv = self.tileViews[key] ?? self.makeTileView(host: host, key: key)
|
|
if vv.superview !== host.superview, let sup = host.superview { sup.insertSubview(vv, belowSubview: host) } // keep BEHIND the transparent WebView
|
|
vv.layoutMode = wantScreen ? .fit : .fill
|
|
if vv.track !== track { vv.track = track }
|
|
// The container tracks the tile rect. getBoundingClientRect is in the WebView's coordinate space;
|
|
// convert it into the superview (where the tiles live) — robust to any WebView offset/inset. The
|
|
// zoom transform (web-forwarded via setTileZoom) lives on the INNER video, so this never fights it.
|
|
vv.frame = host.convert(CGRect(x: x, y: y, width: w, height: h), to: host.superview)
|
|
}
|
|
// Drop views for participants no longer present / camera turned off.
|
|
for (key, vv) in self.tileViews where !wanted.contains(key) {
|
|
vv.removeFromSuperview(); self.tileViews.removeValue(forKey: key)
|
|
}
|
|
call.resolve()
|
|
}
|
|
}
|
|
|
|
// End the CallKit call (remote hung up / user ended from the web UI). No callUUID -> end all.
|
|
@objc func endCall(_ call: CAPPluginCall) {
|
|
if let uuidStr = call.getString("callUUID"), let uuid = UUID(uuidString: uuidStr) {
|
|
requestEnd(uuid)
|
|
} else {
|
|
for uuid in calls.keys { requestEnd(uuid) }
|
|
if let a = activeUUID { requestEnd(a) }
|
|
}
|
|
call.resolve()
|
|
}
|
|
|
|
private func requestEnd(_ uuid: UUID) {
|
|
callController.request(CXTransaction(action: CXEndCallAction(call: uuid))) { _ in }
|
|
calls.removeValue(forKey: uuid)
|
|
endedCalls.insert(uuid)
|
|
}
|
|
|
|
// MARK: - PushKit (VoIP)
|
|
|
|
public func pushRegistry(_ registry: PKPushRegistry, didUpdate pushCredentials: PKPushCredentials, for type: PKPushType) {
|
|
let token = pushCredentials.token.map { String(format: "%02x", $0) }.joined()
|
|
voipToken = token
|
|
notifyListeners("voipToken", data: ["token": token])
|
|
}
|
|
|
|
public func pushRegistry(_ registry: PKPushRegistry, didInvalidatePushTokenFor type: PKPushType) {
|
|
voipToken = ""
|
|
}
|
|
|
|
// Incoming VoIP push. iOS 13+: we MUST reportNewIncomingCall for EVERY push before completion(), or the
|
|
// app is terminated.
|
|
public func pushRegistry(_ registry: PKPushRegistry, didReceiveIncomingPushWith payload: PKPushPayload, for type: PKPushType, completion: @escaping () -> Void) {
|
|
var dict: [String: Any] = [:]
|
|
for (k, v) in payload.dictionaryPayload { if let ks = k as? String { dict[ks] = v } }
|
|
let uuid = UUID(uuidString: (dict["callUUID"] as? String) ?? "") ?? UUID()
|
|
|
|
// CANCEL: caller hung up / declined / timed out. iOS requires a reported call per push, but blindly
|
|
// reporting a NEW incoming call is what caused the "rings back for a second" blip on a call we've
|
|
// already handled. So:
|
|
if (dict["type"] as? String) == "cancel" {
|
|
// iOS 13+ TERMINATES the app if a VoIP push doesn't result in reportNewIncomingCall before
|
|
// completion() — that's the crash after several calls (my earlier "no-blip" optimization SKIPPED
|
|
// the report for known/ended calls, which iOS kills for). So ALWAYS report, then immediately end:
|
|
// * uuid already ringing/active → the report errors harmlessly (NO second ring), reportCall ends it.
|
|
// * late/duplicate cancel (done) → a brief, unavoidable blip (acceptable; a crash is not).
|
|
let u = CXCallUpdate()
|
|
u.remoteHandle = CXHandle(type: .generic, value: (dict["callerName"] as? String) ?? "Call")
|
|
let wasActive = (activeUUID == uuid)
|
|
var ev = calls[uuid] ?? [:]; ev["callUUID"] = uuid.uuidString
|
|
calls.removeValue(forKey: uuid)
|
|
endedCalls.insert(uuid)
|
|
if wasActive { disconnectRoom(); activeUUID = nil }
|
|
provider?.reportNewIncomingCall(with: uuid, update: u) { [weak self] _ in
|
|
self?.provider?.reportCall(with: uuid, endedAt: Date(), reason: .remoteEnded)
|
|
if wasActive { self?.notifyListeners("endCall", data: ev) } // active call cancelled → leave the meeting
|
|
completion()
|
|
}
|
|
return
|
|
}
|
|
|
|
let callerName = (dict["callerName"] as? String) ?? (dict["groupName"] as? String) ?? "Incoming call"
|
|
let hasVideo = (dict["hasVideo"] as? Bool) ?? false
|
|
calls[uuid] = dict
|
|
|
|
let update = CXCallUpdate()
|
|
update.remoteHandle = CXHandle(type: .generic, value: callerName)
|
|
update.localizedCallerName = callerName
|
|
update.hasVideo = hasVideo
|
|
update.supportsHolding = false
|
|
update.supportsGrouping = false
|
|
update.supportsUngrouping = false
|
|
|
|
provider?.reportNewIncomingCall(with: uuid, update: update) { _ in completion() }
|
|
}
|
|
|
|
// MARK: - CXProviderDelegate
|
|
|
|
public func providerDidReset(_ provider: CXProvider) {
|
|
disconnectRoom()
|
|
calls.removeAll(); endedCalls.removeAll(); activeUUID = nil
|
|
}
|
|
|
|
public func provider(_ provider: CXProvider, perform action: CXAnswerCallAction) {
|
|
let uuid = action.callUUID
|
|
let data = calls[uuid] ?? [:]
|
|
activeUUID = uuid
|
|
action.fulfill()
|
|
// Keep the CallKit call ACTIVE (foregrounds the app + keeps the call alive). Connect the LiveKit room
|
|
// NATIVELY so the mic works with the active CallKit call and audio survives backgrounding.
|
|
connectRoom(url: (data["livekitUrl"] as? String) ?? "", token: (data["livekitToken"] as? String) ?? "")
|
|
// We answer muted (house rule) — reflect that on the CallKit system call screen (the mic button there
|
|
// showed unmuted before, even though we were functionally muted).
|
|
callController.request(CXTransaction(action: CXSetMutedCallAction(call: uuid, muted: true))) { _ in }
|
|
var ev = data; ev["callUUID"] = uuid.uuidString
|
|
// Answering from a KILLED/locked state launches the app — the WebView's JS listener may not be attached
|
|
// yet, so retain the event until it is. Without this the "answered" signal is lost, the server's
|
|
// unanswered timer fires, and the call is cancelled ~40s after pickup.
|
|
notifyListeners("answerCall", data: ev, retainUntilConsumed: true)
|
|
}
|
|
|
|
public func provider(_ provider: CXProvider, perform action: CXEndCallAction) {
|
|
var data: [String: Any] = calls[action.callUUID] ?? [:]
|
|
data["callUUID"] = action.callUUID.uuidString
|
|
disconnectRoom()
|
|
notifyListeners("endCall", data: data)
|
|
calls.removeValue(forKey: action.callUUID)
|
|
endedCalls.insert(action.callUUID)
|
|
if activeUUID == action.callUUID { activeUUID = nil }
|
|
action.fulfill()
|
|
}
|
|
|
|
public func provider(_ provider: CXProvider, perform action: CXStartCallAction) {
|
|
provider.reportOutgoingCall(with: action.callUUID, startedConnectingAt: Date())
|
|
action.fulfill()
|
|
}
|
|
|
|
public func provider(_ provider: CXProvider, perform action: CXSetMutedCallAction) {
|
|
let r = room
|
|
Task { [weak self] in
|
|
try? await r?.localParticipant.setMicrophone(enabled: !action.isMuted)
|
|
self?.preferSpeaker() // toggling the mic can flip the route back to the earpiece — re-assert speaker
|
|
// #5: unmuting publishes the mic track — (re)attach the transcriber's renderer to it if transcript is on.
|
|
if !action.isMuted { self?.transcriber.refresh(track: self?.localAudioTrack()) }
|
|
}
|
|
notifyListeners("setMuted", data: ["callUUID": action.callUUID.uuidString, "muted": action.isMuted])
|
|
action.fulfill()
|
|
}
|
|
|
|
// CallKit owns the AVAudioSession lifecycle. Since LiveKit auto-config is OFF, configure the session HERE —
|
|
// when CallKit activates it — and enable LiveKit's audio engine. This is the fix for the intermittent
|
|
// dead mic / no-audio and late speaker routing. Don't call setActive(true): CallKit already activated it.
|
|
public func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) {
|
|
do {
|
|
// Route to the LOUDSPEAKER by default (earpiece was the .voiceChat default). .videoChat +
|
|
// .defaultToSpeaker prefers the speaker while still letting wired/Bluetooth headsets win.
|
|
try audioSession.setCategory(.playAndRecord, mode: .videoChat,
|
|
options: [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP])
|
|
try AudioManager.shared.setEngineAvailability(.default)
|
|
preferSpeaker() // belt-and-braces: if we still landed on the earpiece, force the speaker
|
|
let route = audioSession.currentRoute.outputs.first?.portType.rawValue ?? "none"
|
|
notifyListeners("audioActivated", data: ["ok": true, "route": route])
|
|
} catch {
|
|
notifyListeners("audioActivated", data: ["ok": false, "error": String(describing: error)])
|
|
}
|
|
}
|
|
public func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) {
|
|
try? AudioManager.shared.setEngineAvailability(.none)
|
|
notifyListeners("audioDeactivated", data: ["ok": true])
|
|
}
|
|
}
|
|
|
|
// A tile view = a container holding a LiveKit VideoView. We CANNOT subclass VideoView (it's `public`, not
|
|
// `open`, so subclassing outside its module is illegal), so we compose. Under hole-punch the video sits BEHIND
|
|
// the (transparent) WebView, so touches never reach it — pinch-zoom is captured by the web and forwarded via
|
|
// applyZoom(). The container stays frame-synced to the web tile rect; the zoom transform lives on the inner
|
|
// video, so the two never fight.
|
|
final class TileVideoView: UIView {
|
|
let video = VideoView()
|
|
|
|
// Forward the two properties the plugin sets so call sites read like a VideoView.
|
|
var track: VideoTrack? { get { video.track } set { video.track = newValue } }
|
|
var layoutMode: VideoView.LayoutMode { get { video.layoutMode } set { video.layoutMode = newValue } }
|
|
|
|
override init(frame: CGRect) {
|
|
super.init(frame: frame)
|
|
backgroundColor = .clear // the web tile draws its own frame; gaps show the WebView's black bg
|
|
clipsToBounds = true
|
|
layer.cornerRadius = 12 // match .meet-tile's border-radius so corners don't poke past the web border
|
|
video.layoutMode = .fill
|
|
addSubview(video)
|
|
}
|
|
required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
|
|
|
|
// Position via bounds+center (not frame) so it coexists with the zoom transform.
|
|
override func layoutSubviews() {
|
|
super.layoutSubviews()
|
|
video.bounds = CGRect(origin: .zero, size: bounds.size)
|
|
video.center = CGPoint(x: bounds.midX, y: bounds.midY)
|
|
}
|
|
|
|
// Web-forwarded zoom (scale + translation in points). scale<=1 clears the transform.
|
|
func applyZoom(scale: CGFloat, tx: CGFloat, ty: CGFloat) {
|
|
if scale <= 1.001 { if !video.transform.isIdentity { video.transform = .identity } }
|
|
else { video.transform = CGAffineTransform(translationX: tx, y: ty).scaledBy(x: scale, y: scale) }
|
|
}
|
|
}
|
|
|
|
// #5 Live transcript on iOS. WKWebView has no Web Speech API, so an iOS participant's speech was never captured
|
|
// into the meeting transcript (desktop Chrome/Edge already works). This transcribes THIS device's mic with
|
|
// SFSpeechRecognizer, fed by a LiveKit `AudioRenderer` attached to the local mic track — so it reuses the call's
|
|
// already-open capture (no second AVAudioEngine fighting WebRTC for the audio session). Each finished utterance
|
|
// fires `onFinal`; the plugin relays it to the web, which sends it over the meeting WS exactly like the desktop
|
|
// path. Segments are cut on a short silence gap (and on the recognizer's own isFinal), and the request is
|
|
// restarted per segment so text flows continuously and stays within the recognizer's limits. On-device
|
|
// recognition is used when available (offline, continuous, no ~1-minute cap). All state is main-confined except
|
|
// the locked `request` that the audio-thread renderer appends to.
|
|
final class SpeechTranscriber: NSObject, AudioRenderer, @unchecked Sendable {
|
|
var onFinal: ((String) -> Void)?
|
|
|
|
private let recognizer = SFSpeechRecognizer(locale: Locale(identifier: "en-US"))
|
|
private let lock = NSLock()
|
|
private var request: SFSpeechAudioBufferRecognitionRequest?
|
|
private var task: SFSpeechRecognitionTask?
|
|
private weak var track: LocalAudioTrack?
|
|
private var running = false
|
|
private var latest = ""
|
|
private var lastEmitted = ""
|
|
private var silenceTimer: DispatchWorkItem?
|
|
|
|
// Begin transcription tapping a LiveKit local mic track (native calls). Idempotent; asks for Speech
|
|
// authorization once. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute).
|
|
func start(track: LocalAudioTrack?) { begin { self.attach(track) } }
|
|
|
|
// Begin transcription in EXTERNAL mode (scheduled/web meetings): no LiveKit track to tap — PCM arrives via
|
|
// appendPCM() from the web. Same recognition pipeline.
|
|
func startExternal() { begin { } }
|
|
|
|
private func begin(_ afterStart: @escaping () -> Void) {
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self else { return }
|
|
if self.running { afterStart(); return }
|
|
SFSpeechRecognizer.requestAuthorization { status in
|
|
DispatchQueue.main.async {
|
|
guard status == .authorized, !self.running else { return }
|
|
self.running = true
|
|
self.beginRequest()
|
|
afterStart()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Web-forwarded PCM (external mode): little-endian Int16 mono → Float32 buffer → recognizer.
|
|
func appendPCM(int16 data: Data, sampleRate: Double) {
|
|
let count = data.count / 2
|
|
guard count > 0,
|
|
let fmt = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false),
|
|
let buf = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: AVAudioFrameCount(count)) else { return }
|
|
buf.frameLength = AVAudioFrameCount(count)
|
|
guard let dst = buf.floatChannelData?[0] else { return }
|
|
data.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in
|
|
let src = raw.bindMemory(to: Int16.self)
|
|
for i in 0..<count { dst[i] = Float(src[i]) / 32768.0 }
|
|
}
|
|
lock.lock(); let r = request; lock.unlock()
|
|
r?.append(buf)
|
|
}
|
|
|
|
// (Re)attach to the current local mic track — called on start and whenever the mic (re)publishes on unmute.
|
|
func refresh(track: LocalAudioTrack?) { DispatchQueue.main.async { [weak self] in self?.attach(track) } }
|
|
|
|
func stop() {
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self else { return }
|
|
self.running = false
|
|
self.silenceTimer?.cancel(); self.silenceTimer = nil
|
|
self.track?.remove(audioRenderer: self); self.track = nil
|
|
self.lock.lock(); let r = self.request; self.request = nil; self.lock.unlock()
|
|
r?.endAudio(); self.task?.cancel(); self.task = nil
|
|
}
|
|
}
|
|
|
|
private func attach(_ t: LocalAudioTrack?) {
|
|
guard running, let t = t, track !== t else { return }
|
|
track?.remove(audioRenderer: self)
|
|
t.add(audioRenderer: self)
|
|
track = t
|
|
}
|
|
|
|
private func beginRequest() {
|
|
guard let recognizer = recognizer, recognizer.isAvailable else { return }
|
|
let req = SFSpeechAudioBufferRecognitionRequest()
|
|
req.shouldReportPartialResults = true // stream results; we only EMIT a segment on silence / isFinal
|
|
if recognizer.supportsOnDeviceRecognition { req.requiresOnDeviceRecognition = true }
|
|
lock.lock(); request = req; lock.unlock()
|
|
latest = ""; lastEmitted = ""
|
|
task = recognizer.recognitionTask(with: req) { [weak self] result, error in
|
|
guard let self = self else { return }
|
|
if let result = result {
|
|
let text = result.bestTranscription.formattedString
|
|
let isFinal = result.isFinal
|
|
DispatchQueue.main.async {
|
|
guard self.running else { return }
|
|
self.latest = text
|
|
if isFinal { self.flushAndRestart() } else { self.armSilenceTimer() }
|
|
}
|
|
} else if error != nil {
|
|
DispatchQueue.main.async { if self.running { self.flushAndRestart() } }
|
|
}
|
|
}
|
|
}
|
|
|
|
// A short pause = end of an utterance → emit it and start a fresh request for the next one.
|
|
private func armSilenceTimer() {
|
|
silenceTimer?.cancel()
|
|
let work = DispatchWorkItem { [weak self] in guard let self = self, self.running else { return }; self.flushAndRestart() }
|
|
silenceTimer = work
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 1.4, execute: work)
|
|
}
|
|
|
|
private func flushAndRestart() {
|
|
silenceTimer?.cancel(); silenceTimer = nil
|
|
let text = latest.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if !text.isEmpty && text != lastEmitted { lastEmitted = text; onFinal?(text) }
|
|
guard running else { return }
|
|
lock.lock(); let r = request; request = nil; lock.unlock()
|
|
r?.endAudio(); task?.cancel(); task = nil
|
|
beginRequest()
|
|
}
|
|
|
|
// MARK: AudioRenderer — receives the local mic PCM from LiveKit (audio thread).
|
|
func render(pcmBuffer: AVAudioPCMBuffer) {
|
|
lock.lock(); let r = request; lock.unlock()
|
|
r?.append(pcmBuffer)
|
|
}
|
|
}
|