From 68ff3a878b583c7e8ab4f18d76ee23ee628d8e2c Mon Sep 17 00:00:00 2001 From: sravan Date: Fri, 14 Aug 2026 16:36:18 +0530 Subject: [PATCH] #12 multi-device tiles + #5 native iOS transcript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #12 — same user on two devices now shows as two independent tiles (was: LiveKit kicked the older connection, "audio jumps to whichever joined last"): - LiveKit identity is now the per-connection mesh peerId, not the user id. /api/meetings/token + guest-token mint identity=peerId when the client supplies it (anti-hijack: never mint another live user's peerId). Web maps SFU tracks by identity==peerId, keeping peerIdForUid as a fallback for the transition/native. - Mesh dedup (dropDupPeers) now keys on a stable per-device clientId (persisted, sent on meeting-join, echoed by the server) instead of user id — so two real devices keep separate tiles while a same-device reconnect ghost still collapses. Verified in a real browser: 2 devices -> 2 tiles; same-device reconnect -> 1. - Native: plugin gains reconnectRoom(); after the native WebView joins the mesh it re-homes the LiveKit media onto its peerId identity. syncVideoTiles keys by peerId. Token-identity + anti-hijack + clientId echo verified by a server test. #5 — iOS live transcript (WKWebView has no Web Speech API, so an iOS participant was never transcribed; desktop already works): - native-call plugin transcribes the local mic with SFSpeechRecognizer, fed by a LiveKit AudioRenderer on the local mic track (reuses the call's open mic — no 2nd AVAudioEngine). Finalized segments -> 'transcript' event -> web sends meeting-transcript (same server assembly as desktop). startSR/stopSR use the native recognizer on native calls; Web Speech API path unchanged elsewhere. - NSSpeechRecognitionUsageDescription added to the iOS Info.plist. Native pieces (#12 reconnect, #5 transcript) need a Codemagic build; the web+server half is verified and deploys now (already fixes the reported laptop+phone case). Co-Authored-By: Claude Opus 4.8 --- .../NativeCallPlugin/NativeCallPlugin.swift | 160 ++++++++++++++++++ mobile/scripts/ios-patch.sh | 2 + server/public/home.html | 89 +++++++--- server/routes.js | 37 +++- server/signaling.js | 7 +- 5 files changed, 260 insertions(+), 35 deletions(-) diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index 72b65c2..46fc41a 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -5,6 +5,7 @@ import Capacitor import PushKit import CallKit import AVFoundation +import Speech // #5 native transcript: SFSpeechRecognizer (iOS on-device speech-to-text; WKWebView has no Web Speech API) import LiveKit // SPM product name is 'LiveKit' (Package.swift). (The old CocoaPods module was 'LiveKitClient'.) // Native calling for Biz Connect (iOS). Registered by `cap sync` as window.Capacitor.Plugins.NativeCall. @@ -27,6 +28,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "getToken", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "reportOutgoingCall", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "reportIncomingCall", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "reconnectRoom", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise), @@ -35,6 +37,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "setHolePunch", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setTileZoom", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "startTranscription", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "stopTranscription", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] @@ -56,6 +60,9 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega // on top. Saved so we can restore the WebView when the call ends. private var holePunchOn = false private var savedVCBg: UIColor? // the WebView parent's original background, restored when the call ends + // #5 native transcript: transcribes THIS device's mic (WKWebView has no Web Speech API). Fed by a LiveKit + // AudioRenderer on the local mic track, so it reuses the call's already-open mic (no 2nd audio engine). + private let transcriber = SpeechTranscriber() override public func load() { let config = CXProviderConfiguration(localizedName: "Biz Connect") @@ -89,6 +96,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega // (with shouldPublishTrack=true, the default) auto-publishes/unpublishes the screen-share track. BroadcastManager.shared.delegate = self + // #5 native transcript: each finalized speech segment → the web (meeting-transcript over the WS), which + // merges it into the shared meeting transcript (same path as desktop's Web Speech API). + transcriber.onFinal = { [weak self] text in self?.notifyListeners("transcript", data: ["text": text]) } + // Make the WebView transparent at STARTUP. WKWebView can IGNORE isOpaque=false when it's flipped after // the page has already rendered — the likely reason the hole-punch showed no video. Doing it once, up // front, makes the transparent-meeting areas actually reveal the native video behind the WebView. The @@ -141,6 +152,7 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega private func disconnectRoom() { let r = room room = nil + transcriber.stop() // #5: end any live transcription with the call Task { await r?.disconnect() } removeAllTileViews() DispatchQueue.main.async { [weak self] in guard let self = self, let web = self.bridge?.webView else { return }; self.applyHolePunchRestore(web) } @@ -303,6 +315,22 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } + // #12 Multi-device: swap the media connection to a token whose identity = this device's mesh peerId (unique + // per connection). The web calls this once the native WebView has joined the mesh and has a peerId. On + // answer we connected INSTANTLY with the push token (identity=userId) for zero-latency audio; this re-homes + // the media onto the unique peerId identity so two devices of the same user are distinct LiveKit + // participants (LiveKit allows one connection per identity — otherwise the older device is kicked and + // "audio jumps to whichever joined last"). connectRoom disconnects the old room first; the CallKit call and + // its audio session stay active, so audio just re-attaches. callConnected fires on success → web re-applies + // mic/cam. Guarded to only run during an active call. + @objc func reconnectRoom(_ call: CAPPluginCall) { + let url = call.getString("url") ?? "" + let token = call.getString("token") ?? "" + guard room != nil, !url.isEmpty, !token.isEmpty else { call.resolve(); return } + connectRoom(url: url, token: token) + call.resolve() + } + @objc func setMuted(_ call: CAPPluginCall) { let muted = call.getBool("muted") ?? false // Drive mute THROUGH CallKit so the system call screen and the in-app meeting UI stay in sync (the @@ -362,6 +390,28 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega call.resolve() } + // MARK: - #5 Live transcript (native SFSpeechRecognizer) + + // The current published local mic track — the source we tap for transcription. nil until the user has + // unmuted at least once (the mic is published on unmute), or between reconnects. + private func localAudioTrack() -> LocalAudioTrack? { + return room?.localParticipant.audioTracks.first?.track as? LocalAudioTrack + } + + // Start transcribing this device's mic. The web calls this when the room's transcript is active and we're + // in a native call (WKWebView has no Web Speech API). Attaches a LiveKit AudioRenderer to the local mic + // track and runs SFSpeechRecognizer; finalized segments fire the "transcript" event. Safe to call when the + // mic track doesn't exist yet — we re-attach on unmute (setMicrophone-enable) too. + @objc func startTranscription(_ call: CAPPluginCall) { + transcriber.start(track: localAudioTrack()) + call.resolve() + } + + @objc func stopTranscription(_ call: CAPPluginCall) { + transcriber.stop() + call.resolve() + } + // MARK: - BroadcastManagerDelegate public func broadcastManager(didChangeState isBroadcasting: Bool) { @@ -529,6 +579,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega Task { [weak self] in try? await r?.localParticipant.setMicrophone(enabled: !action.isMuted) self?.preferSpeaker() // toggling the mic can flip the route back to the earpiece — re-assert speaker + // #5: unmuting publishes the mic track — (re)attach the transcriber's renderer to it if transcript is on. + if !action.isMuted { self?.transcriber.refresh(track: self?.localAudioTrack()) } } notifyListeners("setMuted", data: ["callUUID": action.callUUID.uuidString, "muted": action.isMuted]) action.fulfill() @@ -592,3 +644,111 @@ final class TileVideoView: UIView { else { video.transform = CGAffineTransform(translationX: tx, y: ty).scaledBy(x: scale, y: scale) } } } + +// #5 Live transcript on iOS. WKWebView has no Web Speech API, so an iOS participant's speech was never captured +// into the meeting transcript (desktop Chrome/Edge already works). This transcribes THIS device's mic with +// SFSpeechRecognizer, fed by a LiveKit `AudioRenderer` attached to the local mic track — so it reuses the call's +// already-open capture (no second AVAudioEngine fighting WebRTC for the audio session). Each finished utterance +// fires `onFinal`; the plugin relays it to the web, which sends it over the meeting WS exactly like the desktop +// path. Segments are cut on a short silence gap (and on the recognizer's own isFinal), and the request is +// restarted per segment so text flows continuously and stays within the recognizer's limits. On-device +// recognition is used when available (offline, continuous, no ~1-minute cap). All state is main-confined except +// the locked `request` that the audio-thread renderer appends to. +final class SpeechTranscriber: NSObject, AudioRenderer, @unchecked Sendable { + var onFinal: ((String) -> Void)? + + private let recognizer = SFSpeechRecognizer(locale: Locale(identifier: "en-US")) + private let lock = NSLock() + private var request: SFSpeechAudioBufferRecognitionRequest? + private var task: SFSpeechRecognitionTask? + private weak var track: LocalAudioTrack? + private var running = false + private var latest = "" + private var lastEmitted = "" + private var silenceTimer: DispatchWorkItem? + + // Begin transcription (idempotent). Asks for Speech authorization once, then starts recognition and taps the + // current mic track. Safe to call before the mic exists — `refresh` re-attaches when it publishes (unmute). + func start(track: LocalAudioTrack?) { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + if self.running { self.attach(track); return } + SFSpeechRecognizer.requestAuthorization { status in + DispatchQueue.main.async { + guard status == .authorized, !self.running else { return } + self.running = true + self.beginRequest() + self.attach(track) + } + } + } + } + + // (Re)attach to the current local mic track — called on start and whenever the mic (re)publishes on unmute. + func refresh(track: LocalAudioTrack?) { DispatchQueue.main.async { [weak self] in self?.attach(track) } } + + func stop() { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + self.running = false + self.silenceTimer?.cancel(); self.silenceTimer = nil + self.track?.remove(audioRenderer: self); self.track = nil + self.lock.lock(); let r = self.request; self.request = nil; self.lock.unlock() + r?.endAudio(); self.task?.cancel(); self.task = nil + } + } + + private func attach(_ t: LocalAudioTrack?) { + guard running, let t = t, track !== t else { return } + track?.remove(audioRenderer: self) + t.add(audioRenderer: self) + track = t + } + + private func beginRequest() { + guard let recognizer = recognizer, recognizer.isAvailable else { return } + let req = SFSpeechAudioBufferRecognitionRequest() + req.shouldReportPartialResults = true // stream results; we only EMIT a segment on silence / isFinal + if recognizer.supportsOnDeviceRecognition { req.requiresOnDeviceRecognition = true } + lock.lock(); request = req; lock.unlock() + latest = ""; lastEmitted = "" + task = recognizer.recognitionTask(with: req) { [weak self] result, error in + guard let self = self else { return } + if let result = result { + let text = result.bestTranscription.formattedString + let isFinal = result.isFinal + DispatchQueue.main.async { + guard self.running else { return } + self.latest = text + if isFinal { self.flushAndRestart() } else { self.armSilenceTimer() } + } + } else if error != nil { + DispatchQueue.main.async { if self.running { self.flushAndRestart() } } + } + } + } + + // A short pause = end of an utterance → emit it and start a fresh request for the next one. + private func armSilenceTimer() { + silenceTimer?.cancel() + let work = DispatchWorkItem { [weak self] in guard let self = self, self.running else { return }; self.flushAndRestart() } + silenceTimer = work + DispatchQueue.main.asyncAfter(deadline: .now() + 1.4, execute: work) + } + + private func flushAndRestart() { + silenceTimer?.cancel(); silenceTimer = nil + let text = latest.trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty && text != lastEmitted { lastEmitted = text; onFinal?(text) } + guard running else { return } + lock.lock(); let r = request; request = nil; lock.unlock() + r?.endAudio(); task?.cancel(); task = nil + beginRequest() + } + + // MARK: AudioRenderer — receives the local mic PCM from LiveKit (audio thread). + func render(pcmBuffer: AVAudioPCMBuffer) { + lock.lock(); let r = request; lock.unlock() + r?.append(pcmBuffer) + } +} diff --git a/mobile/scripts/ios-patch.sh b/mobile/scripts/ios-patch.sh index f89006a..e31bd16 100644 --- a/mobile/scripts/ios-patch.sh +++ b/mobile/scripts/ios-patch.sh @@ -17,6 +17,7 @@ set_str NSCameraUsageDescription "Biz Connect uses the camera for video cal set_str NSMicrophoneUsageDescription "Biz Connect uses the microphone for voice and video calls." set_str NSPhotoLibraryUsageDescription "Biz Connect needs photo access so you can send images in chat." set_str NSPhotoLibraryAddUsageDescription "Biz Connect saves images and recordings you download to your photos." +set_str NSSpeechRecognitionUsageDescription "Biz Connect uses speech recognition to create live meeting transcripts from your microphone." # Human-readable display name on the home screen. set_str CFBundleDisplayName "Biz Connect" @@ -140,3 +141,4 @@ fi echo "Info.plist patched:" "$PB" -c "Print :NSCameraUsageDescription" "$PLIST" "$PB" -c "Print :NSMicrophoneUsageDescription" "$PLIST" +"$PB" -c "Print :NSSpeechRecognitionUsageDescription" "$PLIST" diff --git a/server/public/home.html b/server/public/home.html index 2f96232..1183bd9 100644 --- a/server/public/home.html +++ b/server/public/home.html @@ -4173,7 +4173,7 @@ async function setupNativeCall(){ }catch(e){ console.warn('[callkit] answerCall failed:', e); } }); // Enforce the muted-by-default rule against the plugin's actual mic once the room connects (belt-and-braces // for builds whose plugin still connects the mic live). meetMic is the source of truth for the mic button. - NC.addListener('callConnected', ()=>{ if(meetNative){ try{ NC.setMuted({ muted:!meetMic }); }catch(_){} } }); + NC.addListener('callConnected', ()=>{ if(meetNative){ try{ NC.setMuted({ muted:!meetMic }); }catch(_){} if(meetCam){ try{ NC.setCamera({ on:true }); }catch(_){} } } }); // re-applies mic/cam after a #12 media reconnect too NC.addListener('callError', (e)=>{ console.warn('[callkit] call error:', (e&&e.error)||''); }); // Muted from the CallKit system UI → reflect it in the meeting window's mic button. NC.addListener('setMuted', (e)=>{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} }); @@ -4199,6 +4199,19 @@ async function callkitReportOutgoing(uuid, room, kind, peerName, hasVideo){ let tk={}; try{ tk=await postJSON('/api/meetings/token',{ room }); }catch(_){} try{ NC.reportOutgoingCall({ callUUID:uuid, room, kind:kind||'dm', peerName:peerName||'Call', hasVideo:!!hasVideo, url:tk.url||'', token:tk.token||'' }); }catch(_){} } +// #12: once the native WebView has joined the mesh (so it owns a per-connection peerId), reconnect the plugin's +// LiveKit media with a token whose identity = that peerId. The plugin connected INSTANTLY on answer using the +// push token (identity=userId) for zero-latency audio; this swaps it to the unique peerId identity a beat later +// so two devices of the same user no longer collide (LiveKit = one connection per identity). callConnected +// re-fires after the reconnect, which re-applies mic/cam. No-op on older plugin builds (no reconnectRoom) — they +// stay on the userId identity, which is still correct for a single device. +async function bzNativeReconnectMedia(){ + if(!meetNative || !meetMyId) return; + const NC=nativeCallPlugin(); if(!NC || !NC.reconnectRoom) return; + let tk={}; try{ tk=await postJSON('/api/meetings/token',{ room:meetRoom, peerId:meetMyId }); }catch(_){ return; } + if(!tk || !tk.token) return; + try{ const p=NC.reconnectRoom({ url:tk.url||'', token:tk.token }); if(p&&p.catch) p.catch(()=>{}); }catch(_){} +} // End the CallKit call (remote hung up / we left / call ended). Safe no-op off-CallKit. function callkitEnd(uuid){ const NC=nativeCallPlugin(); if(!NC||!uuid) return; try{ NC.endCall({ callUUID:uuid }); }catch(_){} } // Ring CallKit from a WebSocket call event — a 2nd, reliable path alongside the VoIP push for when the app is @@ -4823,6 +4836,7 @@ const meetMuted=new Map(); // peerId|'__local' -> muted (for the tile mic-off const meetNames=new Map(); // peerId -> name (peers that arrive before their offer) const meetAvatars=new Map(); // peerId -> avatar URL (for participant-tile profile pics) const meetPeerUids=new Map();// peerId -> user id (to tell when an invitee has joined) +const meetPeerClients=new Map();// peerId -> stable per-device clientId (#12: dedup a reconnecting DEVICE without collapsing a genuine 2nd device of the same user) const meetCamOff=new Map(); // peerId -> camera off? (a disabled remote track still arrives, so show the avatar) const meetInvited=new Map(); // user id -> {name, timer} for invitees who haven't joined yet (#6) let meetReturn=null; // {kind:'dm'|'group', id} — chat to land on when the call ends (null = meetings tab) @@ -4837,9 +4851,13 @@ let meetMultiShare=false; // host setting: allow several people to share at let meetRec=null; // active composite recording (host) {rec, stop()} let meetTranscribe=false; // am I subscribed to a transcript copy let meetRoomTx=false; // is the room transcription active (≥1 subscriber → all mics transcribe) -let meetSR=null; // my SpeechRecognition instance +let meetSR=null; // my SpeechRecognition instance ('native' sentinel when the iOS plugin transcribes) +let _meetTxListener=null; // #5: native 'transcript' event listener handle (iOS native calls) let meetStageId=null; // which shared screen is currently on the stage (peerId|'__local') function meetSend(o){ try{ if(meetWs && meetWs.readyState===1) meetWs.send(JSON.stringify(o)); }catch(_){} } +// #12: a stable per-DEVICE id (persisted). Sent on meeting-join so the mesh can tell a reconnecting device +// (evict its ghost peer) apart from a genuine SECOND device of the same user (keep both tiles). +function bzDeviceId(){ try{ let id=localStorage.getItem('bzc_device_id'); if(!id){ id=(window.crypto&&crypto.randomUUID)?crypto.randomUUID():('dev-'+Date.now().toString(36)+Math.random().toString(36).slice(2,10)); localStorage.setItem('bzc_device_id', id); } return id; }catch(_){ return 'dev-'+Math.random().toString(36).slice(2,12); } } // ================= LiveKit SFU media plane (feature-flagged) ================= // When the server reports sfu:true, meeting MEDIA flows through LiveKit instead of the P2P mesh: @@ -4855,8 +4873,8 @@ async function sfuConnect(){ const LK=await sfuLoadLib(); SFU.lib=LK; // Guests (external link joiners, not signed in) get an unauthenticated guest token for this room. const tk=(ME&&ME.guest) - ? await postJSON('/api/meetings/guest-token',{ room:meetRoom, name:ME.name, identity:ME.id }) // identity must match the guestId sent over signaling - : await postJSON('/api/meetings/token',{ room:meetRoom }); // per-user, per-room join credential + ? await postJSON('/api/meetings/guest-token',{ room:meetRoom, name:ME.name, identity:ME.id, peerId:meetMyId }) // identity=peerId (#12 multi-device); guestId kept for back-compat + : await postJSON('/api/meetings/token',{ room:meetRoom, peerId:meetMyId }); // #12: identity = this connection's mesh peerId, so two devices of one user don't collide on LiveKit // adaptiveStream + dynacast BOTH off: we attach media via manual srcObject (not track.attach), so // LiveKit can't observe tile visibility. With dynacast on, a screen-share layer was paused unless a // camera track was also flowing → the screen showed blank / only when the camera was on, and was @@ -4878,7 +4896,10 @@ function sfuRebuild(pid){ addTile(pid, st, meetNames.get(pid)||'Guest', false); setTileScreen(pid, !!p.screen); meetWatchStream(pid, st); } function sfuAttach(pub, track, participant, _try){ - let pid=peerIdForUid(participant.identity); + // #12: LiveKit identity is now the mesh peerId (unique per connection), so a track maps straight to its + // tile. peerIdForUid is kept as a fallback for any participant still on the old identity=userId scheme + // (e.g. a native device on an older plugin build, before it reconnects with its peerId token). + let pid = meetPeers.has(participant.identity) ? participant.identity : peerIdForUid(participant.identity); if(!pid){ // uid→peerId map (from the mesh join) may lag the LiveKit track — retry a few times… if((_try||0)<6){ setTimeout(()=>sfuAttach(pub,track,participant,(_try||0)+1), 400); return; } // …then fall back: a NATIVE (CallKit + LiveKit) participant joins LiveKit but NOT our WS mesh, so it has @@ -4900,7 +4921,7 @@ function sfuAttach(pub, track, participant, _try){ sfuRebuild(pid); updateShareMode(); } function sfuDetach(pub, track, participant){ - let pid=peerIdForUid(participant.identity); if(!pid) pid='lk:'+participant.identity; const p=SFU.peers.get(pid); if(!p) return; const mt=track.mediaStreamTrack; + let pid = meetPeers.has(participant.identity) ? participant.identity : (peerIdForUid(participant.identity) || ('lk:'+participant.identity)); const p=SFU.peers.get(pid); if(!p) return; const mt=track.mediaStreamTrack; if(p.audio===mt) p.audio=null; else if(p.screen===mt){ p.screen=null; meetSharers.delete(pid); setTileScreen(pid,false); } else if(p.cam===mt) p.cam=null; sfuRebuild(pid); updateShareMode(); } @@ -5586,16 +5607,16 @@ function meetWatchStream(id, stream){ } function meetUnwatch(id){ const r=meetVU.get(id); if(!r) return; try{cancelAnimationFrame(r.raf);}catch(_){} try{r.src.disconnect();}catch(_){} try{r.ctx.close();}catch(_){} meetVU.delete(id); const t=document.getElementById('meet-tile-'+id); if(t) t.classList.remove('speaking'); } function meetUnwatchAll(){ for(const id of Array.from(meetVU.keys())) meetUnwatch(id); } -// #5: the same person showing up as several tiles. Each browser tab / reload / reconnect gets a NEW -// peerId, so a stale session left a ghost tile behind. One person (uid) = one tile: when a uid reappears -// on a new peerId, drop their older peer entirely. -function dropDupPeers(uid, keepPid){ - if(!uid) return; - for(const [pid,u] of [...meetPeerUids]){ - if(u!==uid || pid===keepPid) continue; +// A stale ghost tile: the SAME DEVICE reconnected (reload / dropped WS) on a new peerId before its old +// peer-left arrived. Dedup by the stable per-device clientId — NOT by user id — so a genuine SECOND device +// of the same person (#12 multi-device) keeps its own independent tile instead of being collapsed. +function dropDupPeers(clientId, keepPid){ + if(!clientId) return; // no device id (older client / guest race) → don't collapse; peer-left will clean up + for(const [pid,c] of [...meetPeerClients]){ + if(c!==clientId || pid===keepPid) continue; const p=meetPeers.get(pid); if(p){ try{ p.pc&&p.pc.close(); }catch(_){} meetPeers.delete(pid); } if(SFU.on){ try{ sfuDropPeer(pid); }catch(_){} } - meetSharers.delete(pid); meetPeerUids.delete(pid); meetNames.delete(pid); meetAvatars.delete(pid); + meetSharers.delete(pid); meetPeerUids.delete(pid); meetPeerClients.delete(pid); meetNames.delete(pid); meetAvatars.delete(pid); removeTile(pid); } } @@ -5756,14 +5777,27 @@ async function uploadRecording(blob, durMs){ // names); each subscriber gets a private copy. Unsubscribing only drops YOUR copy, not others'. function toggleTranscribe(){ meetTranscribe=!meetTranscribe; meetSend({type:'meeting-transcribe', on:meetTranscribe}); updateTransBtn(); toast(meetTranscribe?'Transcript on — your private copy is saved to Past meetings after the call':'You left the transcript — your copy is being saved'); } function applyRoomTx(active){ if(active===meetRoomTx) return; meetRoomTx=active; if(active) startSR(); else stopSR(); transcribeNotice(active); } -function startSR(){ if(meetSR) return; const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR){ if(meetTranscribe) toast('Live transcript needs Chrome or Edge'); return; } +function startSR(){ if(meetSR) return; + // #5: on an iOS NATIVE call the WKWebView has no Web Speech API — transcribe via the native plugin + // (SFSpeechRecognizer tapping the call's mic). Its 'transcript' events feed the same meeting-transcript path. + if(meetNative){ const NC=nativeCallPlugin(); if(NC && NC.startTranscription){ + meetSR='native'; + try{ _meetTxListener=NC.addListener('transcript', (e)=>{ const text=((e&&e.text)||'').trim(); if(text) meetSend({type:'meeting-transcript', text}); }); }catch(_){} + try{ const p=NC.startTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} + return; + }} + const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR){ if(meetTranscribe) toast('Live transcript needs Chrome or Edge'); return; } try{ meetSR=new SR(); }catch(_){ return; } meetSR.continuous=true; meetSR.interimResults=false; meetSR.lang='en-US'; meetSR.onresult=(e)=>{ for(let i=e.resultIndex;i{}; meetSR.onend=()=>{ if(meetRoomTx){ try{ meetSR.start(); }catch(_){} } }; try{ meetSR.start(); }catch(_){} } -function stopSR(){ if(meetSR){ try{ meetSR.onend=null; meetSR.stop(); }catch(_){} meetSR=null; } } +function _removeTxListener(){ const h=_meetTxListener; _meetTxListener=null; if(!h) return; try{ if(h.remove) h.remove(); else if(h.then) h.then(x=>{ try{ x&&x.remove&&x.remove(); }catch(_){} }); }catch(_){} } +function stopSR(){ + if(meetSR==='native'){ meetSR=null; _removeTxListener(); const NC=nativeCallPlugin(); if(NC&&NC.stopTranscription){ try{ const p=NC.stopTranscription(); if(p&&p.catch) p.catch(()=>{}); }catch(_){} } return; } + if(meetSR){ try{ meetSR.onend=null; meetSR.stop(); }catch(_){} meetSR=null; } +} function updateTransBtn(){ const b=document.getElementById('meetTransBtn'); if(!b) return; b.classList.toggle('on', meetTranscribe); b.title=meetTranscribe?'Stop my transcript':'Transcribe (your private copy)'; } function transcribeNotice(on){ let el=document.getElementById('txNotice'); if(on){ if(!el){ el=document.createElement('div'); el.id='txNotice'; el.className='tx-notice'; el.innerHTML=ic('fileText',12)+' Transcribing'; document.body.appendChild(el); } } else if(el){ el.remove(); } } // iOS blocks media/WebRTC audio playback until a user gesture, so remote call audio stays SILENT until you @@ -5811,8 +5845,11 @@ function bzNativeSyncTiles(){ if(r.top < maxBottom && (r.top + h) > maxBottom) h = maxBottom - r.top; // legacy clamp if(h<2){ el.classList.remove('bz-hasvid'); return; } let uid, local=false, name='', muted=false, screen=false, camOn=false; - if(id==='__local'){ uid=(ME&&ME.id)||''; local=true; name=(ME&&ME.name)||'You'; muted=!meetMic; camOn=!!meetCam; } - else { uid=meetPeerUids.get(id)||''; name=meetNames.get(id)||'Guest'; muted=!!meetMuted.get(id); screen=meetSharers.has(id); camOn=(meetCamOff.get(id)!==true); } + // #12: the plugin resolves a remote LiveKit participant by this key, and identities are now the mesh + // peerId (per connection) — so pass the tile's peerId (`id`), not the user id. (__local uses the local + // participant directly, so its key is irrelevant.) + if(id==='__local'){ uid=meetMyId||(ME&&ME.id)||''; local=true; name=(ME&&ME.name)||'You'; muted=!meetMic; camOn=!!meetCam; } + else { uid=id; name=meetNames.get(id)||'Guest'; muted=!!meetMuted.get(id); screen=meetSharers.has(id); camOn=(meetCamOff.get(id)!==true); } if(!uid){ el.classList.remove('bz-hasvid'); return; } // hole-punch: on tiles that have native video, hide the web avatar/bg so the video (behind) shows through. el.classList.toggle('bz-hasvid', bzHolePunch && (screen || camOn)); @@ -5887,11 +5924,11 @@ async function enterMeeting(code, audioOnly, opts){ renderCallConnecting(); // branded "Connecting…" until the room is created/joined (esp. on slow links) meetWs=new WebSocket((location.protocol==='https:'?'wss://':'ws://')+location.host+'/ws'); meetWs.onmessage=onMeetMsg; - meetWs.onopen=()=>{ if(code){ meetRoom=code; renderCall(); meetSend({type:'meeting-join', room:code, name:(ME.name||ME.email||'Guest'), guestId:(ME&&ME.guest)?ME.id:undefined}); } else { meetSend({type:'meeting-create'}); } }; + meetWs.onopen=()=>{ if(code){ meetRoom=code; renderCall(); meetSend({type:'meeting-join', room:code, name:(ME.name||ME.email||'Guest'), guestId:(ME&&ME.guest)?ME.id:undefined, clientId:bzDeviceId()}); } else { meetSend({type:'meeting-create'}); } }; } async function onMeetMsg(e){ let m; try{ m=JSON.parse(e.data); }catch(_){ return; } - if(m.type==='meeting-created'){ meetRoom=m.room; renderCall(); meetSend({type:'meeting-join', room:m.room, name:(ME.name||ME.email||'Guest'), guestId:(ME&&ME.guest)?ME.id:undefined}); if(meetAnnounceGroup){ const g=meetAnnounceGroup; meetAnnounceGroup=null; try{ postJSON('/api/messages',{group:g, body:'📹 Started a group call — join with code '+m.room}); }catch(_){} } return; } + if(m.type==='meeting-created'){ meetRoom=m.room; renderCall(); meetSend({type:'meeting-join', room:m.room, name:(ME.name||ME.email||'Guest'), guestId:(ME&&ME.guest)?ME.id:undefined, clientId:bzDeviceId()}); if(meetAnnounceGroup){ const g=meetAnnounceGroup; meetAnnounceGroup=null; try{ postJSON('/api/messages',{group:g, body:'📹 Started a group call — join with code '+m.room}); }catch(_){} } return; } if(m.type==='meeting-joined'){ meetMyId=m.peerId; if(_inLobby){ _inLobby=false; renderCall(); } // admitted from the lobby → replace the "waiting…" screen with the call @@ -5899,13 +5936,15 @@ async function onMeetMsg(e){ meetWatchStream('__local', meetLocalStream); // active-speaker detection on my own mic // Existing peers OFFER to me (their offers carry their tracks incl. any active screen share); // I just set up the connections and wait. Avoids the "newcomer can't receive screen" bug. - for(const p of (m.peers||[])){ meetNames.set(p.peerId,p.name); if(p.avatar) meetAvatars.set(p.peerId,p.avatar); if(p.uid){ meetPeerUids.set(p.peerId,p.uid); meetInviteJoined(p.uid); dropDupPeers(p.uid, p.peerId); } meetMakePeer(p.peerId,p.name); } + for(const p of (m.peers||[])){ meetNames.set(p.peerId,p.name); if(p.avatar) meetAvatars.set(p.peerId,p.avatar); if(p.clientId) meetPeerClients.set(p.peerId,p.clientId); if(p.uid){ meetPeerUids.set(p.peerId,p.uid); meetInviteJoined(p.uid); } dropDupPeers(p.clientId, p.peerId); meetMakePeer(p.peerId,p.name); } if(_dmCallWaiting && !(m.peers&&m.peers.length)) addWaitingTile(_dmCallWaiting.name, _dmCallWaiting.avatar); // #7: show who we're calling while it rings // SFU: connect to LiveKit for media once (peer uid→peerId map is populated above). Mic/cam are // off at join, so nothing publishes yet — toggleMic/toggleCam publish on demand. if(SFU.on && !meetNative){ try{ await sfuConnect(); }catch(err){ if(ME&&ME.guest){ toast((err&&err.message)||'This meeting link has expired or isn’t active.'); leaveMeeting(true); return; } const e2=document.getElementById('meetErr'); if(e2) e2.textContent='Could not connect meeting media'; } } // native: the plugin already holds the LiveKit media (one connection per identity) — we joined the mesh - // for the UI only. Media is native; tell peers our mic is live. + // for the UI only. Now that we have our peerId, reconnect the plugin's media to a peerId-identity token + // (#12 multi-device) so this device is a unique LiveKit participant. + if(meetNative){ try{ bzNativeReconnectMedia(); }catch(_){} } meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); // tell existing peers my state if(meetIsHost) meetSend({type:'meeting-host', to:meetMyId}); // announce host so others know // #4a: now that we're actually in the call (post-admission, media connected), honor the mic/cam the @@ -5927,7 +5966,7 @@ async function onMeetMsg(e){ if(m.type==='meeting-peer-joined'){ stopRingback(); removeWaitingTile(); _dmCallWaiting=null; // #17/#7: they answered → stop ring + drop the ringing tile try{ playJoinChime(); toast((m.name||'Someone')+' joined the call'); }catch(_){} // #7: sound + a brief who-joined note - meetNames.set(m.peerId,m.name); if(m.avatar) meetAvatars.set(m.peerId,m.avatar); if(m.uid){ meetPeerUids.set(m.peerId,m.uid); meetInviteJoined(m.uid); dropDupPeers(m.uid, m.peerId); } + meetNames.set(m.peerId,m.name); if(m.avatar) meetAvatars.set(m.peerId,m.avatar); if(m.clientId) meetPeerClients.set(m.peerId,m.clientId); if(m.uid){ meetPeerUids.set(m.peerId,m.uid); meetInviteJoined(m.uid); } dropDupPeers(m.clientId, m.peerId); const pc=meetMakePeer(m.peerId,m.name); // I'm an existing peer → I OFFER to the newcomer (carries my screen) if(pc){ try{ const offer=await pc.createOffer(); await pc.setLocalDescription(offer); meetSend({type:'meeting-signal',to:m.peerId,data:{sdp:pc.localDescription}}); }catch(_){} } // (SFU: LiveKit handles media, no offer) meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); @@ -5942,7 +5981,7 @@ async function onMeetMsg(e){ if(m.type==='meeting-peer-screen'){ if(m.on) meetSharers.add(m.from); else meetSharers.delete(m.from); setTileScreen(m.from, !!m.on); refreshMeetPanel(); return; } if(m.type==='meeting-sharemode'){ meetMultiShare=!!m.multi; refreshMeetPanel(); return; } if(m.type==='meeting-muteall'){ if(meetMic && meetLocalStream){ meetMic=false; meetLocalStream.getAudioTracks().forEach(t=>t.enabled=false); updateMicBtn(); setTileMute('__local', true); meetSend({type:'meeting-state', muted:true, camOff:!meetCam}); } toast('You were muted by the host'); return; } - if(m.type==='meeting-peer-left'){ const p=meetPeers.get(m.peerId); if(p){ try{p.pc.close();}catch(_){} meetPeers.delete(m.peerId);} if(SFU.on) sfuDropPeer(m.peerId); meetSharers.delete(m.peerId); meetPeerUids.delete(m.peerId); meetNames.delete(m.peerId); meetAvatars.delete(m.peerId); removeTile(m.peerId); refreshMeetPanel(); return; } // #4b: drop uid/name too, else they stay in hereUids and never reappear under "Add people" + if(m.type==='meeting-peer-left'){ const p=meetPeers.get(m.peerId); if(p){ try{p.pc.close();}catch(_){} meetPeers.delete(m.peerId);} if(SFU.on) sfuDropPeer(m.peerId); meetSharers.delete(m.peerId); meetPeerUids.delete(m.peerId); meetPeerClients.delete(m.peerId); meetNames.delete(m.peerId); meetAvatars.delete(m.peerId); removeTile(m.peerId); refreshMeetPanel(); return; } // #4b: drop uid/name too, else they stay in hereUids and never reappear under "Add people" if(m.type==='meeting-signal'){ const from=m.from, d=m.data||{}; if(d.sdp){ @@ -6036,7 +6075,7 @@ function leaveMeeting(forced){ meetSend({type:'meeting-leave'}); if(SFU.on) sfuDisconnect(); // tear down the LiveKit room meetUnwatchAll(); meetSharers.clear(); - meetPeers.forEach(p=>{ try{p.pc.close();}catch(_){} }); meetPeers.clear(); meetNames.clear(); meetAvatars.clear(); meetPeerUids.clear(); meetCamOff.clear(); meetInvited.forEach(e=>{ if(e.timer) clearTimeout(e.timer); }); meetInvited.clear(); meetMuted.clear(); + meetPeers.forEach(p=>{ try{p.pc.close();}catch(_){} }); meetPeers.clear(); meetNames.clear(); meetAvatars.clear(); meetPeerUids.clear(); meetPeerClients.clear(); meetCamOff.clear(); meetInvited.forEach(e=>{ if(e.timer) clearTimeout(e.timer); }); meetInvited.clear(); meetMuted.clear(); if(meetLocalStream){ try{ meetLocalStream.getTracks().forEach(t=>t.stop()); }catch(_){} meetLocalStream=null; } if(meetWs){ try{ meetWs.close(); }catch(_){} meetWs=null; } meetRoom=null; meetMyId=null; meetState='idle'; meetIsHost=false; meetHostId=null; meetRailLive(false); resetMeetChat(); _inLobby=false; diff --git a/server/routes.js b/server/routes.js index 499d265..ecd07ca 100644 --- a/server/routes.js +++ b/server/routes.js @@ -1038,12 +1038,25 @@ route('POST', '/api/meetings/token', async (req, res) => { const u = await currentUser(req); if (!u) return json(res, 401, { error: 'unauthorized' }); if (!LIVEKIT_ENABLED) return json(res, 501, { error: 'sfu not configured' }); - const { room } = await readBody(req); - const rm = String(room || '').trim(); + const body = await readBody(req); + const rm = String(body.room || '').trim(); if (!/^[A-Za-z0-9._-]{4,64}$/.test(rm)) return json(res, 400, { error: 'invalid room' }); + // #12 Multi-device: mint the token with identity = the caller's mesh peerId (unique per connection) when it's + // supplied, so the SAME user joining from two devices no longer collides on LiveKit — which allows exactly + // one connection per identity, so with identity=userId the older device was kicked ("audio jumps to whichever + // joined last"). Falls back to the user id when no peerId is passed (e.g. a native OUTGOING token fetched + // before the WebView has joined the mesh; the plugin reconnects with a peerId token once it has one). The + // client got its peerId from `meeting-joined`. Anti-hijack: refuse a peerId that's a DIFFERENT live user's. + let identity = u.id; + const pid = (typeof body.peerId === 'string') ? body.peerId.trim() : ''; + if (/^[A-Za-z0-9_-]{4,64}$/.test(pid)) { + let ok = true; + try { const peers = require('./presence').meetingRooms.get(rm); const pr = peers && peers.get(pid); if (pr && pr.uid && pr.uid !== u.id) ok = false; } catch (_) {} + if (ok) identity = pid; + } const metadata = JSON.stringify({ avatarUrl: u.avatar_url || '' }); - const token = livekitToken(u.id, u.name || u.email, rm, metadata); - json(res, 200, { token, url: LIVEKIT_URL, identity: u.id, name: u.name || u.email }); + const token = livekitToken(identity, u.name || u.email, rm, metadata); + json(res, 200, { token, url: LIVEKIT_URL, identity, name: u.name || u.email }); }); // GUEST (no-login) LiveKit token — lets an external person invited by link join a meeting without a @@ -1051,7 +1064,8 @@ route('POST', '/api/meetings/token', async (req, res) => { // meeting, so a token can't be created for an arbitrary/expired code. Identity is a throwaway guest id. route('POST', '/api/meetings/guest-token', async (req, res) => { if (!LIVEKIT_ENABLED) return json(res, 501, { error: 'sfu not configured' }); - const { room, name, identity } = await readBody(req); + const body = await readBody(req); + const { room, name, identity } = body; const rm = String(room || '').trim(); if (!/^\d{6}$/.test(rm)) return json(res, 400, { error: 'invalid room' }); const live = (() => { try { return require('./presence').meetingRooms.has(rm); } catch (_) { return false; } })(); @@ -1071,8 +1085,17 @@ route('POST', '/api/meetings/guest-token', async (req, res) => { // signaling (meeting-join guestId) — that mapping is how their media attaches to their tile. const gid = (typeof identity === 'string' && /^guest-[a-z0-9]+$/i.test(identity)) ? identity.slice(0, 64) : ('guest-' + crypto.randomBytes(8).toString('hex')); const gname = String(name || 'Guest').slice(0, 60); - const token = livekitToken(gid, gname, rm, JSON.stringify({ guest: true })); - json(res, 200, { token, url: LIVEKIT_URL, identity: gid, name: gname }); + // #12 Multi-device (same as /api/meetings/token): prefer the guest's per-connection mesh peerId as the + // LiveKit identity so two devices don't collide; fall back to the throwaway guest id. Anti-hijack guarded. + let lkid = gid; + const pid = (typeof body.peerId === 'string') ? body.peerId.trim() : ''; + if (/^[A-Za-z0-9_-]{4,64}$/.test(pid)) { + let ok = true; + try { const peers = require('./presence').meetingRooms.get(rm); const pr = peers && peers.get(pid); if (pr && pr.uid && pr.uid !== gid) ok = false; } catch (_) {} + if (ok) lkid = pid; + } + const token = livekitToken(lkid, gname, rm, JSON.stringify({ guest: true })); + json(res, 200, { token, url: LIVEKIT_URL, identity: lkid, name: gname }); }); // Decline an incoming 1:1 call: drops the caller, posts a "Call declined" line, clears the call. diff --git a/server/signaling.js b/server/signaling.js index 37cedcc..fc59a48 100644 --- a/server/signaling.js +++ b/server/signaling.js @@ -58,9 +58,9 @@ function finishMeetingJoin(ws, room, peers) { const hostUserId = roomHost.get(room); const avatar = ws._meetingAvatar || null; const isHost = !!(ws._meetingUserId && hostUserId && ws._meetingUserId === hostUserId); - ws.send(JSON.stringify({ type: 'meeting-joined', room, peerId, isHost, peers: [...peers.entries()].map(([id, p]) => ({ peerId: id, name: p.name, avatar: p.avatar || null, uid: p.uid || null })) })); - for (const [, p] of peers) { if (p.ws.readyState === 1) p.ws.send(JSON.stringify({ type: 'meeting-peer-joined', peerId, name, avatar, uid: ws._meetingUserId || null })); } - peers.set(peerId, { ws, name, avatar, uid: ws._meetingUserId || null }); + ws.send(JSON.stringify({ type: 'meeting-joined', room, peerId, isHost, peers: [...peers.entries()].map(([id, p]) => ({ peerId: id, name: p.name, avatar: p.avatar || null, uid: p.uid || null, clientId: p.clientId || null })) })); + for (const [, p] of peers) { if (p.ws.readyState === 1) p.ws.send(JSON.stringify({ type: 'meeting-peer-joined', peerId, name, avatar, uid: ws._meetingUserId || null, clientId: ws._clientId || null })); } + peers.set(peerId, { ws, name, avatar, uid: ws._meetingUserId || null, clientId: ws._clientId || null }); noteRoomStat(room, ws, peers.size); // #7: track the high-water participant count for the call log if (ws._meetingUserId) { try { require('./calls').markDmAnswered(room, ws._meetingUserId); } catch (_) {} CHAT.broadcastPresence(ws._meetingUserId); } const tsubs = transcriptSubs.get(room); if (tsubs && tsubs.size > 0) ws.send(JSON.stringify({ type: 'meeting-transcribe-state', active: true })); @@ -142,6 +142,7 @@ async function handle(ws, m, req) { const peerId = A.token(6); const name = String(m.name || 'Guest').slice(0, 60); ws.kind = 'meeting'; ws._meetingRoom = room; ws._peerId = peerId; ws._peerName = name; + ws._clientId = (typeof m.clientId === 'string' && m.clientId) ? m.clientId.slice(0, 64) : null; // #12: stable per-device id → dedup a reconnecting device without collapsing a real 2nd device // Host = the meeting's creator. roomHost is set on call/meeting creation; scheduled meetings fall back to created_by. let hostUserId = roomHost.get(room); if (hostUserId === undefined) { try { const s = await R.scheduledMeetings.byCode(room); if (s) { hostUserId = s.created_by; roomHost.set(room, hostUserId); } } catch (_) {} }