diff --git a/codemagic.yaml b/codemagic.yaml index f289256..233b5e0 100644 --- a/codemagic.yaml +++ b/codemagic.yaml @@ -76,6 +76,15 @@ workflows: # generates already contains the new target. ruby mobile/scripts/add-share-extension.rb + - name: Add the Broadcast (screen-share) Extension target + script: | + # Inject the ReplayKit broadcast upload extension (lets the user share their iPhone screen). + # It links the LiveKit Swift package (LKSampleHandler). Runs after cap sync so the SPM project + # + App.entitlements already exist. A build failure here is most likely the SPM product-link or + # the missing App Group capability on the com.bizgaze.connect.broadcast App ID (a one-time manual + # step in the Apple Developer portal — see mobile/IOS_SETUP.md). + ruby mobile/scripts/add-broadcast-extension.rb + - name: Set up code signing script: | # Create the distribution certificate + provisioning profile from the ASC API key and add the @@ -103,6 +112,13 @@ workflows: --type IOS_APP_STORE \ --certificate-key="@env:CERTIFICATE_PRIVATE_KEY" \ --create + # THIRD bundle id: the broadcast (screen-share) extension. Same story as .share — its App Group + # capability (group.com.bizgaze.connect) must be enabled MANUALLY on the App ID in the Apple + # Developer portal (fetch-signing-files registers the id + profile but does NOT toggle App Group). + app-store-connect fetch-signing-files "${BUNDLE_ID}.broadcast" \ + --type IOS_APP_STORE \ + --certificate-key="@env:CERTIFICATE_PRIVATE_KEY" \ + --create keychain add-certificates # (No "Install CocoaPods" step under SPM — there is no Podfile. Xcode resolves the Swift packages diff --git a/mobile/IOS_SETUP.md b/mobile/IOS_SETUP.md index ac815da..9e46d36 100644 --- a/mobile/IOS_SETUP.md +++ b/mobile/IOS_SETUP.md @@ -125,3 +125,22 @@ to the Files folder, the Photos "Connect" album, Manage storage — works withou Also enabled by this change (main app Info.plist, done automatically by `ios-patch.sh`): - `UIFileSharingEnabled` + `LSSupportsOpeningDocumentsInPlace` → the **Biz Connect** folder in Files. - `CFBundleURLTypes` scheme **`bizconnect`** → lets the extension bounce back into the app after staging. + +## Broadcast Extension (share your iPhone SCREEN in a call) — one-time Apple portal setup + +Screen sharing from iOS uses a **Broadcast Upload Extension** (`com.bizgaze.connect.broadcast`) — the same +pattern as the Share Extension. The Codemagic build injects the target, links LiveKit into it, and fetches a +profile automatically, but the **App Group capability can only be toggled by hand** in the Apple portal: + +1. Reuse the SAME App Group as the Share Extension: **`group.com.bizgaze.connect`** (no new group needed). +2. **Enable the App Groups capability on the broadcast App ID** and assign it to that group: + - `com.bizgaze.connect.broadcast` (create this App ID if the first build hasn't yet — `fetch-signing-files + --create` registers it, then edit it to add App Groups). The app (`com.bizgaze.connect`) already has the + group from the Share Extension setup above. + After enabling it, re-run the Codemagic build so `fetch-signing-files` regenerates the profile. + +Until the App Group is on the broadcast App ID, the **archive fails code-signing** (entitlement mismatch) — +that's the expected first-build failure. The shared App Group is how the extension (ReplayKit capture) hands +screen frames to the app over LiveKit's IPC socket. Receiving OTHERS' shared screens needs none of this — it +already works. `ios-patch.sh` sets `RTCScreenSharingExtension` + `RTCAppGroupIdentifier` in the app Info.plist +so LiveKit finds the extension + group. diff --git a/mobile/ios-broadcast/BroadcastExtension.entitlements b/mobile/ios-broadcast/BroadcastExtension.entitlements new file mode 100644 index 0000000..3c822a6 --- /dev/null +++ b/mobile/ios-broadcast/BroadcastExtension.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.com.bizgaze.connect + + + diff --git a/mobile/ios-broadcast/Info.plist b/mobile/ios-broadcast/Info.plist new file mode 100644 index 0000000..09376c1 --- /dev/null +++ b/mobile/ios-broadcast/Info.plist @@ -0,0 +1,29 @@ + + + + + CFBundleDisplayName + Biz Connect Screen + CFBundleName + $(PRODUCT_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + NSExtension + + NSExtensionPointIdentifier + com.apple.broadcast-services-upload + NSExtensionPrincipalClass + $(PRODUCT_MODULE_NAME).SampleHandler + RPBroadcastProcessMode + RPBroadcastProcessModeSampleBuffer + + + diff --git a/mobile/ios-broadcast/SampleHandler.swift b/mobile/ios-broadcast/SampleHandler.swift new file mode 100644 index 0000000..40ac82e --- /dev/null +++ b/mobile/ios-broadcast/SampleHandler.swift @@ -0,0 +1,16 @@ +import ReplayKit +import LiveKit + +// Principal class of the Broadcast Upload Extension (ReplayKit) that lets the iOS user share their screen. +// +// HOW IT WORKS: when the user starts a system broadcast, iOS launches THIS extension. LiveKit's LKSampleHandler +// does everything — on broadcastStarted it opens an IPC socket in the shared App Group (group.com.bizgaze.connect) +// and streams the ReplayKit sample buffers to the MAIN app, whose LiveKit connection publishes them as a +// screen-share track. The extension itself never creates a Room / initialises WebRTC, so it stays well under the +// 50 MB extension memory limit. So this subclass can be empty. +// +// The extension finds the app group + reports state to the app via LiveKit's Darwin-notification/socket +// convention, keyed off this extension's bundle id (com.bizgaze.connect.broadcast) and the default app group +// group.. Both are also set explicitly on the app side (RTCScreenSharingExtension / +// RTCAppGroupIdentifier in Info.plist) so there's no ambiguity. +class SampleHandler: LKSampleHandler {} diff --git a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift index cddb6fd..906ff04 100644 --- a/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift +++ b/mobile/plugins/native-call/ios/Sources/NativeCallPlugin/NativeCallPlugin.swift @@ -19,7 +19,7 @@ import LiveKit // SPM product name is 'LiveKit' (Package.swift). (The old CocoaP // the token from the WebView via reportOutgoingCall(). The WebView is UI only for native calls (it must // NOT also join the room — LiveKit allows one connection per identity). @objc(NativeCallPlugin) -public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelegate, CXProviderDelegate { +public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelegate, CXProviderDelegate, BroadcastManagerDelegate { public let identifier = "NativeCallPlugin" public let jsName = "NativeCall" public let pluginMethods: [CAPPluginMethod] = [ @@ -29,6 +29,8 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega CAPPluginMethod(name: "setMuted", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setCamera", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "switchCamera", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "startScreenShare", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "stopScreenShare", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "syncVideoTiles", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "endCall", returnType: CAPPluginReturnPromise) ] @@ -75,6 +77,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega // preferSpeaker and fixed it). Listening for route changes makes that self-healing. NotificationCenter.default.addObserver(self, selector: #selector(audioRouteChanged(_:)), name: AVAudioSession.routeChangeNotification, object: nil) + + // Screen sharing (ReplayKit broadcast extension). LiveKit tells us when a broadcast starts/stops and + // (with shouldPublishTrack=true, the default) auto-publishes/unpublishes the screen-share track. + BroadcastManager.shared.delegate = self } @objc private func audioRouteChanged(_ note: Notification) { @@ -91,7 +97,10 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega // engine in didActivate can block on first use — determining it up front avoids that. AVAudioSession.sharedInstance().requestRecordPermission { _ in } let old = room - let r = Room() + // Route screen-share through the ReplayKit broadcast extension (so the user can share their screen + // even when the app is backgrounded, and it captures the whole phone, not just the WebView). + let opts = RoomOptions(defaultScreenShareCaptureOptions: ScreenShareCaptureOptions(useBroadcastExtension: true)) + let r = Room(roomOptions: opts) room = r Task { [weak self] in await old?.disconnect() // drop any previous/stale connection so we never leave DUPLICATE participants @@ -331,6 +340,26 @@ public class NativeCallPlugin: CAPPlugin, CAPBridgedPlugin, PKPushRegistryDelega } } + // Screen sharing from iOS: show the system broadcast picker. When the user starts the broadcast, the + // extension streams the screen to us over IPC and LiveKit publishes it (BroadcastManager.shouldPublishTrack + // defaults true). broadcastManager(didChangeState:) fires screenShareState back to the web either way. + @objc func startScreenShare(_ call: CAPPluginCall) { + guard room != nil else { call.reject("no active call"); return } + DispatchQueue.main.async { BroadcastManager.shared.requestActivation() } // presents RPSystemBroadcastPickerView + call.resolve() + } + + @objc func stopScreenShare(_ call: CAPPluginCall) { + BroadcastManager.shared.requestStop() + call.resolve() + } + + // MARK: - BroadcastManagerDelegate + + public func broadcastManager(didChangeState isBroadcasting: Bool) { + notifyListeners("screenShareState", data: ["sharing": isBroadcasting]) + } + // Position native video views to match the web meeting tiles. `tiles` = [{uid, local, x, y, w, h}] in // CSS px (== points; getBoundingClientRect coords). We create/move a VideoView for each participant that // has a live camera track, and remove views for tiles that are gone or whose camera is off (so the web diff --git a/mobile/scripts/add-broadcast-extension.rb b/mobile/scripts/add-broadcast-extension.rb new file mode 100644 index 0000000..5b9e4af --- /dev/null +++ b/mobile/scripts/add-broadcast-extension.rb @@ -0,0 +1,122 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true +# +# Inject the Broadcast Upload Extension (ReplayKit screen sharing) into the Capacitor-generated Xcode project. +# Mirrors add-share-extension.rb, with the extra step that this extension LINKS the LiveKit Swift package +# (LKSampleHandler lives in the LiveKit product), so it adds a package product dependency to the new target. +# +# WHAT IT WIRES: +# * a new app-extension target "BroadcastExtension" (bundle id .broadcast) whose source is our +# SampleHandler.swift (subclass of LiveKit's LKSampleHandler) + Info.plist + entitlements, copied from +# mobile/ios-broadcast/ +# * the App Group entitlement (group.com.bizgaze.connect) on the extension (LiveKit's IPC socket lives there) +# * a Swift Package product dependency on LiveKit (github.com/livekit/client-sdk-swift 2.15.3) so the +# extension can subclass LKSampleHandler +# * the extension embedded into the app ("Embed App Extensions") + set as a build dependency +# +# Idempotent: if the target already exists it is removed and rebuilt. + +require 'xcodeproj' +require 'fileutils' + +ROOT = File.expand_path('..', __dir__) +PROJECT = File.join(ROOT, 'ios', 'App', 'App.xcodeproj') +SRC_DIR = File.join(ROOT, 'ios-broadcast') +APP_DIR = File.join(ROOT, 'ios', 'App') +EXT_NAME = 'BroadcastExtension' +EXT_DIR = File.join(APP_DIR, EXT_NAME) +APP_TARGET = 'App' +APP_BUNDLE = ENV.fetch('BUNDLE_ID', 'com.bizgaze.connect') +EXT_BUNDLE = "#{APP_BUNDLE}.broadcast" +APP_GROUP = 'group.com.bizgaze.connect' +LK_URL = 'https://github.com/livekit/client-sdk-swift.git' +LK_VERSION = '2.15.3' + +abort "xcodeproj not found at #{PROJECT}" unless File.directory?(PROJECT) + +project = Xcodeproj::Project.open(PROJECT) +app = project.targets.find { |t| t.name == APP_TARGET } +abort "App target not found" unless app + +# ── Clean any previous injection so this is idempotent ────────────────────────────────────────────── +project.targets.select { |t| t.name == EXT_NAME }.each do |t| + t.build_configuration_list&.build_configurations&.each { |c| c.remove_from_project } + t.remove_from_project +end +if (grp = project.main_group.children.find { |g| g.respond_to?(:display_name) && g.display_name == EXT_NAME }) + grp.remove_from_project +end + +# ── Copy our sources into the app project so paths inside the .xcodeproj are stable ────────────────── +FileUtils.mkdir_p(EXT_DIR) +%w[SampleHandler.swift Info.plist BroadcastExtension.entitlements].each do |f| + FileUtils.cp(File.join(SRC_DIR, f), File.join(EXT_DIR, f)) +end + +# ── Create the extension target ────────────────────────────────────────────────────────────────────── +deployment = app.deployment_target || project.build_configurations.first&.build_settings&.[]('IPHONEOS_DEPLOYMENT_TARGET') || '15.0' +ext = project.new_target(:app_extension, EXT_NAME, :ios, deployment, project.products_group, :swift) + +group = project.main_group.new_group(EXT_NAME, EXT_NAME.to_s) +swift_ref = group.new_reference(File.join(EXT_DIR, 'SampleHandler.swift')) +ext.add_file_references([swift_ref]) + +ext.build_configurations.each do |cfg| + s = cfg.build_settings + s['PRODUCT_BUNDLE_IDENTIFIER'] = EXT_BUNDLE + s['PRODUCT_NAME'] = '$(TARGET_NAME)' + s['INFOPLIST_FILE'] = "#{EXT_NAME}/Info.plist" + s['CODE_SIGN_ENTITLEMENTS'] = "#{EXT_NAME}/BroadcastExtension.entitlements" + s['IPHONEOS_DEPLOYMENT_TARGET'] = deployment + s['SWIFT_VERSION'] = '5.0' + s['TARGETED_DEVICE_FAMILY'] = '1,2' + s['GENERATE_INFOPLIST_FILE'] = 'NO' + s['SKIP_INSTALL'] = 'YES' + s['CODE_SIGN_STYLE'] = 'Manual' + s['MARKETING_VERSION'] = '1.0' + s['CURRENT_PROJECT_VERSION'] = ENV.fetch('BUILD_NUMBER', '1') + s['LD_RUNPATH_SEARCH_PATHS'] = '$(inherited) @executable_path/Frameworks @executable_path/../../Frameworks' +end + +# ── Link LiveKit (Swift Package product) so the extension can subclass LKSampleHandler ──────────────── +# The app already resolves client-sdk-swift 2.15.3 (via the native-call plugin's Package.swift). Add a +# project-level remote package reference to the SAME repo+version (SPM dedupes it) and attach the "LiveKit" +# product to this extension target. +root = project.root_object +pkg = root.package_references.find { |r| r.respond_to?(:repositoryURL) && r.repositoryURL == LK_URL } +unless pkg + pkg = project.new(Xcodeproj::Project::Object::XCRemoteSwiftPackageReference) + pkg.repositoryURL = LK_URL + pkg.requirement = { 'kind' => 'exactVersion', 'version' => LK_VERSION } + root.package_references << pkg +end +prod = project.new(Xcodeproj::Project::Object::XCSwiftPackageProductDependency) +prod.package = pkg +prod.product_name = 'LiveKit' +ext.package_product_dependencies << prod +bf = project.new(Xcodeproj::Project::Object::PBXBuildFile) +bf.product_ref = prod +ext.frameworks_build_phase.files << bf + +# ── App Group entitlement on the MAIN app target too (merge — keep aps-environment etc.) ────────────── +app_ent_path = File.join(APP_DIR, 'App', 'App.entitlements') +app_ent = File.exist?(app_ent_path) ? (Xcodeproj::Plist.read_from_path(app_ent_path) || {}) : {} +groups = app_ent['com.apple.security.application-groups'] || [] +groups << APP_GROUP unless groups.include?(APP_GROUP) +app_ent['com.apple.security.application-groups'] = groups +Xcodeproj::Plist.write_to_path(app_ent, app_ent_path) +app.build_configurations.each { |cfg| cfg.build_settings['CODE_SIGN_ENTITLEMENTS'] = 'App/App.entitlements' } + +# ── Embed the extension into the app + depend on it ────────────────────────────────────────────────── +app.add_dependency(ext) +embed = app.build_phases.find { |p| p.respond_to?(:symbol_dst_subfolder_spec) && p.symbol_dst_subfolder_spec == :plug_ins } +embed ||= begin + phase = app.new_copy_files_build_phase('Embed App Extensions') + phase.symbol_dst_subfolder_spec = :plug_ins + phase +end +build_file = embed.add_file_reference(ext.product_reference) +build_file.settings = { 'ATTRIBUTES' => ['RemoveHeadersOnCopy'] } + +project.save +puts "Broadcast Extension injected: #{EXT_NAME} (#{EXT_BUNDLE}) linked to LiveKit #{LK_VERSION}, embedded in #{APP_TARGET}" diff --git a/mobile/scripts/ios-patch.sh b/mobile/scripts/ios-patch.sh index 54f5e53..f89006a 100644 --- a/mobile/scripts/ios-patch.sh +++ b/mobile/scripts/ios-patch.sh @@ -91,6 +91,14 @@ fi "$PB" -c "Add :aps-environment string production" "$ENT" 2>/dev/null || "$PB" -c "Set :aps-environment production" "$ENT" echo "Entitlements: aps-environment=production ensured in $ENT" +# ── Screen sharing from iOS (ReplayKit broadcast extension + LiveKit) ─────────────────────────────── +# LiveKit's BroadcastManager finds the broadcast upload extension + the shared App Group via these two keys +# (BroadcastBundleInfo reads RTCScreenSharingExtension / RTCAppGroupIdentifier). They match the extension +# added by add-broadcast-extension.rb (bundle id .broadcast) and the App Group used by the share +# extension. Set explicitly so there's no reliance on the default-derivation. +set_str RTCScreenSharingExtension "com.bizgaze.connect.broadcast" +set_str RTCAppGroupIdentifier "group.com.bizgaze.connect" + # We use only standard encryption (HTTPS/TLS), which is exempt — declaring this up front stops App Store # Connect from asking the "export compliance" question on every single build/TestFlight upload. "$PB" -c "Add :ITSAppUsesNonExemptEncryption bool false" "$PLIST" 2>/dev/null \ diff --git a/server/public/home.html b/server/public/home.html index 8a5c664..6c68d93 100644 --- a/server/public/home.html +++ b/server/public/home.html @@ -3879,6 +3879,8 @@ async function setupNativeCall(){ NC.addListener('callError', (e)=>{ console.warn('[callkit] call error:', (e&&e.error)||''); }); // Muted from the CallKit system UI → reflect it in the meeting window's mic button. NC.addListener('setMuted', (e)=>{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} }); + // Screen share (ReplayKit) started/stopped from the system — reflect it and tell peers so their stage shows it. + NC.addListener('screenShareState', (e)=>{ if(meetState!=='call'||!meetNative) return; const on=!!(e&&e.sharing); meetScreen=on; try{ updateScreenBtn(); setLocalSharing(on); meetSend({type:'meeting-screen', on}); }catch(_){} }); // Ended on the CallKit system screen / plugin ended on remote hang-up. Leave the meeting window too (guarded // so our own in-app hang-up, which already called the plugin, doesn't loop). Then tell the server. NC.addListener('endCall', (d)=>{ try{ @@ -5226,7 +5228,12 @@ function meetMakePeer(peerId, name){ // (no extra tiles, the peer's video just shows the screen). Falls back to addTrack+renegotiate if // no video sender exists yet (camera never turned on). Stopping restores the camera (or avatar). async function toggleScreen(){ - if(meetNative){ toast('Sharing your screen from iPhone isn’t supported yet — but you’ll see others’ shared screens.'); return; } // iOS has no getDisplayMedia / ReplayKit ext wired + if(meetNative){ // native call: the plugin shares the whole phone screen via the ReplayKit broadcast extension + const NC=nativeCallPlugin(); + if(!NC||!NC.startScreenShare){ toast('Update the app to share your screen'); return; } + try{ if(meetScreen) await NC.stopScreenShare(); else await NC.startScreenShare(); }catch(_){} + return; // meetScreen + the button flip happen on the authoritative screenShareState event + } if(meetScreen){ stopScreen(); return; } if(!meetMultiShare && meetSharers.size>0){ toast('Someone is already sharing their screen'); return; } if(SFU.on){