From aa4aeeaba19492fbdf4a1921451176d5d6f95720 Mon Sep 17 00:00:00 2001 From: sravan Date: Fri, 30 Oct 2026 17:25:18 +0530 Subject: [PATCH] =?UTF-8?q?android(call):=20Increment=202a=20=E2=80=94=20n?= =?UTF-8?q?ative=20LiveKit=20call=20audio,=20background=20service,=20speak?= =?UTF-8?q?er/BT=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces WebView call media on Android with a native LiveKit connection (iOS parity), so call audio survives backgrounding and the output can be switched to Speaker/Bluetooth/Earpiece — the #3 (missing speaker control) and #4 (background audio) items. Gated behind a new, default-off server flag ANDROID_NATIVE_MEDIA (independent of ANDROID_CALLS, which only governs the native ring). Flag off = today's WebView media, video included; flag on = native audio (video is Increment 2b, so enable it for VOICE testing first). No behavior change until flipped. Native (mobile/plugins/native-call/android): - build.gradle: add kotlin-android + io.livekit:livekit-android 2.29.2 (+ twilio audioswitch, coroutines) + JitPack repo; minSdk 24 (LiveKit floor). - CallMedia.kt: LiveKit Room wrapper — connect/mic/disconnect + AudioSwitchHandler routing (auto + selectDevice) and route-change events. - CallForegroundService.java: microphone-type foreground service for background audio (no-ops without RECORD_AUDIO so it never crashes the call). - NativeCallPlugin.java: real bodies for reportOutgoingCall/reconnectRoom/setMuted /endCall + new setAudioRoute/getAudioRoute; fires callConnected/callError/audioRoute. - AndroidManifest: FG service + FOREGROUND_SERVICE(_MICROPHONE)/RECORD_AUDIO/ MODIFY_AUDIO_SETTINGS/BLUETOOTH_CONNECT. - android-patch.js: bump the generated app minSdk to 24. Server: - config.js/routes.js: ANDROID_NATIVE_MEDIA → /api/meetings/config.androidNativeMedia. Web (home.html): - Android flips to the native-media path when androidNativeMedia is set; the existing outgoing/answer/mic/end code then drives the plugin unchanged. - Speaker button wired for Android native: cycles speaker/earpiece/bluetooth via NC.setAudioRoute and reflects the real output from the audioRoute event. Video stays no-op on the native path (Increment 2b). Co-Authored-By: Claude Opus 4.8 --- .../plugins/native-call/android/build.gradle | 38 +++++- .../android/src/main/AndroidManifest.xml | 17 +++ .../nativecall/CallForegroundService.java | 97 +++++++++++++++ .../bizgaze/connect/nativecall/CallMedia.kt | 117 ++++++++++++++++++ .../connect/nativecall/NativeCallPlugin.java | 112 +++++++++++++++-- mobile/scripts/android-patch.js | 21 ++++ server/config.js | 9 ++ server/public/home.html | 53 ++++++-- server/routes.js | 4 +- 9 files changed, 444 insertions(+), 24 deletions(-) create mode 100644 mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallForegroundService.java create mode 100644 mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt diff --git a/mobile/plugins/native-call/android/build.gradle b/mobile/plugins/native-call/android/build.gradle index 0a6b206..ef14a65 100644 --- a/mobile/plugins/native-call/android/build.gradle +++ b/mobile/plugins/native-call/android/build.gradle @@ -1,32 +1,48 @@ // Android build for the native-call plugin. Mirrors the official @capacitor/* plugin module layout (see -// node_modules/@capacitor/status-bar/android/build.gradle) so it drops into the CI-generated app exactly like -// the plugins that already build. Java (not Kotlin) to match the proven official-plugin structure — no extra -// kotlin-gradle-plugin wiring in the library module, one less thing to fail when authored blind. +// node_modules/@capacitor/status-bar/android/build.gradle) so it drops into the CI-generated app like the +// plugins that already build. +// +// INCREMENT 2 adds native LiveKit call media (background audio + Speaker/Bluetooth routing), which is a Kotlin + +// coroutines SDK — so this module now also applies kotlin-android (the ring/plugin glue stays Java). Capacitor +// pulls the Kotlin stdlib via a BOM but does NOT apply the Kotlin Gradle plugin at the root, so we bring it here +// (the standard community-plugin pattern); the classpath is additive to the generated root's AGP. ext { junitVersion = project.hasProperty('junitVersion') ? rootProject.ext.junitVersion : '4.13.2' androidxCoreVersion = project.hasProperty('androidxCoreVersion') ? rootProject.ext.androidxCoreVersion : '1.12.0' androidxAppCompatVersion = project.hasProperty('androidxAppCompatVersion') ? rootProject.ext.androidxAppCompatVersion : '1.6.1' // Same FCM version the @capacitor/push-notifications plugin pulls in, so we don't fork the Firebase version. firebaseMessagingVersion = project.hasProperty('firebaseMessagingVersion') ? rootProject.ext.firebaseMessagingVersion : '23.3.1' + // Follow the project's Kotlin (Capacitor pins kotlin_version at the root); fall back to a recent stable. + bzcKotlinVersion = rootProject.hasProperty('kotlin_version') ? rootProject.ext.kotlin_version : '1.9.24' + // LiveKit Android SDK — native background call media (Increment 2). Pinned; bump deliberately. + livekitVersion = '2.29.2' + audioswitchVersion = '1.2.0' + coroutinesVersion = '1.8.1' } buildscript { + ext.bzc_kotlin = rootProject.hasProperty('kotlin_version') ? rootProject.ext.kotlin_version : '1.9.24' repositories { google() mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:8.2.1' + // Kotlin Gradle plugin so this module can compile the .kt LiveKit wrapper. Additive to the root's AGP. + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$bzc_kotlin" } } apply plugin: 'com.android.library' +apply plugin: 'kotlin-android' android { namespace "com.bizgaze.connect.nativecall" - compileSdk project.hasProperty('compileSdkVersion') ? rootProject.ext.compileSdkVersion : 34 + // LiveKit 2.29 compiles against a recent platform; honor the CI root (Capacitor 8 = 35) and floor at 35. + compileSdk project.hasProperty('compileSdkVersion') ? rootProject.ext.compileSdkVersion : 35 defaultConfig { - minSdkVersion project.hasProperty('minSdkVersion') ? rootProject.ext.minSdkVersion : 23 + // LiveKit Android's floor. The app min is bumped to 24 in CI (android-patch.js) so the merge holds. + minSdkVersion 24 targetSdkVersion project.hasProperty('targetSdkVersion') ? rootProject.ext.targetSdkVersion : 34 versionCode 1 versionName "1.0" @@ -44,11 +60,16 @@ android { sourceCompatibility JavaVersion.VERSION_17 targetCompatibility JavaVersion.VERSION_17 } + kotlinOptions { + jvmTarget = '17' + } } repositories { google() mavenCentral() + // LiveKit pulls transitive artifacts (its WebRTC fork) that live on JitPack. + maven { url 'https://jitpack.io' } } dependencies { @@ -62,4 +83,11 @@ dependencies { // RemoteMessage / FirebaseMessaging (transitive from the push plugin, but declared so this module compiles // against it directly; same version → no resolution conflict). implementation "com.google.firebase:firebase-messaging:$firebaseMessagingVersion" + // Kotlin + coroutines for the native LiveKit media wrapper (CallMedia.kt). + implementation "org.jetbrains.kotlin:kotlin-stdlib:$bzcKotlinVersion" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutinesVersion" + // Native call media (background audio) + audio-device routing (speaker/bluetooth/earpiece). + implementation "io.livekit:livekit-android:$livekitVersion" + // AudioDevice types used by LiveKit's AudioSwitchHandler; declared so they're on our compile classpath. + implementation "com.twilio:audioswitch:$audioswitchVersion" } diff --git a/mobile/plugins/native-call/android/src/main/AndroidManifest.xml b/mobile/plugins/native-call/android/src/main/AndroidManifest.xml index b766b27..8cc4691 100644 --- a/mobile/plugins/native-call/android/src/main/AndroidManifest.xml +++ b/mobile/plugins/native-call/android/src/main/AndroidManifest.xml @@ -7,6 +7,16 @@ + + + + + + + + diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallForegroundService.java b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallForegroundService.java new file mode 100644 index 0000000..af28aa5 --- /dev/null +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallForegroundService.java @@ -0,0 +1,97 @@ +package com.bizgaze.connect.nativecall; + +import android.Manifest; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.app.Service; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ServiceInfo; +import android.os.Build; +import android.os.IBinder; +import android.util.Log; + +import androidx.core.app.NotificationCompat; +import androidx.core.content.ContextCompat; + +// Keeps the call's process alive while the app is backgrounded so native LiveKit audio keeps flowing (Increment 2). +// Android 14+ requires a TYPED foreground service; we use the "microphone" type (a call publishes the mic). That +// type needs RECORD_AUDIO at startForeground time, so start() no-ops without it (the call still works in the +// foreground; the plugin requests the grant so the next call gets background audio). Stopped on call end. +public class CallForegroundService extends Service { + private static final String CHANNEL_ID = "bzc_ongoing_call"; + private static final int NOTIF_ID = 2749; + + // Start the ongoing-call foreground service — but only if RECORD_AUDIO is granted (the microphone FG type + // requires it; starting without it throws on Android 14+). No-op otherwise so we never crash the call. + static void start(Context ctx) { + Context app = ctx.getApplicationContext(); + if (ContextCompat.checkSelfPermission(app, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + Log.i("bzcall", "FG service skipped — RECORD_AUDIO not granted yet (foreground audio only)"); + return; + } + try { + ContextCompat.startForegroundService(app, new Intent(app, CallForegroundService.class)); + } catch (Exception e) { Log.w("bzcall", "FG service start failed: " + e.getMessage()); } + } + + static void stop(Context ctx) { + try { ctx.getApplicationContext().stopService(new Intent(ctx.getApplicationContext(), CallForegroundService.class)); } + catch (Exception ignored) {} + } + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + ensureChannel(); + Notification n = buildNotification(); + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(NOTIF_ID, n, ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE); + } else { + startForeground(NOTIF_ID, n); + } + Log.i("bzcall", "ongoing-call foreground service started"); + } catch (Exception e) { + // Must still call startForeground or the OS kills us with an ANR; fall back to the untyped form. + Log.w("bzcall", "startForeground(microphone) failed: " + e.getMessage()); + try { startForeground(NOTIF_ID, n); } catch (Exception ignored) {} + } + return START_NOT_STICKY; // don't resurrect after the call/process ends + } + + private void ensureChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return; + NotificationManager nm = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); + if (nm == null || nm.getNotificationChannel(CHANNEL_ID) != null) return; + NotificationChannel ch = new NotificationChannel(CHANNEL_ID, "Ongoing call", NotificationManager.IMPORTANCE_LOW); + ch.setDescription("Shown while a Biz Connect call is in progress"); + ch.setShowBadge(false); + ch.setSound(null, null); + nm.createNotificationChannel(ch); + } + + private Notification buildNotification() { + NotificationCompat.Builder b = new NotificationCompat.Builder(this, CHANNEL_ID) + .setSmallIcon(android.R.drawable.sym_action_call) + .setContentTitle("Ongoing call") + .setContentText("Tap to return to the call") + .setOngoing(true) + .setCategory(NotificationCompat.CATEGORY_CALL) + .setPriority(NotificationCompat.PRIORITY_LOW); + try { + Intent launch = getPackageManager().getLaunchIntentForPackage(getPackageName()); + if (launch != null) { + launch.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_SINGLE_TOP); + int flags = PendingIntent.FLAG_UPDATE_CURRENT; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) flags |= PendingIntent.FLAG_IMMUTABLE; + b.setContentIntent(PendingIntent.getActivity(this, 0, launch, flags)); + } + } catch (Exception ignored) {} + return b.build(); + } + + @Override public IBinder onBind(Intent intent) { return null; } +} diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt new file mode 100644 index 0000000..5757715 --- /dev/null +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt @@ -0,0 +1,117 @@ +package com.bizgaze.connect.nativecall + +import android.content.Context +import android.util.Log +import com.twilio.audioswitch.AudioDevice +import io.livekit.android.LiveKit +import io.livekit.android.LiveKitOverrides +import io.livekit.android.audio.AudioOptions +import io.livekit.android.audio.AudioSwitchHandler +import io.livekit.android.room.Room +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +// Native LiveKit call media for Android (Increment 2) — mirrors the iOS plugin's single-Room handling. A +// process-level singleton so the connection survives a WebView reload (e.g. answering from a killed app); the +// foreground service (CallForegroundService) keeps the process alive so audio continues when backgrounded. +// +// Audio routing (speaker / bluetooth / earpiece / wired) is handled by LiveKit's AudioSwitchHandler: it +// auto-selects by priority (bluetooth > wired > speaker > earpiece) and we expose selectDevice() for the manual +// Speaker/Bluetooth control. Playback (hearing the other side) needs no permission; publishing the mic needs +// RECORD_AUDIO — but we answer MUTED (the web re-applies the real mic state on callConnected), so a user with no +// mic grant still hears audio and only needs the grant to unmute. +object CallMedia { + private const val TAG = "bzcall" + private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + private var room: Room? = null + private var audio: AudioSwitchHandler? = null + + // Java-friendly callback the plugin implements to forward to notifyListeners(...). All invoked on main. + interface Events { + fun onConnected() + fun onError(message: String) + fun onRoute(selected: String, available: List) + } + + @JvmStatic var events: Events? = null + + @JvmStatic val isConnected: Boolean get() = room != null + + // Connect (or re-home) the call's LiveKit room. startMuted mirrors the iOS "answer muted" house rule. + @JvmStatic + fun connect(context: Context, url: String, token: String, startMuted: Boolean) { + val app = context.applicationContext + scope.launch { + try { + room?.let { try { it.disconnect() } catch (_: Throwable) {} } + val handler = AudioSwitchHandler(app) + audio = handler + val r = LiveKit.create( + app, + overrides = LiveKitOverrides(audioOptions = AudioOptions(audioHandler = handler)) + ) + room = r + handler.registerAudioDeviceChangeListener { _, _ -> emitRoute() } + r.connect(url, token) + r.localParticipant.setMicrophoneEnabled(!startMuted) + Log.i(TAG, "native media connected (startMuted=$startMuted)") + events?.onConnected() + emitRoute() + } catch (e: Throwable) { + Log.w(TAG, "native media connect failed: ${e.message}") + events?.onError(e.message ?: "connect failed") + } + } + } + + @JvmStatic + fun setMicEnabled(enabled: Boolean) { + val r = room ?: return + scope.launch { + try { r.localParticipant.setMicrophoneEnabled(enabled) } + catch (e: Throwable) { Log.w(TAG, "setMic failed: ${e.message}") } + } + } + + // Manual output selection for the Speaker/Bluetooth/Earpiece control. If the requested device isn't present, + // the current (auto) selection is left alone. + @JvmStatic + fun setRoute(route: String) { + val h = audio ?: return + val target: AudioDevice? = when (route) { + "speaker" -> h.availableAudioDevices.firstOrNull { it is AudioDevice.Speakerphone } + "earpiece" -> h.availableAudioDevices.firstOrNull { it is AudioDevice.Earpiece } + "bluetooth" -> h.availableAudioDevices.firstOrNull { it is AudioDevice.BluetoothHeadset } + "wired" -> h.availableAudioDevices.firstOrNull { it is AudioDevice.WiredHeadset } + else -> null + } + if (target != null) h.selectDevice(target) + emitRoute() + } + + @JvmStatic fun currentRoute(): String = routeName(audio?.selectedAudioDevice) + @JvmStatic fun availableRoutes(): List = audio?.availableAudioDevices?.map { routeName(it) } ?: emptyList() + + private fun emitRoute() { + val h = audio ?: return + events?.onRoute(routeName(h.selectedAudioDevice), h.availableAudioDevices.map { routeName(it) }) + } + + private fun routeName(d: AudioDevice?): String = when (d) { + is AudioDevice.Speakerphone -> "speaker" + is AudioDevice.Earpiece -> "earpiece" + is AudioDevice.BluetoothHeadset -> "bluetooth" + is AudioDevice.WiredHeadset -> "wired" + else -> "unknown" + } + + @JvmStatic + fun disconnect() { + val r = room + room = null + audio = null + if (r != null) scope.launch { try { r.disconnect() } catch (_: Throwable) {} } + } +} diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java index 3f61983..0beb39c 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java @@ -1,5 +1,12 @@ package com.bizgaze.connect.nativecall; +import android.Manifest; +import android.content.pm.PackageManager; + +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +import com.getcapacitor.JSArray; import com.getcapacitor.JSObject; import com.getcapacitor.Plugin; import com.getcapacitor.PluginCall; @@ -32,6 +39,7 @@ public class NativeCallPlugin extends Plugin { @Override public void load() { instance = this; + wireMedia(); if (pendingAnswer != null) { JSObject a = pendingAnswer; pendingAnswer = null; @@ -39,6 +47,45 @@ public class NativeCallPlugin extends Plugin { } } + // Bridge CallMedia's (Kotlin, LiveKit) events to the shared web listeners. Routed through the live plugin + // instance so a WebView reload doesn't fire into a dead bridge. The event names/payloads mirror the iOS plugin + // (callConnected / callError), plus audioRoute for the Android Speaker/Bluetooth control. + private void wireMedia() { + CallMedia.setEvents(new CallMedia.Events() { + @Override public void onConnected() { + NativeCallPlugin p = instance; + if (p != null) p.notifyListeners("callConnected", new JSObject().put("ok", true)); + } + @Override public void onError(String message) { + NativeCallPlugin p = instance; + if (p != null) p.notifyListeners("callError", new JSObject().put("error", message == null ? "" : message)); + } + @Override public void onRoute(String selected, java.util.List available) { + NativeCallPlugin p = instance; + if (p == null) return; + JSObject o = new JSObject(); + o.put("output", selected == null ? "unknown" : selected); + JSArray arr = new JSArray(); + if (available != null) for (String s : available) arr.put(s); + o.put("available", arr); + p.notifyListeners("audioRoute", o); + } + }); + } + + // LiveKit needs RECORD_AUDIO to publish the mic, and the microphone foreground service needs it to start. + // We answer MUTED (playback needs no permission), so this is best-effort: it prompts without blocking, and the + // grant persists so unmute + background audio work from this call on (or the next). + private void ensureMicPermission() { + try { + if (getContext() != null + && ContextCompat.checkSelfPermission(getContext(), Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED + && getActivity() != null) { + ActivityCompat.requestPermissions(getActivity(), new String[]{ Manifest.permission.RECORD_AUDIO }, 7731); + } + } catch (Exception ignored) {} + } + @Override protected void handleOnDestroy() { super.handleOnDestroy(); @@ -90,32 +137,79 @@ public class NativeCallPlugin extends Plugin { call.resolve(); } - // Outgoing call from the web app. Increment 1: media stays in the WebView, so there is nothing to ring or - // connect natively here — the callee is rung by the server's FCM call-wake. Just record it so a later - // endCall/cancel can match. (Increment 2 will start a foreground service + native LiveKit here.) + // Outgoing call from the web app. Increment 2 (native media): start the ongoing-call foreground service and + // connect the LiveKit room with the url/token the web minted (/api/meetings/token). Connect MUTED — the web + // re-applies the real mic state on the callConnected event. The callee is rung by the server's FCM call-wake. @PluginMethod public void reportOutgoingCall(PluginCall call) { String uuid = call.getString("callUUID", ""); if (!uuid.isEmpty()) CallStore.markActive(uuid); + startNativeMedia(call.getString("url", ""), call.getString("token", "")); call.resolve(); } - // Stop ringing / clear the incoming UI for a call (remote hung up, we ended, or another device answered). + // Stop ringing / clear the incoming UI AND tear down native media (remote hung up, we ended, or another device + // answered). Idempotent: disconnect()/stop() are safe no-ops when nothing is connected/running. @PluginMethod public void endCall(PluginCall call) { String uuid = call.getString("callUUID", ""); if (!uuid.isEmpty()) CallStore.markEnded(uuid); CallStore.cancel(getContext()); IncomingCallActivity.dismissIfShowing(); + CallMedia.disconnect(); + CallForegroundService.stop(getContext()); call.resolve(); } - // Increment 1: mute is handled by the WebView (it owns the media). No-op so the shared bridge doesn't reject. - @PluginMethod public void setMuted(PluginCall call) { call.resolve(); } + // Mute/unmute the native mic (Increment 2 — the plugin owns the media now). + @PluginMethod + public void setMuted(PluginCall call) { + boolean muted = Boolean.TRUE.equals(call.getBoolean("muted", false)); + CallMedia.setMicEnabled(!muted); + call.resolve(); + } - // --- Increment 2+ methods (video / screen-share / hole-punch / #12 reconnect / #5 transcript). Safe no-ops - // for now so the shared web bridge never rejects on Android; real bodies land with the native-media build. --- - @PluginMethod public void reconnectRoom(PluginCall call) { call.resolve(); } + // Connect (answer path) or re-home (#12 multi-device) the native LiveKit room with web-minted creds. On + // Android this is the primary connect on answer: the FCM call-wake carries no livekit creds, so the web calls + // this (bzNativeReconnectMedia) with a peerId token once the meeting WebSocket has joined. + @PluginMethod + public void reconnectRoom(PluginCall call) { + startNativeMedia(call.getString("url", ""), call.getString("token", "")); + call.resolve(); + } + + // Select the call audio output (Speaker / Bluetooth / Earpiece / Wired) for the in-call control. Resolves the + // route actually in effect so the web button can reflect it. + @PluginMethod + public void setAudioRoute(PluginCall call) { + String route = call.getString("route", ""); + if (!route.isEmpty()) CallMedia.setRoute(route); + JSObject ret = new JSObject(); + ret.put("output", CallMedia.currentRoute()); + call.resolve(ret); + } + + // Report the current audio output + the devices available to switch to (used to seed/refresh the control). + @PluginMethod + public void getAudioRoute(PluginCall call) { + JSObject ret = new JSObject(); + ret.put("output", CallMedia.currentRoute()); + JSArray arr = new JSArray(); + for (String s : CallMedia.availableRoutes()) arr.put(s); + ret.put("available", arr); + call.resolve(ret); + } + + // Shared connect helper for the outgoing + answer paths. + private void startNativeMedia(String url, String token) { + if (url == null || url.isEmpty() || token == null || token.isEmpty()) return; + ensureMicPermission(); + CallForegroundService.start(getContext()); + CallMedia.connect(getContext(), url, token, true); // start muted; web re-applies on callConnected + } + + // --- Increment 2b methods (video tiles / screen-share / hole-punch / #5 transcript). Safe no-ops for now so + // the shared web bridge never rejects on Android; real bodies land with the native-video build. --- @PluginMethod public void setCamera(PluginCall call) { call.resolve(); } @PluginMethod public void switchCamera(PluginCall call) { call.resolve(); } @PluginMethod public void startScreenShare(PluginCall call) { call.resolve(); } diff --git a/mobile/scripts/android-patch.js b/mobile/scripts/android-patch.js index 0a15201..c216fbf 100644 --- a/mobile/scripts/android-patch.js +++ b/mobile/scripts/android-patch.js @@ -16,6 +16,27 @@ if (!p || !fs.existsSync(p)) { console.log(' (AndroidManifest.xml not found — try { let s = fs.readFileSync(p, 'utf8'); + // (0) LiveKit Android (native call media, Increment 2) requires a higher minSdk than Capacitor's default. + // Bump it in the generated variables.gradle so the library manifest merge holds (a library minSdk above the + // app's fails the build). variables.gradle sits at android/variables.gradle (3 dirs up from app/src/main). + try { + const path = require('path'); + const varsPath = path.resolve(path.dirname(p), '..', '..', '..', 'variables.gradle'); + if (fs.existsSync(varsPath)) { + let v = fs.readFileSync(varsPath, 'utf8'); + const m = v.match(/minSdkVersion\s*=\s*(\d+)/); + if (m && parseInt(m[1], 10) < 24) { + v = v.replace(/minSdkVersion\s*=\s*\d+/, 'minSdkVersion = 24'); + fs.writeFileSync(varsPath, v); + console.log(' minSdkVersion bumped to 24 (LiveKit Android floor)'); + } else { + console.log(' minSdkVersion already >= 24 or not found — no bump'); + } + } else { + console.log(' variables.gradle not found — minSdk bump skipped'); + } + } catch (e) { console.log(' minSdk bump skipped:', e.message); } + // (1) Native-call FirebaseMessagingService swap. The native-call plugin declares its own MESSAGING_EVENT // service (BzCallMessagingService) that handles call-wake data messages AND forwards regular push to Capacitor // via super. Remove @capacitor/push-notifications' own MessagingService here — the APP manifest is the highest diff --git a/server/config.js b/server/config.js index 4540002..0200fc8 100644 --- a/server/config.js +++ b/server/config.js @@ -55,6 +55,14 @@ const CALLKIT_ENABLED = process.env.CALLKIT_ENABLED === '1'; // FCM configured (FCM_SERVICE_ACCOUNT) + an app build carrying the native-call Android plugin. Set ANDROID_CALLS=1. const ANDROID_CALLS = process.env.ANDROID_CALLS === '1'; +// Native call MEDIA on Android (Increment 2): LiveKit audio in the plugin + a foreground service, instead of the +// WebView carrying the media. INDEPENDENT of ANDROID_CALLS (which only governs the native ring) so media can be +// flipped on for testing after the ring is solid. OFF (default) → Android keeps WebView call media (works today, +// video included). ON → the Android app drives call audio natively (background audio + Speaker/Bluetooth), but +// video is not yet native (Increment 2b) — so enable this for VOICE testing first. Requires an app build carrying +// the LiveKit-Android plugin. Set ANDROID_NATIVE_MEDIA=1. +const ANDROID_NATIVE_MEDIA = process.env.ANDROID_NATIVE_MEDIA === '1'; + module.exports = { PORT: process.env.PORT || 8090, HTTPS_PORT: process.env.HTTPS_PORT || 8443, @@ -73,6 +81,7 @@ module.exports = { GIPHY_API_KEY, CALLKIT_ENABLED, ANDROID_CALLS, + ANDROID_NATIVE_MEDIA, PUBLIC_DIR, REC_DIR, TRANS_DIR, diff --git a/server/public/home.html b/server/public/home.html index 41f6f42..30c2999 100644 --- a/server/public/home.html +++ b/server/public/home.html @@ -4346,11 +4346,19 @@ async function setupNativeCall(){ const plat=nativePlatform(); let cfg={}; try{ cfg=await fetch('/api/meetings/config').then(r=>r.json()); }catch(_){} if(plat==='android'){ - // Android Increment 1: native full-screen ring (over the lock screen, wakes from closed via FCM data), but - // the WebView still carries the call media. Gated on androidCall (server ANDROID_CALLS=1). The 'android' FCM + // Android native calling (ring + answer) gated on androidCall (server ANDROID_CALLS=1). The 'android' FCM // token is already registered by setupNativePush — the server call-wake targets it, so no token step here. if(!cfg || !cfg.androidCall){ console.log('[nativecall] Android native calling disabled by server — using WebView calls'); return; } - _callkitReady=true; _callWebViewMedia=true; + _callkitReady=true; + // Increment 2: when the server enables native media, the plugin carries the call AUDIO (background audio + + // Speaker/Bluetooth) and the WebView must NOT also join SFU media. Until then (default), media stays in the + // WebView (video calls included). androidNativeMedia === true → _callWebViewMedia=false (native media path). + _callWebViewMedia = !(cfg && cfg.androidNativeMedia); + if(!_callWebViewMedia){ + console.log('[nativecall] Android native call media ON — background audio + native routing'); + // Android CAN switch the output (unlike iOS): mirror the plugin's real route into the speaker button. + try{ NC.addListener('audioRoute', (e)=>{ bzOnAndroidRoute(e); }); }catch(_){} + } } else { // iOS CallKit: native media (the plugin owns this identity's one LiveKit connection). Flipped on once native // media carries the audio. @@ -4385,7 +4393,7 @@ async function setupNativeCall(){ }catch(e){ console.warn('[nativecall] answerCall failed:', e); } }); // Enforce the muted-by-default rule against the plugin's actual mic once the room connects (belt-and-braces // for builds whose plugin still connects the mic live). meetMic is the source of truth for the mic button. - NC.addListener('callConnected', ()=>{ if(meetNative){ try{ NC.setMuted({ muted:!meetMic }); }catch(_){} if(meetCam){ try{ NC.setCamera({ on:true }); }catch(_){} } } }); // re-applies mic/cam after a #12 media reconnect too + NC.addListener('callConnected', ()=>{ if(meetNative){ try{ NC.setMuted({ muted:!meetMic }); }catch(_){} if(meetCam){ try{ NC.setCamera({ on:true }); }catch(_){} try{ if(nativeCallAudioOn() && NC.getAudioRoute){ NC.getAudioRoute().then((r)=>bzOnAndroidRoute(r)).catch(()=>{}); } }catch(_){} } }); // re-applies mic/cam (and seeds the Android audio route) after a #12 media reconnect too NC.addListener('callError', (e)=>{ console.warn('[callkit] call error:', (e&&e.error)||''); }); // Muted from the CallKit system UI → reflect it in the meeting window's mic button. NC.addListener('setMuted', (e)=>{ if(!e||meetState!=='call'||!meetNative) return; meetMic=!e.muted; try{ updateMicBtn(); setTileMute('__local', !meetMic); meetSend({type:'meeting-state', muted:!meetMic, camOff:!meetCam}); }catch(_){} }); @@ -5588,7 +5596,12 @@ function bzApplyRoute(route){ } // The speaker toggle is usable if the browser supports output switching (setSinkId) OR the native iOS route // plugin is present. -function canRouteAudio(){ try{ if(nativeAudioRoute()) return true; return typeof HTMLMediaElement!=='undefined' && typeof HTMLMediaElement.prototype.setSinkId==='function'; }catch(_){ return false; } } +function canRouteAudio(){ try{ if(nativeAudioRoute()) return true; if(nativeCallAudioOn()) return true; return typeof HTMLMediaElement!=='undefined' && typeof HTMLMediaElement.prototype.setSinkId==='function'; }catch(_){ return false; } } +// Android native call media: the NativeCall plugin owns the audio and CAN switch the output (speaker/earpiece/ +// bluetooth), unlike iOS. True only when the server enabled native media and the plugin is present. +function nativeCallAudioOn(){ try{ return nativePlatform()==='android' && _callkitReady && !_callWebViewMedia && !!nativeCallPlugin(); }catch(_){ return false; } } +let _androidRoutes=[]; // outputs the plugin reports we can switch to (from the audioRoute event / getAudioRoute) +function bzOnAndroidRoute(e){ if(!e) return; if(e.output) _nativeOutput=e.output; if(Array.isArray(e.available)) _androidRoutes=e.available; refreshSpkBtn(); } // new #4: let the user drag a floating bar out of the way — it otherwise covers part of the shared // screen. Drag from anywhere on the bar except a control. Position is remembered per bar. function makeDraggable(el, key){ @@ -5637,11 +5650,22 @@ let meetRoute=(()=>{ try{ return localStorage.getItem('bzc_route')||'speaker'; } let _nativeOutput=null; function iosRoute(){ return _nativeOutput; } // truthy only on iOS native once the plugin has reported function spkIcon(){ - if(iosRoute()){ return _nativeOutput==='bluetooth' ? 'bluetooth' : (_nativeOutput==='wired' ? 'headphones' : 'speaker'); } + // iosRoute() is truthy on iOS AND on Android native media (both drive the icon from the real output _nativeOutput). + if(iosRoute()){ + if(_nativeOutput==='bluetooth') return 'bluetooth'; + if(_nativeOutput==='wired') return 'headphones'; + if(_nativeOutput==='earpiece') return 'speakerOff'; // Android can route to the earpiece (iOS reports 'receiver') + return 'speaker'; + } return meetRoute==='bt' ? 'bluetooth' : (meetRoute==='speaker' ? 'speaker' : 'speakerOff'); } function routeLabel(){ - if(iosRoute()){ return _nativeOutput==='bluetooth' ? 'Bluetooth' : (_nativeOutput==='wired' ? 'Headset' : 'Speaker'); } + if(iosRoute()){ + if(_nativeOutput==='bluetooth') return 'Bluetooth'; + if(_nativeOutput==='wired') return 'Headset'; + if(_nativeOutput==='earpiece') return 'Earpiece'; + return 'Speaker'; + } return meetRoute==='bt' ? 'Bluetooth / headset' : (meetRoute==='speaker' ? 'Speaker' : 'Earpiece'); } function refreshSpkBtn(){ @@ -5671,7 +5695,7 @@ function bzInitNativeRoute(){ function bzOnNativeRoute(output){ if(!output) return; _nativeOutput=output; - meetRoute = (output==='bluetooth'||output==='wired') ? 'bt' : (output==='speaker' ? 'speaker' : meetRoute); + meetRoute = (output==='bluetooth'||output==='wired') ? 'bt' : (output==='earpiece' ? 'earpiece' : (output==='speaker' ? 'speaker' : meetRoute)); refreshSpkBtn(); } // Detect a connected BT/headset output so the cycle can include it (and the icon can show it). @@ -5783,6 +5807,19 @@ async function toggleSpeakerphone(){ : 'On speaker — connect a Bluetooth/wired headset to use it'); return; } + // Android native call media: the plugin CAN switch the route — cycle through the outputs it reports (bluetooth/ + // wired only appear while a headset is connected). + if(nativeCallAudioOn()){ + const NC=nativeCallPlugin(); if(!NC){ return; } + const avail=(_androidRoutes && _androidRoutes.length) ? _androidRoutes : ['speaker','earpiece']; + const order=['speaker','earpiece','bluetooth','wired'].filter(r=>avail.indexOf(r)>=0); + const cur=_nativeOutput||'speaker'; + const i=order.indexOf(cur); + const next=order[(i<0?0:(i+1))%(order.length||1)] || 'speaker'; + try{ const r=await NC.setAudioRoute({ route: next }); if(r && r.output){ _nativeOutput=r.output; } else { _nativeOutput=next; } }catch(_){ _nativeOutput=next; } + refreshSpkBtn(); toast(routeLabel()); + return; + } let outs=[]; try{ outs=(await navigator.mediaDevices.enumerateDevices()).filter(x=>x.kind==='audiooutput'); }catch(_){} const bt=outs.find(d=>isBtLabel(d.label)); _btConnected=!!bt; const routes = ['speaker','earpiece'].concat(bt?['bt']:[]); diff --git a/server/routes.js b/server/routes.js index d6e77df..946f4dd 100644 --- a/server/routes.js +++ b/server/routes.js @@ -116,7 +116,7 @@ const API_KEY_SCOPES = ['report:read', 'audit:read']; const { onlineAgents, meetingRooms, groupCalls, dmCalls } = require('./presence'); const CALLS = require('./calls'); require('./reminders'); // start the 10-minute meeting-reminder loop -const { REC_DIR, TRANS_DIR, UPLOADS_DIR, SESSION_TTL, REFRESH_TTL, LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET, LIVEKIT_ENABLED, PUBLIC_BASE_URL, GIPHY_API_KEY, CALLKIT_ENABLED, ANDROID_CALLS } = require('./config'); +const { REC_DIR, TRANS_DIR, UPLOADS_DIR, SESSION_TTL, REFRESH_TTL, LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET, LIVEKIT_ENABLED, PUBLIC_BASE_URL, GIPHY_API_KEY, CALLKIT_ENABLED, ANDROID_CALLS, ANDROID_NATIVE_MEDIA } = require('./config'); const https = require('https'); // Small GET-JSON helper for the GIPHY proxy (keeps the key server-side). function fetchJSON(url) { @@ -993,7 +993,7 @@ route('POST', '/api/calls/invite', async (req, res) => { // Does this deployment use the LiveKit SFU for meeting media? The client asks on load; if sfu is // false it uses the built-in P2P mesh. url is the browser-facing signaling endpoint (wss://…). route('GET', '/api/meetings/config', (req, res) => { - json(res, 200, { sfu: LIVEKIT_ENABLED, url: LIVEKIT_ENABLED ? LIVEKIT_URL : '', callkit: CALLKIT_ENABLED, androidCall: ANDROID_CALLS, fcm: PUSH.fcmReady() }); + json(res, 200, { sfu: LIVEKIT_ENABLED, url: LIVEKIT_ENABLED ? LIVEKIT_URL : '', callkit: CALLKIT_ENABLED, androidCall: ANDROID_CALLS, androidNativeMedia: ANDROID_NATIVE_MEDIA, fcm: PUSH.fcmReady() }); }); // #5 GIF search — server-side GIPHY proxy so the API key never reaches the browser. Empty q → trending.