diff --git a/TECHSTACK.md b/TECHSTACK.md index 354c210..45515cf 100644 --- a/TECHSTACK.md +++ b/TECHSTACK.md @@ -1,6 +1,6 @@ # Tech Stack — BizGaze Connect -> Living document. **Last updated: 2026-10-30** (rev 3). Update this file whenever a dependency, platform, service, or +> Living document. **Last updated: 2026-11-03** (rev 4). Update this file whenever a dependency, platform, service, or > tool is added/removed/version-bumped. See [Maintaining this file](#maintaining-this-file) and the > [Changelog](#changelog) at the bottom. @@ -109,7 +109,9 @@ hash, per-user salt, timing-safe compare), opaque random session tokens, SHA-256 (`SFSpeechRecognizer`). **Android:** Java, full-screen incoming-call UI over the lock screen + a `FirebaseMessagingService` woken by high-priority FCM data messages. **Increment 2a** adds native LiveKit **audio** (Kotlin `CallMedia` + a `microphone` foreground service for background audio + `AudioSwitchHandler` - routing), gated by the server flag `ANDROID_NATIVE_MEDIA`; native video = Increment 2b. + routing), gated by the server flag `ANDROID_NATIVE_MEDIA`. **Increment 2b** adds native **video** (Kotlin + `CallVideo`: camera publish/flip + remote/local `TextureViewRenderer` tiles hole-punched behind a transparent + WebView), gated by `ANDROID_NATIVE_VIDEO`; screen-share (MediaProjection) is still pending. - `audio-route` (iOS audio output routing), `media-library` (save to Photos/gallery), `share-inbox` (inbound share sheet), `file-opener` (native Quick Look / open-with). - **iOS build:** Capacitor 8 + **SPM** (CocoaPods retired); Xcode 26; a share extension + a broadcast extension. @@ -201,6 +203,10 @@ Bump the **Last updated** date at the top and add a line to the Changelog below. ## Changelog +- **2026-11-03 (rev 4)** — Android native call **video** (Increment 2b): `CallVideo` renders camera/screen tiles + with LiveKit `TextureViewRenderer` hole-punched behind a transparent WebView; `CallMedia` gains camera + publish/flip + track lookup; new `ANDROID_NATIVE_VIDEO` server flag. Also fixed the white meeting background + (hole-punch no longer turns on without native video). - **2026-10-30 (rev 3)** — Android native call media (Increment 2a): added `io.livekit:livekit-android` 2.29.2 + `com.twilio:audioswitch` + Kotlin to the native-call plugin, a `microphone` foreground service, the `ANDROID_NATIVE_MEDIA` server flag, and raised the Android app to minSdk 24. Build fix: `android-patch.js` diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt index 3e87fbe..266a2eb 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt @@ -7,7 +7,11 @@ import io.livekit.android.LiveKit import io.livekit.android.LiveKitOverrides import io.livekit.android.AudioOptions import io.livekit.android.audio.AudioSwitchHandler +import io.livekit.android.renderer.TextureViewRenderer import io.livekit.android.room.Room +import io.livekit.android.room.track.LocalVideoTrack +import io.livekit.android.room.track.Track +import io.livekit.android.room.track.VideoTrack import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob @@ -114,4 +118,55 @@ object CallMedia { audio = null if (r != null) scope.launch { try { r.disconnect() } catch (_: Throwable) {} } } + + // ---- Native video (Increment 2b) ------------------------------------------------------------------- + // The WebView owns the meeting grid but has no LiveKit connection on a native call, so it can't render any + // video. CallVideo draws native TextureViewRenderers behind the (transparent) WebView, positioned to match + // each web tile; these helpers give it the camera control + the live tracks to bind. Mirrors the iOS plugin. + + // Publish/unpublish the local camera (front by default). Fire-and-forget: the web re-syncs tiles right after, + // and a tile only shows video once a live track exists — so a failure just leaves the web avatar showing. + @JvmStatic + fun setCameraEnabled(enabled: Boolean) { + val r = room ?: return + scope.launch { + try { r.localParticipant.setCameraEnabled(enabled) } + catch (e: Throwable) { Log.w(TAG, "setCamera($enabled) failed: ${e.message}") } + } + } + + // Flip front/back. No-arg switchCamera() moves to the next available camera. + @JvmStatic + fun switchCamera() { + val r = room ?: return + val track = r.localParticipant.getTrackPublication(Track.Source.CAMERA)?.track as? LocalVideoTrack ?: return + try { track.switchCamera() } catch (e: Throwable) { Log.w(TAG, "switchCamera failed: ${e.message}") } + } + + // The live camera track for a participant (local, or remote by LiveKit identity = mesh peerId), or null when + // the camera is off/muted so the web avatar shows through instead. + @JvmStatic + fun cameraTrack(uid: String, isLocal: Boolean): VideoTrack? { + val r = room ?: return null + val p = if (isLocal) r.localParticipant else (r.getParticipantByIdentity(uid) ?: return null) + val pub = p.getTrackPublication(Track.Source.CAMERA) ?: return null + if (pub.muted) return null + return pub.track as? VideoTrack + } + + // The remote screen-share track for a participant, or null if they aren't sharing (or not subscribed yet). + @JvmStatic + fun screenTrack(uid: String): VideoTrack? { + val r = room ?: return null + val p = r.getParticipantByIdentity(uid) ?: return null + val pub = p.getTrackPublication(Track.Source.SCREEN_SHARE) ?: return null + if (pub.muted) return null + return pub.track as? VideoTrack + } + + // Init a renderer against the room's EGL context before a track is attached. No-op if no active room. + @JvmStatic + fun initRenderer(renderer: TextureViewRenderer) { + try { room?.initVideoRenderer(renderer) } catch (e: Throwable) { Log.w(TAG, "initRenderer failed: ${e.message}") } + } } diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt new file mode 100644 index 0000000..fd3f9ed --- /dev/null +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt @@ -0,0 +1,137 @@ +package com.bizgaze.connect.nativecall + +import android.app.Activity +import android.graphics.Color +import android.graphics.drawable.ColorDrawable +import android.util.Log +import android.view.ViewGroup +import android.webkit.WebView +import android.widget.FrameLayout +import io.livekit.android.renderer.TextureViewRenderer +import io.livekit.android.room.track.VideoTrack +import livekit.org.webrtc.RendererCommon + +// Native video tiles for Android native calls (Increment 2b) — the Android counterpart to the iOS plugin's +// TileVideoView path. The WebView owns the meeting UI (grid, roster, controls) but, on a native call, has NO +// LiveKit connection — the media lives only in the plugin's native LiveKit room (CallMedia). So we render the +// video NATIVELY: +// +// • Hole-punch: the WebView is made transparent and its parent painted black; a container FrameLayout is +// inserted BEHIND the WebView to host the video. The web meeting UI (opaque controls, avatars for +// camera-off tiles) then paints on top, while tiles flagged `bz-hasvid` go transparent so the native +// video behind shows through. We use TextureViewRenderer (NOT SurfaceViewRenderer) because a TextureView +// composites normally in the view tree and shows through a transparent sibling on top — a SurfaceView +// punches its own window hole and can't. +// • The web polls syncVideoTiles([{uid, local, screen, x, y, w, h}]) with each tile's on-screen rect in CSS +// px; we place/size a renderer per participant that has a live camera (or screen) track, scaling CSS px → +// device px by display density, and bind the matching LiveKit track. Camera-off participants get no +// renderer, so their web avatar shows. +// +// All view work runs on the UI thread. Track lookup + renderer init go through CallMedia (which owns the room). +object CallVideo { + private const val TAG = "bzcall" + private var holePunchOn = false + private var container: FrameLayout? = null + private var savedParentBg: Int? = null + // key ("__local" | peerId) -> renderer, and the track currently bound to it (to avoid needless re-binds). + private val renderers = HashMap() + private val bound = HashMap() + + // One meeting tile from the web (coords are CSS px, viewport-relative — same space as getBoundingClientRect). + data class Tile( + val key: String, val uid: String, val isLocal: Boolean, val wantScreen: Boolean, + val x: Double, val y: Double, val w: Double, val h: Double + ) + + @JvmStatic + fun setHolePunch(activity: Activity?, webView: WebView?, on: Boolean) { + if (activity == null || webView == null) return + activity.runOnUiThread { + try { + if (on) { + webView.setBackgroundColor(Color.TRANSPARENT) + val parent = webView.parent as? ViewGroup + if (parent != null) { + if (!holePunchOn) savedParentBg = (parent.background as? ColorDrawable)?.color + parent.setBackgroundColor(Color.BLACK) + if (container == null) { + val c = FrameLayout(webView.context) + val idx = parent.indexOfChild(webView) + // Insert at the WebView's index → container sits BEHIND it (drawn first). + parent.addView(c, if (idx < 0) 0 else idx, + ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + container = c + } + } + holePunchOn = true + Log.i(TAG, "hole-punch ON (native video)") + } else { + teardown(webView) + } + } catch (e: Throwable) { Log.w(TAG, "setHolePunch($on) failed: ${e.message}") } + } + } + + @JvmStatic + fun syncTiles(activity: Activity?, webView: WebView?, tiles: List) { + if (activity == null || webView == null) return + activity.runOnUiThread { + val c = container ?: return@runOnUiThread + val density = webView.resources.displayMetrics.density + val wanted = HashSet() + for (t in tiles) { + if (t.w < 2 || t.h < 2) continue + val track = (if (t.wantScreen) CallMedia.screenTrack(t.uid) else CallMedia.cameraTrack(t.uid, t.isLocal)) ?: continue + wanted.add(t.key) + var r = renderers[t.key] + if (r == null) { + r = TextureViewRenderer(webView.context) + CallMedia.initRenderer(r) + c.addView(r) + renderers[t.key] = r + } + r.setMirror(t.isLocal && !t.wantScreen) // mirror the local camera preview only + r.setScalingType(if (t.wantScreen) RendererCommon.ScalingType.SCALE_ASPECT_FIT else RendererCommon.ScalingType.SCALE_ASPECT_FILL) + val lp = FrameLayout.LayoutParams((t.w * density).toInt(), (t.h * density).toInt()) + lp.leftMargin = (t.x * density).toInt() + lp.topMargin = (t.y * density).toInt() + r.layoutParams = lp + val cur = bound[t.key] + if (cur !== track) { + if (cur != null) try { cur.removeRenderer(r) } catch (_: Throwable) {} + try { track.addRenderer(r) } catch (_: Throwable) {} + bound[t.key] = track + } + } + // Drop renderers for tiles that are gone or whose camera turned off (so the web avatar shows). + val it = renderers.entries.iterator() + while (it.hasNext()) { + val e = it.next() + if (!wanted.contains(e.key)) { + val r = e.value + val tr = bound.remove(e.key) + if (tr != null) try { tr.removeRenderer(r) } catch (_: Throwable) {} + try { c.removeView(r) } catch (_: Throwable) {} + try { r.release() } catch (_: Throwable) {} + it.remove() + } + } + } + } + + private fun teardown(webView: WebView) { + for ((key, r) in renderers) { + val tr = bound[key] + if (tr != null) try { tr.removeRenderer(r) } catch (_: Throwable) {} + try { r.release() } catch (_: Throwable) {} + } + renderers.clear() + bound.clear() + val c = container + if (c != null) { try { (c.parent as? ViewGroup)?.removeView(c) } catch (_: Throwable) {}; container = null } + try { (webView.parent as? ViewGroup)?.setBackgroundColor(savedParentBg ?: Color.TRANSPARENT) } catch (_: Throwable) {} + try { webView.setBackgroundColor(Color.WHITE) } catch (_: Throwable) {} // restore the normal (opaque) app background + holePunchOn = false + Log.i(TAG, "hole-punch OFF") + } +} diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java index 0beb39c..11613f3 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java @@ -14,6 +14,8 @@ import com.getcapacitor.PluginMethod; import com.getcapacitor.annotation.CapacitorPlugin; import com.google.firebase.messaging.FirebaseMessaging; +import org.json.JSONObject; + // Native calling for Biz Connect (Android). Registered by `cap sync` as window.Capacitor.Plugins.NativeCall — // the SAME jsName the iOS plugin uses, so the web bridge (setupNativeCall in home.html) drives both platforms // through one code path. @@ -86,6 +88,18 @@ public class NativeCallPlugin extends Plugin { } catch (Exception ignored) {} } + // Native camera capture (Increment 2b) needs the CAMERA runtime grant. Best-effort like the mic: prompt + // without blocking; the grant persists so the camera works from this toggle on (or the next). + private void ensureCameraPermission() { + try { + if (getContext() != null + && ContextCompat.checkSelfPermission(getContext(), Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED + && getActivity() != null) { + ActivityCompat.requestPermissions(getActivity(), new String[]{ Manifest.permission.CAMERA }, 7732); + } + } catch (Exception ignored) {} + } + @Override protected void handleOnDestroy() { super.handleOnDestroy(); @@ -156,6 +170,7 @@ public class NativeCallPlugin extends Plugin { if (!uuid.isEmpty()) CallStore.markEnded(uuid); CallStore.cancel(getContext()); IncomingCallActivity.dismissIfShowing(); + try { CallVideo.setHolePunch(getActivity(), getBridge() != null ? getBridge().getWebView() : null, false); } catch (Exception ignored) {} CallMedia.disconnect(); CallForegroundService.stop(getContext()); call.resolve(); @@ -208,17 +223,65 @@ public class NativeCallPlugin extends Plugin { CallMedia.connect(getContext(), url, token, true); // start muted; web re-applies on callConnected } - // --- Increment 2b methods (video tiles / screen-share / hole-punch / #5 transcript). Safe no-ops for now so - // the shared web bridge never rejects on Android; real bodies land with the native-video build. --- - @PluginMethod public void setCamera(PluginCall call) { call.resolve(); } - @PluginMethod public void switchCamera(PluginCall call) { call.resolve(); } + // --- Increment 2b: native video tiles (camera + hole-punch rendering). --- + + // Publish/unpublish the local camera. The web re-syncs tiles right after this resolves so the local tile + // picks up (or drops) the native video. + @PluginMethod + public void setCamera(PluginCall call) { + boolean on = Boolean.TRUE.equals(call.getBoolean("on", false)); + if (on) ensureCameraPermission(); + CallMedia.setCameraEnabled(on); + JSObject ret = new JSObject(); + ret.put("on", on); + call.resolve(ret); + } + + // Flip the local camera front/back. + @PluginMethod + public void switchCamera(PluginCall call) { + CallMedia.switchCamera(); + call.resolve(); + } + + // Hole-punch: transparent WebView over native video behind it. Toggled by the web when a native call's + // meeting view is shown/left (gated on the ANDROID_NATIVE_VIDEO server flag). + @PluginMethod + public void setHolePunch(PluginCall call) { + boolean on = Boolean.TRUE.equals(call.getBoolean("on", false)); + CallVideo.setHolePunch(getActivity(), getBridge() != null ? getBridge().getWebView() : null, on); + call.resolve(); + } + + // Position native video renderers to match the web meeting tiles. tiles = [{uid, local, screen, x, y, w, h}] + // in CSS px. Polled by the web while a native call is on screen. + @PluginMethod + public void syncVideoTiles(PluginCall call) { + java.util.List list = new java.util.ArrayList<>(); + JSArray arr = call.getArray("tiles"); + if (arr != null) { + for (int i = 0; i < arr.length(); i++) { + JSONObject o = arr.optJSONObject(i); + if (o == null) continue; + String uid = o.optString("uid", ""); + if (uid.isEmpty()) continue; + boolean local = o.optBoolean("local", false); + boolean screen = o.optBoolean("screen", false); + String key = local ? "__local" : uid; + list.add(new CallVideo.Tile(key, uid, local, screen, + o.optDouble("x", 0), o.optDouble("y", 0), o.optDouble("w", 0), o.optDouble("h", 0))); + } + } + CallVideo.syncTiles(getActivity(), getBridge() != null ? getBridge().getWebView() : null, list); + call.resolve(); + } + + // --- Still no-ops: screen-share (MediaProjection) + tile zoom + #5 transcript land in a later increment. --- @PluginMethod public void startScreenShare(PluginCall call) { call.resolve(); } @PluginMethod public void stopScreenShare(PluginCall call) { call.resolve(); } @PluginMethod public void startMeetingScreenShare(PluginCall call) { call.resolve(); } @PluginMethod public void stopMeetingScreenShare(PluginCall call) { call.resolve(); } - @PluginMethod public void setHolePunch(PluginCall call) { call.resolve(); } @PluginMethod public void setTileZoom(PluginCall call) { call.resolve(); } - @PluginMethod public void syncVideoTiles(PluginCall call) { call.resolve(); } @PluginMethod public void startTranscription(PluginCall call) { call.resolve(); } @PluginMethod public void stopTranscription(PluginCall call) { call.resolve(); } @PluginMethod public void feedAudio(PluginCall call) { call.resolve(); }