From 1e01130c6fa82afa8ab87abee1db298603991f80 Mon Sep 17 00:00:00 2001 From: sravan Date: Tue, 3 Nov 2026 22:14:42 +0530 Subject: [PATCH] =?UTF-8?q?android(call):=20Increment=202c=20=E2=80=94=20s?= =?UTF-8?q?creen-share=20+=20received-screen=20pinch-zoom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screen-share (send): startScreenShare requests the MediaProjection grant via startActivityForResult, then CallMedia.setScreenShareEnabled(true, ScreenCaptureParams(data)) publishes it over the native LiveKit room. stopScreenShare unpublishes. Both emit screenShareState so the web button reflects reality (incl. a declined grant). LiveKit's SDK manifest already provides the mediaProjection foreground service + FOREGROUND_SERVICE_MEDIA_ PROJECTION (merged into the app), so no manifest change is needed. Received shared screen: implemented setTileZoom (was a no-op) — the web captures pinch/pan under hole-punch and forwards {scale,tx,ty}; CallVideo applies it as a renderer transform (scale about centre + translate ×density), scale<=1 clears it. Mirrors the iOS applyZoom contract. Screen tiles already render aspect-fit. APK-only: the web bridge already calls these methods and received-screen rendering is gated on ANDROID_NATIVE_VIDEO (already on) — no server/web change. All LiveKit 2.29.2 APIs (setScreenShareEnabled, ScreenCaptureParams) verified against source. Clarity-when-zoomed (adaptive-stream) is a follow-up if needed. Co-Authored-By: Claude Opus 4.8 --- TECHSTACK.md | 10 +++- .../bizgaze/connect/nativecall/CallMedia.kt | 21 ++++++++ .../bizgaze/connect/nativecall/CallVideo.kt | 20 +++++++ .../connect/nativecall/NativeCallPlugin.java | 54 +++++++++++++++++-- 4 files changed, 99 insertions(+), 6 deletions(-) diff --git a/TECHSTACK.md b/TECHSTACK.md index 6c2c1ab..c3f7b8d 100644 --- a/TECHSTACK.md +++ b/TECHSTACK.md @@ -1,6 +1,6 @@ # Tech Stack — BizGaze Connect -> Living document. **Last updated: 2026-11-03** (rev 5). Update this file whenever a dependency, platform, service, or +> Living document. **Last updated: 2026-11-03** (rev 6). Update this file whenever a dependency, platform, service, or > tool is added/removed/version-bumped. See [Maintaining this file](#maintaining-this-file) and the > [Changelog](#changelog) at the bottom. @@ -111,7 +111,9 @@ hash, per-user salt, timing-safe compare), opaque random session tokens, SHA-256 **audio** (Kotlin `CallMedia` + a `microphone` foreground service for background audio + `AudioSwitchHandler` routing), gated by the server flag `ANDROID_NATIVE_MEDIA`. **Increment 2b** adds native **video** (Kotlin `CallVideo`: camera publish/flip + remote/local `TextureViewRenderer` tiles hole-punched behind a transparent - WebView), gated by `ANDROID_NATIVE_VIDEO`; screen-share (MediaProjection) is still pending. + WebView), gated by `ANDROID_NATIVE_VIDEO`. **Increment 2c** adds **screen-share** (MediaProjection grant → + LiveKit `setScreenShareEnabled`; LiveKit's own `mediaProjection` foreground service) + **pinch-zoom** of a + received shared screen (web forwards the gesture under hole-punch → renderer transform). - `audio-route` (iOS audio output routing), `media-library` (save to Photos/gallery), `share-inbox` (inbound share sheet), `file-opener` (native Quick Look / open-with). - **iOS build:** Capacitor 8 + **SPM** (CocoaPods retired); Xcode 26; a share extension + a broadcast extension. @@ -203,6 +205,10 @@ Bump the **Last updated** date at the top and add a line to the Changelog below. ## Changelog +- **2026-11-03 (rev 6)** — Android native call **screen-share** (Increment 2c): `CallMedia.setScreenShareEnabled` + (MediaProjection → LiveKit; SDK-provided `mediaProjection` foreground service) + plugin `startScreenShare`/ + `stopScreenShare` (permission via `startActivityForResult`), and **pinch-zoom** for a received shared screen + (`CallVideo.setTileZoom`). APK-only — no server/web change. - **2026-11-03 (rev 5)** — Reliability: Postgres pool safety timeouts (`connectionTimeoutMillis`, `statement_timeout`, `idle_in_transaction_session_timeout`, max 10→20) so a saturated pool fails fast instead of hanging forever; and a **pre-deploy HTML syntax guard** (`server/tools/check-syntax.js`) run in the Docker diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt index 266a2eb..1295247 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallMedia.kt @@ -1,6 +1,7 @@ package com.bizgaze.connect.nativecall import android.content.Context +import android.content.Intent import android.util.Log import com.twilio.audioswitch.AudioDevice import io.livekit.android.LiveKit @@ -12,6 +13,7 @@ import io.livekit.android.room.Room import io.livekit.android.room.track.LocalVideoTrack import io.livekit.android.room.track.Track import io.livekit.android.room.track.VideoTrack +import io.livekit.android.room.track.screencapture.ScreenCaptureParams import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob @@ -169,4 +171,23 @@ object CallMedia { fun initRenderer(renderer: TextureViewRenderer) { try { room?.initVideoRenderer(renderer) } catch (e: Throwable) { Log.w(TAG, "initRenderer failed: ${e.message}") } } + + // Publish/unpublish this device's screen (Increment 2c). `data` is the MediaProjection permission result + // Intent from MediaProjectionManager.createScreenCaptureIntent() (the plugin requests it). LiveKit runs its + // own `mediaProjection` foreground service (declared in the SDK manifest, merged into the app), so we just + // hand it the permission result. Fire-and-forget; the plugin emits screenShareState to the web. + @JvmStatic + fun setScreenShareEnabled(enabled: Boolean, data: Intent?) { + val r = room ?: return + scope.launch { + try { + if (enabled) { + if (data == null) return@launch + r.localParticipant.setScreenShareEnabled(true, ScreenCaptureParams(data)) + } else { + r.localParticipant.setScreenShareEnabled(false) + } + } catch (e: Throwable) { Log.w(TAG, "setScreenShareEnabled($enabled) failed: ${e.message}") } + } + } } diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt index fd3f9ed..7fc2bd8 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/CallVideo.kt @@ -119,6 +119,26 @@ object CallVideo { } } + // Pinch-zoom for a shared screen (Increment 2c). Under hole-punch the native video sits behind the + // transparent WebView, so touches land on the web, which captures the pinch/pan and forwards the transform + // here (scale 1–5, tx/ty = pan in CSS px). We apply it to the renderer as a view transform — scale around its + // centre, then translate by tx/ty scaled to device px. scale<=1 clears it. Mirrors the iOS applyZoom(). + @JvmStatic + fun setTileZoom(activity: Activity?, uid: String, isLocal: Boolean, scale: Double, tx: Double, ty: Double) { + if (activity == null) return + val key = if (isLocal) "__local" else uid + activity.runOnUiThread { + val r = renderers[key] ?: return@runOnUiThread + if (scale <= 1.001) { + r.scaleX = 1f; r.scaleY = 1f; r.translationX = 0f; r.translationY = 0f + } else { + val d = r.resources.displayMetrics.density + r.scaleX = scale.toFloat(); r.scaleY = scale.toFloat() + r.translationX = (tx * d).toFloat(); r.translationY = (ty * d).toFloat() + } + } + } + private fun teardown(webView: WebView) { for ((key, r) in renderers) { val tr = bound[key] diff --git a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java index 11613f3..c163579 100644 --- a/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java +++ b/mobile/plugins/native-call/android/src/main/java/com/bizgaze/connect/nativecall/NativeCallPlugin.java @@ -1,8 +1,13 @@ package com.bizgaze.connect.nativecall; import android.Manifest; +import android.app.Activity; +import android.content.Context; +import android.content.Intent; import android.content.pm.PackageManager; +import android.media.projection.MediaProjectionManager; +import androidx.activity.result.ActivityResult; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; @@ -11,6 +16,7 @@ import com.getcapacitor.JSObject; import com.getcapacitor.Plugin; import com.getcapacitor.PluginCall; import com.getcapacitor.PluginMethod; +import com.getcapacitor.annotation.ActivityCallback; import com.getcapacitor.annotation.CapacitorPlugin; import com.google.firebase.messaging.FirebaseMessaging; @@ -276,12 +282,52 @@ public class NativeCallPlugin extends Plugin { call.resolve(); } - // --- Still no-ops: screen-share (MediaProjection) + tile zoom + #5 transcript land in a later increment. --- - @PluginMethod public void startScreenShare(PluginCall call) { call.resolve(); } - @PluginMethod public void stopScreenShare(PluginCall call) { call.resolve(); } + // --- Increment 2c: screen share (MediaProjection) + pinch-zoom for a received shared screen. --- + + // Share this device's screen: ask for the MediaProjection grant, then (on the result) publish it via LiveKit. + @PluginMethod + public void startScreenShare(PluginCall call) { + try { + MediaProjectionManager mpm = (MediaProjectionManager) getContext().getSystemService(Context.MEDIA_PROJECTION_SERVICE); + if (mpm == null) { call.reject("screen capture unavailable"); return; } + startActivityForResult(call, mpm.createScreenCaptureIntent(), "onScreenCaptureResult"); + } catch (Exception e) { call.reject("screen share failed: " + e.getMessage()); } + } + + // MediaProjection permission result. On grant, publish the screen (LiveKit runs its own mediaProjection + // foreground service). Emit screenShareState either way so the web button reflects reality (incl. a decline). + @ActivityCallback + private void onScreenCaptureResult(PluginCall call, ActivityResult result) { + if (call == null) return; + boolean ok = result != null && result.getResultCode() == Activity.RESULT_OK && result.getData() != null; + if (ok) CallMedia.setScreenShareEnabled(true, result.getData()); + notifyListeners("screenShareState", new JSObject().put("sharing", ok)); + call.resolve(); + } + + @PluginMethod + public void stopScreenShare(PluginCall call) { + CallMedia.setScreenShareEnabled(false, null); + notifyListeners("screenShareState", new JSObject().put("sharing", false)); + call.resolve(); + } + + // Pinch-zoom a received shared screen (web captures the gesture under hole-punch and forwards the transform). + @PluginMethod + public void setTileZoom(PluginCall call) { + String uid = call.getString("uid", ""); + boolean local = Boolean.TRUE.equals(call.getBoolean("local", false)); + Double scale = call.getDouble("scale", 1.0); + Double tx = call.getDouble("tx", 0.0); + Double ty = call.getDouble("ty", 0.0); + CallVideo.setTileZoom(getActivity(), uid, local, + scale == null ? 1.0 : scale, tx == null ? 0.0 : tx, ty == null ? 0.0 : ty); + call.resolve(); + } + + // --- Still no-ops: screen share INTO a scheduled SFU meeting (iOS-only path) + #5 transcript. --- @PluginMethod public void startMeetingScreenShare(PluginCall call) { call.resolve(); } @PluginMethod public void stopMeetingScreenShare(PluginCall call) { call.resolve(); } - @PluginMethod public void setTileZoom(PluginCall call) { call.resolve(); } @PluginMethod public void startTranscription(PluginCall call) { call.resolve(); } @PluginMethod public void stopTranscription(PluginCall call) { call.resolve(); } @PluginMethod public void feedAudio(PluginCall call) { call.resolve(); }