android(call): Increment 2c — screen-share + received-screen pinch-zoom

Screen-share (send): startScreenShare requests the MediaProjection grant via
startActivityForResult, then CallMedia.setScreenShareEnabled(true,
ScreenCaptureParams(data)) publishes it over the native LiveKit room.
stopScreenShare unpublishes. Both emit screenShareState so the web button
reflects reality (incl. a declined grant). LiveKit's SDK manifest already
provides the mediaProjection foreground service + FOREGROUND_SERVICE_MEDIA_
PROJECTION (merged into the app), so no manifest change is needed.

Received shared screen: implemented setTileZoom (was a no-op) — the web
captures pinch/pan under hole-punch and forwards {scale,tx,ty}; CallVideo
applies it as a renderer transform (scale about centre + translate ×density),
scale<=1 clears it. Mirrors the iOS applyZoom contract. Screen tiles already
render aspect-fit.

APK-only: the web bridge already calls these methods and received-screen
rendering is gated on ANDROID_NATIVE_VIDEO (already on) — no server/web change.
All LiveKit 2.29.2 APIs (setScreenShareEnabled, ScreenCaptureParams) verified
against source. Clarity-when-zoomed (adaptive-stream) is a follow-up if needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
SravanandClaude Opus 4.8 committed 2026-11-03 22:14:42 +05:30
1 parent a1f7eef28a
commit 1e01130c6f
4 files changed
+99 -6

No files matched your search

@@ -1,6 +1,7 @@
package com.bizgaze.connect.nativecall
import android.content.Context
import android.content.Intent
import android.util.Log
import com.twilio.audioswitch.AudioDevice
import io.livekit.android.LiveKit
@@ -12,6 +13,7 @@ import io.livekit.android.room.Room
import io.livekit.android.room.track.LocalVideoTrack
import io.livekit.android.room.track.Track
import io.livekit.android.room.track.VideoTrack
import io.livekit.android.room.track.screencapture.ScreenCaptureParams
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
@@ -169,4 +171,23 @@ object CallMedia {
fun initRenderer(renderer: TextureViewRenderer) {
try { room?.initVideoRenderer(renderer) } catch (e: Throwable) { Log.w(TAG, "initRenderer failed: ${e.message}") }
}
// Publish/unpublish this device's screen (Increment 2c). `data` is the MediaProjection permission result
// Intent from MediaProjectionManager.createScreenCaptureIntent() (the plugin requests it). LiveKit runs its
// own `mediaProjection` foreground service (declared in the SDK manifest, merged into the app), so we just
// hand it the permission result. Fire-and-forget; the plugin emits screenShareState to the web.
@JvmStatic
fun setScreenShareEnabled(enabled: Boolean, data: Intent?) {
val r = room ?: return
scope.launch {
try {
if (enabled) {
if (data == null) return@launch
r.localParticipant.setScreenShareEnabled(true, ScreenCaptureParams(data))
} else {
r.localParticipant.setScreenShareEnabled(false)
}
} catch (e: Throwable) { Log.w(TAG, "setScreenShareEnabled($enabled) failed: ${e.message}") }
}
}
}
@@ -119,6 +119,26 @@ object CallVideo {
}
}
// Pinch-zoom for a shared screen (Increment 2c). Under hole-punch the native video sits behind the
// transparent WebView, so touches land on the web, which captures the pinch/pan and forwards the transform
// here (scale 1–5, tx/ty = pan in CSS px). We apply it to the renderer as a view transform — scale around its
// centre, then translate by tx/ty scaled to device px. scale<=1 clears it. Mirrors the iOS applyZoom().
@JvmStatic
fun setTileZoom(activity: Activity?, uid: String, isLocal: Boolean, scale: Double, tx: Double, ty: Double) {
if (activity == null) return
val key = if (isLocal) "__local" else uid
activity.runOnUiThread {
val r = renderers[key] ?: return@runOnUiThread
if (scale <= 1.001) {
r.scaleX = 1f; r.scaleY = 1f; r.translationX = 0f; r.translationY = 0f
} else {
val d = r.resources.displayMetrics.density
r.scaleX = scale.toFloat(); r.scaleY = scale.toFloat()
r.translationX = (tx * d).toFloat(); r.translationY = (ty * d).toFloat()
}
}
}
private fun teardown(webView: WebView) {
for ((key, r) in renderers) {
val tr = bound[key]
@@ -1,8 +1,13 @@
package com.bizgaze.connect.nativecall;
import android.Manifest;
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.media.projection.MediaProjectionManager;
import androidx.activity.result.ActivityResult;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
@@ -11,6 +16,7 @@ import com.getcapacitor.JSObject;
import com.getcapacitor.Plugin;
import com.getcapacitor.PluginCall;
import com.getcapacitor.PluginMethod;
import com.getcapacitor.annotation.ActivityCallback;
import com.getcapacitor.annotation.CapacitorPlugin;
import com.google.firebase.messaging.FirebaseMessaging;
@@ -276,12 +282,52 @@ public class NativeCallPlugin extends Plugin {
call.resolve();
}
// --- Still no-ops: screen-share (MediaProjection) + tile zoom + #5 transcript land in a later increment. ---
@PluginMethod public void startScreenShare(PluginCall call) { call.resolve(); }
@PluginMethod public void stopScreenShare(PluginCall call) { call.resolve(); }
// --- Increment 2c: screen share (MediaProjection) + pinch-zoom for a received shared screen. ---
// Share this device's screen: ask for the MediaProjection grant, then (on the result) publish it via LiveKit.
@PluginMethod
public void startScreenShare(PluginCall call) {
try {
MediaProjectionManager mpm = (MediaProjectionManager) getContext().getSystemService(Context.MEDIA_PROJECTION_SERVICE);
if (mpm == null) { call.reject("screen capture unavailable"); return; }
startActivityForResult(call, mpm.createScreenCaptureIntent(), "onScreenCaptureResult");
} catch (Exception e) { call.reject("screen share failed: " + e.getMessage()); }
}
// MediaProjection permission result. On grant, publish the screen (LiveKit runs its own mediaProjection
// foreground service). Emit screenShareState either way so the web button reflects reality (incl. a decline).
@ActivityCallback
private void onScreenCaptureResult(PluginCall call, ActivityResult result) {
if (call == null) return;
boolean ok = result != null && result.getResultCode() == Activity.RESULT_OK && result.getData() != null;
if (ok) CallMedia.setScreenShareEnabled(true, result.getData());
notifyListeners("screenShareState", new JSObject().put("sharing", ok));
call.resolve();
}
@PluginMethod
public void stopScreenShare(PluginCall call) {
CallMedia.setScreenShareEnabled(false, null);
notifyListeners("screenShareState", new JSObject().put("sharing", false));
call.resolve();
}
// Pinch-zoom a received shared screen (web captures the gesture under hole-punch and forwards the transform).
@PluginMethod
public void setTileZoom(PluginCall call) {
String uid = call.getString("uid", "");
boolean local = Boolean.TRUE.equals(call.getBoolean("local", false));
Double scale = call.getDouble("scale", 1.0);
Double tx = call.getDouble("tx", 0.0);
Double ty = call.getDouble("ty", 0.0);
CallVideo.setTileZoom(getActivity(), uid, local,
scale == null ? 1.0 : scale, tx == null ? 0.0 : tx, ty == null ? 0.0 : ty);
call.resolve();
}
// --- Still no-ops: screen share INTO a scheduled SFU meeting (iOS-only path) + #5 transcript. ---
@PluginMethod public void startMeetingScreenShare(PluginCall call) { call.resolve(); }
@PluginMethod public void stopMeetingScreenShare(PluginCall call) { call.resolve(); }
@PluginMethod public void setTileZoom(PluginCall call) { call.resolve(); }
@PluginMethod public void startTranscription(PluginCall call) { call.resolve(); }
@PluginMethod public void stopTranscription(PluginCall call) { call.resolve(); }
@PluginMethod public void feedAudio(PluginCall call) { call.resolve(); }