android(call): Increment 2c — screen-share + received-screen pinch-zoom
Screen-share (send): startScreenShare requests the MediaProjection grant via
startActivityForResult, then CallMedia.setScreenShareEnabled(true,
ScreenCaptureParams(data)) publishes it over the native LiveKit room.
stopScreenShare unpublishes. Both emit screenShareState so the web button
reflects reality (incl. a declined grant). LiveKit's SDK manifest already
provides the mediaProjection foreground service + FOREGROUND_SERVICE_MEDIA_
PROJECTION (merged into the app), so no manifest change is needed.
Received shared screen: implemented setTileZoom (was a no-op) — the web
captures pinch/pan under hole-punch and forwards {scale,tx,ty}; CallVideo
applies it as a renderer transform (scale about centre + translate ×density),
scale<=1 clears it. Mirrors the iOS applyZoom contract. Screen tiles already
render aspect-fit.
APK-only: the web bridge already calls these methods and received-screen
rendering is gated on ANDROID_NATIVE_VIDEO (already on) — no server/web change.
All LiveKit 2.29.2 APIs (setScreenShareEnabled, ScreenCaptureParams) verified
against source. Clarity-when-zoomed (adaptive-stream) is a follow-up if needed.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
a1f7eef28a
commit
1e01130c6f
4 files changed
+99
-6
No files matched your search
+21
@@ -1,6 +1,7 @@
|
||||
package com.bizgaze.connect.nativecall
|
||||
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.util.Log
|
||||
import com.twilio.audioswitch.AudioDevice
|
||||
import io.livekit.android.LiveKit
|
||||
@@ -12,6 +13,7 @@ import io.livekit.android.room.Room
|
||||
import io.livekit.android.room.track.LocalVideoTrack
|
||||
import io.livekit.android.room.track.Track
|
||||
import io.livekit.android.room.track.VideoTrack
|
||||
import io.livekit.android.room.track.screencapture.ScreenCaptureParams
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
@@ -169,4 +171,23 @@ object CallMedia {
|
||||
fun initRenderer(renderer: TextureViewRenderer) {
|
||||
try { room?.initVideoRenderer(renderer) } catch (e: Throwable) { Log.w(TAG, "initRenderer failed: ${e.message}") }
|
||||
}
|
||||
|
||||
// Publish/unpublish this device's screen (Increment 2c). `data` is the MediaProjection permission result
|
||||
// Intent from MediaProjectionManager.createScreenCaptureIntent() (the plugin requests it). LiveKit runs its
|
||||
// own `mediaProjection` foreground service (declared in the SDK manifest, merged into the app), so we just
|
||||
// hand it the permission result. Fire-and-forget; the plugin emits screenShareState to the web.
|
||||
@JvmStatic
|
||||
fun setScreenShareEnabled(enabled: Boolean, data: Intent?) {
|
||||
val r = room ?: return
|
||||
scope.launch {
|
||||
try {
|
||||
if (enabled) {
|
||||
if (data == null) return@launch
|
||||
r.localParticipant.setScreenShareEnabled(true, ScreenCaptureParams(data))
|
||||
} else {
|
||||
r.localParticipant.setScreenShareEnabled(false)
|
||||
}
|
||||
} catch (e: Throwable) { Log.w(TAG, "setScreenShareEnabled($enabled) failed: ${e.message}") }
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -119,6 +119,26 @@ object CallVideo {
|
||||
}
|
||||
}
|
||||
|
||||
// Pinch-zoom for a shared screen (Increment 2c). Under hole-punch the native video sits behind the
|
||||
// transparent WebView, so touches land on the web, which captures the pinch/pan and forwards the transform
|
||||
// here (scale 1–5, tx/ty = pan in CSS px). We apply it to the renderer as a view transform — scale around its
|
||||
// centre, then translate by tx/ty scaled to device px. scale<=1 clears it. Mirrors the iOS applyZoom().
|
||||
@JvmStatic
|
||||
fun setTileZoom(activity: Activity?, uid: String, isLocal: Boolean, scale: Double, tx: Double, ty: Double) {
|
||||
if (activity == null) return
|
||||
val key = if (isLocal) "__local" else uid
|
||||
activity.runOnUiThread {
|
||||
val r = renderers[key] ?: return@runOnUiThread
|
||||
if (scale <= 1.001) {
|
||||
r.scaleX = 1f; r.scaleY = 1f; r.translationX = 0f; r.translationY = 0f
|
||||
} else {
|
||||
val d = r.resources.displayMetrics.density
|
||||
r.scaleX = scale.toFloat(); r.scaleY = scale.toFloat()
|
||||
r.translationX = (tx * d).toFloat(); r.translationY = (ty * d).toFloat()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun teardown(webView: WebView) {
|
||||
for ((key, r) in renderers) {
|
||||
val tr = bound[key]
|
||||
|
||||
+50
-4
@@ -1,8 +1,13 @@
|
||||
package com.bizgaze.connect.nativecall;
|
||||
|
||||
import android.Manifest;
|
||||
import android.app.Activity;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.media.projection.MediaProjectionManager;
|
||||
|
||||
import androidx.activity.result.ActivityResult;
|
||||
import androidx.core.app.ActivityCompat;
|
||||
import androidx.core.content.ContextCompat;
|
||||
|
||||
@@ -11,6 +16,7 @@ import com.getcapacitor.JSObject;
|
||||
import com.getcapacitor.Plugin;
|
||||
import com.getcapacitor.PluginCall;
|
||||
import com.getcapacitor.PluginMethod;
|
||||
import com.getcapacitor.annotation.ActivityCallback;
|
||||
import com.getcapacitor.annotation.CapacitorPlugin;
|
||||
import com.google.firebase.messaging.FirebaseMessaging;
|
||||
|
||||
@@ -276,12 +282,52 @@ public class NativeCallPlugin extends Plugin {
|
||||
call.resolve();
|
||||
}
|
||||
|
||||
// --- Still no-ops: screen-share (MediaProjection) + tile zoom + #5 transcript land in a later increment. ---
|
||||
@PluginMethod public void startScreenShare(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void stopScreenShare(PluginCall call) { call.resolve(); }
|
||||
// --- Increment 2c: screen share (MediaProjection) + pinch-zoom for a received shared screen. ---
|
||||
|
||||
// Share this device's screen: ask for the MediaProjection grant, then (on the result) publish it via LiveKit.
|
||||
@PluginMethod
|
||||
public void startScreenShare(PluginCall call) {
|
||||
try {
|
||||
MediaProjectionManager mpm = (MediaProjectionManager) getContext().getSystemService(Context.MEDIA_PROJECTION_SERVICE);
|
||||
if (mpm == null) { call.reject("screen capture unavailable"); return; }
|
||||
startActivityForResult(call, mpm.createScreenCaptureIntent(), "onScreenCaptureResult");
|
||||
} catch (Exception e) { call.reject("screen share failed: " + e.getMessage()); }
|
||||
}
|
||||
|
||||
// MediaProjection permission result. On grant, publish the screen (LiveKit runs its own mediaProjection
|
||||
// foreground service). Emit screenShareState either way so the web button reflects reality (incl. a decline).
|
||||
@ActivityCallback
|
||||
private void onScreenCaptureResult(PluginCall call, ActivityResult result) {
|
||||
if (call == null) return;
|
||||
boolean ok = result != null && result.getResultCode() == Activity.RESULT_OK && result.getData() != null;
|
||||
if (ok) CallMedia.setScreenShareEnabled(true, result.getData());
|
||||
notifyListeners("screenShareState", new JSObject().put("sharing", ok));
|
||||
call.resolve();
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void stopScreenShare(PluginCall call) {
|
||||
CallMedia.setScreenShareEnabled(false, null);
|
||||
notifyListeners("screenShareState", new JSObject().put("sharing", false));
|
||||
call.resolve();
|
||||
}
|
||||
|
||||
// Pinch-zoom a received shared screen (web captures the gesture under hole-punch and forwards the transform).
|
||||
@PluginMethod
|
||||
public void setTileZoom(PluginCall call) {
|
||||
String uid = call.getString("uid", "");
|
||||
boolean local = Boolean.TRUE.equals(call.getBoolean("local", false));
|
||||
Double scale = call.getDouble("scale", 1.0);
|
||||
Double tx = call.getDouble("tx", 0.0);
|
||||
Double ty = call.getDouble("ty", 0.0);
|
||||
CallVideo.setTileZoom(getActivity(), uid, local,
|
||||
scale == null ? 1.0 : scale, tx == null ? 0.0 : tx, ty == null ? 0.0 : ty);
|
||||
call.resolve();
|
||||
}
|
||||
|
||||
// --- Still no-ops: screen share INTO a scheduled SFU meeting (iOS-only path) + #5 transcript. ---
|
||||
@PluginMethod public void startMeetingScreenShare(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void stopMeetingScreenShare(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void setTileZoom(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void startTranscription(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void stopTranscription(PluginCall call) { call.resolve(); }
|
||||
@PluginMethod public void feedAudio(PluginCall call) { call.resolve(); }
|
||||
|
||||
Reference in new issue
Block a user