feat(remote-control): viewer can control a desktop sharer's screen, with consent (0.1.13/batch70)
Fixes the core "viewer can't control the sharer even on desktop" gap. Three root
causes addressed:
- share.html DISCARDED every input-channel message (onmessage=()=>{}). It now
parses the viewer's mouse/keyboard events and forwards them to the desktop shell.
- The main desktop app had NO OS injector (it lived only in the separate agent).
Ported the nut-js injector (agent/input/inject.js) into desktop/input, wired an
inject IPC + injectInput bridge, HARD-gated behind a consent flag (rcArmed).
- /share runs in an iframe (no direct bridge access) → it postMessages input to
the top frame (home.html), which relays to the native bridge.
Consent + safety: the sharer sees an Allow/Deny prompt the first time the agent
interacts; while active a persistent "your screen is being controlled — Stop"
banner; instant revoke; auto-release on session end/teardown. Browser sharers stay
view-only (no OS injection possible). nut-js is an optionalDependency (N-API, ABI-
stable across Electron) — degrades to no-op if the native module is unavailable.
Windows-first; maps to the primary display.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
// OS input injection layer.
|
||||
//
|
||||
// Cross-platform mouse/keyboard control via @nut-tree-fork/nut-js (optional
|
||||
// native dependency). If nut-js isn't installed (e.g. CI, or a sandbox without
|
||||
// a display), this module degrades to a logging no-op so the rest of the agent
|
||||
// still runs and can be tested. On Windows, nut-js drives the Win32 SendInput
|
||||
// API under the hood — the same mechanism TeamViewer/AnyDesk use.
|
||||
|
||||
let nut = null;
|
||||
try {
|
||||
// eslint-disable-next-line import/no-extraneous-dependencies
|
||||
nut = require('@nut-tree-fork/nut-js');
|
||||
nut.mouse.config.autoDelayMs = 0;
|
||||
nut.keyboard.config.autoDelayMs = 0;
|
||||
} catch {
|
||||
nut = null;
|
||||
}
|
||||
|
||||
const available = !!nut;
|
||||
|
||||
// Map browser KeyboardEvent.key values to nut-js Key enum names.
|
||||
function mapKey(key, code) {
|
||||
if (!nut) return null;
|
||||
const K = nut.Key;
|
||||
const direct = {
|
||||
'Enter': K.Enter, 'Backspace': K.Backspace, 'Tab': K.Tab, 'Escape': K.Escape,
|
||||
' ': K.Space, 'ArrowLeft': K.Left, 'ArrowRight': K.Right, 'ArrowUp': K.Up, 'ArrowDown': K.Down,
|
||||
'Home': K.Home, 'End': K.End, 'PageUp': K.PageUp, 'PageDown': K.PageDown, 'Delete': K.Delete,
|
||||
'Control': K.LeftControl, 'Shift': K.LeftShift, 'Alt': K.LeftAlt, 'Meta': K.LeftSuper,
|
||||
'CapsLock': K.CapsLock,
|
||||
};
|
||||
if (direct[key] !== undefined) return [direct[key]];
|
||||
if (/^F\d{1,2}$/.test(key) && K[key] !== undefined) return [K[key]];
|
||||
if (key && key.length === 1) {
|
||||
const upper = key.toUpperCase();
|
||||
if (/[A-Z]/.test(upper) && K[upper] !== undefined) return [K[upper]];
|
||||
if (/[0-9]/.test(key) && K['Num' + key] !== undefined) return [K['Num' + key]];
|
||||
// Fall back to typing the literal character (handles symbols/shifted chars)
|
||||
return { type: key };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Cache the screen size (this nut-js exposes screen.width()/height(), not getResolution()).
|
||||
// Recomputed once per session (cleared in releaseAll) so a resolution change is picked up.
|
||||
let _screen = null;
|
||||
async function screenSize() {
|
||||
if (!_screen) _screen = { w: await nut.screen.width(), h: await nut.screen.height() };
|
||||
return _screen;
|
||||
}
|
||||
async function moveTo(xNorm, yNorm) {
|
||||
if (!nut) return;
|
||||
const { w, h } = await screenSize();
|
||||
await nut.mouse.setPosition(new nut.Point(Math.round(xNorm * w), Math.round(yNorm * h)));
|
||||
}
|
||||
|
||||
function buttonEnum(b) {
|
||||
if (!nut) return null;
|
||||
return b === 2 ? nut.Button.RIGHT : b === 1 ? nut.Button.MIDDLE : nut.Button.LEFT;
|
||||
}
|
||||
|
||||
const pressed = new Set();
|
||||
|
||||
// Inject a single normalized input event coming from the viewer.
|
||||
async function inject(evt) {
|
||||
if (!nut) {
|
||||
if (evt.kind !== 'mousemove') console.log('[input:noop]', JSON.stringify(evt));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
switch (evt.kind) {
|
||||
case 'mousemove':
|
||||
await moveTo(evt.x, evt.y); break;
|
||||
case 'mousedown':
|
||||
await moveTo(evt.x, evt.y); await nut.mouse.pressButton(buttonEnum(evt.button)); break;
|
||||
case 'mouseup':
|
||||
await nut.mouse.releaseButton(buttonEnum(evt.button)); break;
|
||||
case 'dblclick':
|
||||
await moveTo(evt.x, evt.y); await nut.mouse.doubleClick(nut.Button.LEFT); break;
|
||||
case 'scroll':
|
||||
if (evt.dy) await (evt.dy > 0 ? nut.mouse.scrollDown(Math.abs(evt.dy)) : nut.mouse.scrollUp(Math.abs(evt.dy)));
|
||||
if (evt.dx) await (evt.dx > 0 ? nut.mouse.scrollRight(Math.abs(evt.dx)) : nut.mouse.scrollLeft(Math.abs(evt.dx)));
|
||||
break;
|
||||
case 'keydown': {
|
||||
const m = mapKey(evt.key, evt.code);
|
||||
if (!m) break;
|
||||
if (m.type) { await nut.keyboard.type(m.type); break; }
|
||||
await nut.keyboard.pressKey(...m); m.forEach((k) => pressed.add(k));
|
||||
break;
|
||||
}
|
||||
case 'keyup': {
|
||||
const m = mapKey(evt.key, evt.code);
|
||||
if (!m || m.type) break;
|
||||
await nut.keyboard.releaseKey(...m); m.forEach((k) => pressed.delete(k));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[input] inject error:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Safety: release any stuck modifier keys when a session ends.
|
||||
async function releaseAll() {
|
||||
if (!nut) { pressed.clear(); return; }
|
||||
for (const k of pressed) { try { await nut.keyboard.releaseKey(k); } catch {} }
|
||||
pressed.clear();
|
||||
_screen = null;
|
||||
}
|
||||
|
||||
module.exports = { inject, releaseAll, available, mapKey };
|
||||
@@ -83,6 +83,22 @@ function avatarToTempPng(src) {
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Remote control: OS input injection for a screen the local user is SHARING ----
|
||||
// The renderer (share flow) forwards a viewer's mouse/keyboard events here for injection. Injection is
|
||||
// HARD-GATED behind an explicit consent flag (rcArmed): nothing is injected until the local user clicks
|
||||
// "Allow control", and it stops the instant they revoke or the session ends. nut-js is optional — if the
|
||||
// native module isn't present it degrades to a no-op (no crash), so control simply won't take effect.
|
||||
let injector = null;
|
||||
try { injector = require('./input/inject'); } catch (_) { injector = null; }
|
||||
let rcArmed = false;
|
||||
// The renderer arms/disarms control (mirrors the on-screen consent banner). Disarming releases any
|
||||
// stuck keys immediately.
|
||||
ipcMain.on('rc-arm', (_e, on) => { rcArmed = !!on; if (!rcArmed && injector && injector.releaseAll) { try { injector.releaseAll(); } catch (_) {} } });
|
||||
ipcMain.on('rc-input', (_e, evt) => { if (rcArmed && injector && injector.inject && evt) { try { injector.inject(evt); } catch (_) {} } });
|
||||
// Whether OS injection is even possible on this machine (native module loaded). The renderer uses this
|
||||
// to show "control needs the desktop app" vs an actual Allow prompt.
|
||||
ipcMain.on('rc-available', (e) => { e.returnValue = !!(injector && injector.available); });
|
||||
|
||||
// Pre-warm the DP cache for the renderer's contacts (called after chats load), so the FIRST
|
||||
// notification from anyone already has their photo — no per-toast download wait.
|
||||
ipcMain.handle('precache-avatars', async (_e, urls = []) => {
|
||||
|
||||
Generated
+1457
-58
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "biz-connect-desktop",
|
||||
"version": "0.1.12",
|
||||
"version": "0.1.13",
|
||||
"description": "Biz Connect technician desktop client — loads the Connect web UI with native screen capture",
|
||||
"author": {
|
||||
"name": "BizGaze",
|
||||
@@ -14,6 +14,9 @@
|
||||
"dependencies": {
|
||||
"electron-updater": "^6.3.9"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@nut-tree-fork/nut-js": "^4.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^31.0.0",
|
||||
"electron-builder": "^24.13.3"
|
||||
|
||||
@@ -28,6 +28,12 @@ contextBridge.exposeInMainWorld('bizConnectNative', Object.freeze({
|
||||
// Ask the shell to show native spelling suggestions for the word at page coords (x,y) — used to bring
|
||||
// up corrections on a LEFT click in the message box (not just right-click).
|
||||
spellSuggestAt: (x, y) => { try { ipcRenderer.send('spell-suggest', { x, y }); } catch (_) {} },
|
||||
// Remote control (screen the local user is sharing): whether OS injection is possible on this machine,
|
||||
// arm/disarm the consent gate, and forward a viewer's input event for injection. Injection only happens
|
||||
// while armed (the user granted control) — see main.js rcArmed.
|
||||
rcAvailable: () => { try { return !!ipcRenderer.sendSync('rc-available'); } catch (_) { return false; } },
|
||||
rcArm: (on) => { try { ipcRenderer.send('rc-arm', !!on); } catch (_) {} },
|
||||
rcInput: (evt) => { try { ipcRenderer.send('rc-input', evt); } catch (_) {} },
|
||||
// Manual "Check for updates" from Settings. Resolves {status:'available'|'current'|'dev'|'error', version?}.
|
||||
// On 'available' the shell downloads in the background and prompts to restart when ready.
|
||||
checkForUpdates: () => ipcRenderer.invoke('check-updates'),
|
||||
|
||||
Reference in New Issue
Block a user