feat(remote-control): viewer can control a desktop sharer's screen, with consent (0.1.13/batch70)
Fixes the core "viewer can't control the sharer even on desktop" gap. Three root
causes addressed:
- share.html DISCARDED every input-channel message (onmessage=()=>{}). It now
parses the viewer's mouse/keyboard events and forwards them to the desktop shell.
- The main desktop app had NO OS injector (it lived only in the separate agent).
Ported the nut-js injector (agent/input/inject.js) into desktop/input, wired an
inject IPC + injectInput bridge, HARD-gated behind a consent flag (rcArmed).
- /share runs in an iframe (no direct bridge access) → it postMessages input to
the top frame (home.html), which relays to the native bridge.
Consent + safety: the sharer sees an Allow/Deny prompt the first time the agent
interacts; while active a persistent "your screen is being controlled — Stop"
banner; instant revoke; auto-release on session end/teardown. Browser sharers stay
view-only (no OS injection possible). nut-js is an optionalDependency (N-API, ABI-
stable across Electron) — degrades to no-op if the native module is unavailable.
Windows-first; maps to the primary display.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
// OS input injection layer.
|
||||
//
|
||||
// Cross-platform mouse/keyboard control via @nut-tree-fork/nut-js (optional
|
||||
// native dependency). If nut-js isn't installed (e.g. CI, or a sandbox without
|
||||
// a display), this module degrades to a logging no-op so the rest of the agent
|
||||
// still runs and can be tested. On Windows, nut-js drives the Win32 SendInput
|
||||
// API under the hood — the same mechanism TeamViewer/AnyDesk use.
|
||||
|
||||
let nut = null;
|
||||
try {
|
||||
// eslint-disable-next-line import/no-extraneous-dependencies
|
||||
nut = require('@nut-tree-fork/nut-js');
|
||||
nut.mouse.config.autoDelayMs = 0;
|
||||
nut.keyboard.config.autoDelayMs = 0;
|
||||
} catch {
|
||||
nut = null;
|
||||
}
|
||||
|
||||
const available = !!nut;
|
||||
|
||||
// Map browser KeyboardEvent.key values to nut-js Key enum names.
|
||||
function mapKey(key, code) {
|
||||
if (!nut) return null;
|
||||
const K = nut.Key;
|
||||
const direct = {
|
||||
'Enter': K.Enter, 'Backspace': K.Backspace, 'Tab': K.Tab, 'Escape': K.Escape,
|
||||
' ': K.Space, 'ArrowLeft': K.Left, 'ArrowRight': K.Right, 'ArrowUp': K.Up, 'ArrowDown': K.Down,
|
||||
'Home': K.Home, 'End': K.End, 'PageUp': K.PageUp, 'PageDown': K.PageDown, 'Delete': K.Delete,
|
||||
'Control': K.LeftControl, 'Shift': K.LeftShift, 'Alt': K.LeftAlt, 'Meta': K.LeftSuper,
|
||||
'CapsLock': K.CapsLock,
|
||||
};
|
||||
if (direct[key] !== undefined) return [direct[key]];
|
||||
if (/^F\d{1,2}$/.test(key) && K[key] !== undefined) return [K[key]];
|
||||
if (key && key.length === 1) {
|
||||
const upper = key.toUpperCase();
|
||||
if (/[A-Z]/.test(upper) && K[upper] !== undefined) return [K[upper]];
|
||||
if (/[0-9]/.test(key) && K['Num' + key] !== undefined) return [K['Num' + key]];
|
||||
// Fall back to typing the literal character (handles symbols/shifted chars)
|
||||
return { type: key };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Cache the screen size (this nut-js exposes screen.width()/height(), not getResolution()).
|
||||
// Recomputed once per session (cleared in releaseAll) so a resolution change is picked up.
|
||||
let _screen = null;
|
||||
async function screenSize() {
|
||||
if (!_screen) _screen = { w: await nut.screen.width(), h: await nut.screen.height() };
|
||||
return _screen;
|
||||
}
|
||||
async function moveTo(xNorm, yNorm) {
|
||||
if (!nut) return;
|
||||
const { w, h } = await screenSize();
|
||||
await nut.mouse.setPosition(new nut.Point(Math.round(xNorm * w), Math.round(yNorm * h)));
|
||||
}
|
||||
|
||||
function buttonEnum(b) {
|
||||
if (!nut) return null;
|
||||
return b === 2 ? nut.Button.RIGHT : b === 1 ? nut.Button.MIDDLE : nut.Button.LEFT;
|
||||
}
|
||||
|
||||
const pressed = new Set();
|
||||
|
||||
// Inject a single normalized input event coming from the viewer.
|
||||
async function inject(evt) {
|
||||
if (!nut) {
|
||||
if (evt.kind !== 'mousemove') console.log('[input:noop]', JSON.stringify(evt));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
switch (evt.kind) {
|
||||
case 'mousemove':
|
||||
await moveTo(evt.x, evt.y); break;
|
||||
case 'mousedown':
|
||||
await moveTo(evt.x, evt.y); await nut.mouse.pressButton(buttonEnum(evt.button)); break;
|
||||
case 'mouseup':
|
||||
await nut.mouse.releaseButton(buttonEnum(evt.button)); break;
|
||||
case 'dblclick':
|
||||
await moveTo(evt.x, evt.y); await nut.mouse.doubleClick(nut.Button.LEFT); break;
|
||||
case 'scroll':
|
||||
if (evt.dy) await (evt.dy > 0 ? nut.mouse.scrollDown(Math.abs(evt.dy)) : nut.mouse.scrollUp(Math.abs(evt.dy)));
|
||||
if (evt.dx) await (evt.dx > 0 ? nut.mouse.scrollRight(Math.abs(evt.dx)) : nut.mouse.scrollLeft(Math.abs(evt.dx)));
|
||||
break;
|
||||
case 'keydown': {
|
||||
const m = mapKey(evt.key, evt.code);
|
||||
if (!m) break;
|
||||
if (m.type) { await nut.keyboard.type(m.type); break; }
|
||||
await nut.keyboard.pressKey(...m); m.forEach((k) => pressed.add(k));
|
||||
break;
|
||||
}
|
||||
case 'keyup': {
|
||||
const m = mapKey(evt.key, evt.code);
|
||||
if (!m || m.type) break;
|
||||
await nut.keyboard.releaseKey(...m); m.forEach((k) => pressed.delete(k));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[input] inject error:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Safety: release any stuck modifier keys when a session ends.
|
||||
async function releaseAll() {
|
||||
if (!nut) { pressed.clear(); return; }
|
||||
for (const k of pressed) { try { await nut.keyboard.releaseKey(k); } catch {} }
|
||||
pressed.clear();
|
||||
_screen = null;
|
||||
}
|
||||
|
||||
module.exports = { inject, releaseAll, available, mapKey };
|
||||
@@ -83,6 +83,22 @@ function avatarToTempPng(src) {
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Remote control: OS input injection for a screen the local user is SHARING ----
|
||||
// The renderer (share flow) forwards a viewer's mouse/keyboard events here for injection. Injection is
|
||||
// HARD-GATED behind an explicit consent flag (rcArmed): nothing is injected until the local user clicks
|
||||
// "Allow control", and it stops the instant they revoke or the session ends. nut-js is optional — if the
|
||||
// native module isn't present it degrades to a no-op (no crash), so control simply won't take effect.
|
||||
let injector = null;
|
||||
try { injector = require('./input/inject'); } catch (_) { injector = null; }
|
||||
let rcArmed = false;
|
||||
// The renderer arms/disarms control (mirrors the on-screen consent banner). Disarming releases any
|
||||
// stuck keys immediately.
|
||||
ipcMain.on('rc-arm', (_e, on) => { rcArmed = !!on; if (!rcArmed && injector && injector.releaseAll) { try { injector.releaseAll(); } catch (_) {} } });
|
||||
ipcMain.on('rc-input', (_e, evt) => { if (rcArmed && injector && injector.inject && evt) { try { injector.inject(evt); } catch (_) {} } });
|
||||
// Whether OS injection is even possible on this machine (native module loaded). The renderer uses this
|
||||
// to show "control needs the desktop app" vs an actual Allow prompt.
|
||||
ipcMain.on('rc-available', (e) => { e.returnValue = !!(injector && injector.available); });
|
||||
|
||||
// Pre-warm the DP cache for the renderer's contacts (called after chats load), so the FIRST
|
||||
// notification from anyone already has their photo — no per-toast download wait.
|
||||
ipcMain.handle('precache-avatars', async (_e, urls = []) => {
|
||||
|
||||
Generated
+1457
-58
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "biz-connect-desktop",
|
||||
"version": "0.1.12",
|
||||
"version": "0.1.13",
|
||||
"description": "Biz Connect technician desktop client — loads the Connect web UI with native screen capture",
|
||||
"author": {
|
||||
"name": "BizGaze",
|
||||
@@ -14,6 +14,9 @@
|
||||
"dependencies": {
|
||||
"electron-updater": "^6.3.9"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@nut-tree-fork/nut-js": "^4.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^31.0.0",
|
||||
"electron-builder": "^24.13.3"
|
||||
|
||||
@@ -28,6 +28,12 @@ contextBridge.exposeInMainWorld('bizConnectNative', Object.freeze({
|
||||
// Ask the shell to show native spelling suggestions for the word at page coords (x,y) — used to bring
|
||||
// up corrections on a LEFT click in the message box (not just right-click).
|
||||
spellSuggestAt: (x, y) => { try { ipcRenderer.send('spell-suggest', { x, y }); } catch (_) {} },
|
||||
// Remote control (screen the local user is sharing): whether OS injection is possible on this machine,
|
||||
// arm/disarm the consent gate, and forward a viewer's input event for injection. Injection only happens
|
||||
// while armed (the user granted control) — see main.js rcArmed.
|
||||
rcAvailable: () => { try { return !!ipcRenderer.sendSync('rc-available'); } catch (_) { return false; } },
|
||||
rcArm: (on) => { try { ipcRenderer.send('rc-arm', !!on); } catch (_) {} },
|
||||
rcInput: (evt) => { try { ipcRenderer.send('rc-input', evt); } catch (_) {} },
|
||||
// Manual "Check for updates" from Settings. Resolves {status:'available'|'current'|'dev'|'error', version?}.
|
||||
// On 'available' the shell downloads in the background and prompts to restart when ready.
|
||||
checkForUpdates: () => ipcRenderer.invoke('check-updates'),
|
||||
|
||||
+12
-1
@@ -884,7 +884,7 @@
|
||||
<body>
|
||||
<script src="/icons.js?v=5"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@twemoji/api@15.1.0/dist/twemoji.min.js" crossorigin="anonymous"></script>
|
||||
<script>window.__BUILD='2026-07-10-batch69';console.log('%cBiz Connect','color:#1F3B73;font-weight:bold','build '+window.__BUILD);
|
||||
<script>window.__BUILD='2026-07-10-batch70';console.log('%cBiz Connect','color:#1F3B73;font-weight:bold','build '+window.__BUILD);
|
||||
// Render modern (Twemoji) emojis in place of the OS's flat ones. No-op if the CDN didn't load
|
||||
// (emojis stay as plain Unicode). (#5)
|
||||
function twemojify(el){ try{ if(el && window.twemoji) window.twemoji.parse(el, { folder:'svg', ext:'.svg' }); }catch(_){} }</script>
|
||||
@@ -3757,6 +3757,17 @@ async function doRegister(){
|
||||
}catch(e){ showErr('rg_err', e.message); }
|
||||
}
|
||||
|
||||
// Relay remote-control messages from the embedded /share iframe to the native desktop shell for OS
|
||||
// injection. The iframe (share.html) can't reach the native bridge directly (it lives on this top
|
||||
// frame), so it postMessages here: we answer the availability handshake and forward arm/input events
|
||||
// only when running in the desktop app. Injection is still gated by the shell's consent flag.
|
||||
window.addEventListener('message',(e)=>{
|
||||
if(e.origin!==location.origin) return; const d=e.data||{}; const n=window.bizConnectNative;
|
||||
if(d.type==='rc-ping'){ const desktop=!!(n&&n.rcAvailable&&n.rcAvailable()); try{ e.source&&e.source.postMessage({type:'rc-pong', desktop}, location.origin); }catch(_){} return; }
|
||||
if(!n) return;
|
||||
if(d.type==='rc-arm'){ try{ n.rcArm&&n.rcArm(!!d.on); }catch(_){} return; }
|
||||
if(d.type==='rc-input'){ try{ n.rcInput&&n.rcInput(d.evt); }catch(_){} return; }
|
||||
});
|
||||
// ---------- Boot: show the app if signed in, otherwise the login ----------
|
||||
(async function(){
|
||||
let me=null;
|
||||
|
||||
@@ -58,6 +58,19 @@
|
||||
.profile .pmenu a{display:block;padding:.6rem .9rem;color:#1f2430;text-decoration:none;font-size:.9rem;cursor:pointer}
|
||||
.profile .pmenu a:hover{background:#f1f5f9}
|
||||
.profile .pmenu a.danger{color:#b91c1c;border-top:1px solid #eef1f6}
|
||||
/* Remote-control consent prompt + "being controlled" banner (sharer side) */
|
||||
.rc-consent{position:fixed;inset:0;background:rgba(15,23,42,.5);display:flex;align-items:center;justify-content:center;z-index:100000;padding:1rem;}
|
||||
.rc-consent .rc-card{background:#fff;border-radius:14px;padding:1.4rem 1.5rem;max-width:380px;width:100%;box-shadow:0 18px 44px rgba(0,0,0,.3);text-align:center;font-family:'Segoe UI',system-ui,sans-serif;}
|
||||
.rc-consent .rc-h{font-size:1.05rem;font-weight:700;color:#1F3B73;margin-bottom:.4rem;}
|
||||
.rc-consent p{font-size:.9rem;color:#475569;line-height:1.5;margin:0 0 1.1rem;}
|
||||
.rc-consent .rc-btns{display:flex;gap:.6rem;}
|
||||
.rc-consent button{flex:1;padding:.7rem;border-radius:9px;font-size:.92rem;font-weight:600;cursor:pointer;border:1px solid #e3e8f2;}
|
||||
.rc-consent .rc-deny{background:#fff;color:#334155;}
|
||||
.rc-consent .rc-allow{background:#1F3B73;color:#fff;border-color:#1F3B73;}
|
||||
.rc-banner{position:fixed;top:12px;left:50%;transform:translateX(-50%);z-index:100000;display:flex;align-items:center;gap:.6rem;background:#0f172a;color:#fff;border-radius:999px;padding:.5rem .6rem .5rem .9rem;font-family:'Segoe UI',system-ui,sans-serif;font-size:.85rem;box-shadow:0 6px 20px rgba(0,0,0,.35);}
|
||||
.rc-banner .rc-dot{width:9px;height:9px;border-radius:50%;background:#22c55e;box-shadow:0 0 0 0 rgba(34,197,94,.6);animation:rcpulse 1.6s infinite;}
|
||||
@keyframes rcpulse{0%{box-shadow:0 0 0 0 rgba(34,197,94,.6)}70%{box-shadow:0 0 0 8px rgba(34,197,94,0)}100%{box-shadow:0 0 0 0 rgba(34,197,94,0)}}
|
||||
.rc-banner button{background:#ef4444;color:#fff;border:none;border-radius:999px;padding:.35rem .8rem;font-size:.8rem;font-weight:700;cursor:pointer;}
|
||||
</style>
|
||||
<script src="/icons.js?v=3"></script>
|
||||
</head>
|
||||
@@ -180,7 +193,7 @@ async function startStreaming(){
|
||||
pc=new RTCPeerConnection(ICE);
|
||||
buildBar();
|
||||
localStream.getTracks().forEach(t=>pc.addTrack(t,localStream));
|
||||
pc.ondatachannel=(ev)=>{ev.channel.onmessage=()=>{};};
|
||||
pc.ondatachannel=(ev)=>{ const ch=ev.channel; if(ch&&ch.label==='input'){ ch.onmessage=(e)=>rcOnInput(e.data); } else if(ch){ ch.onmessage=()=>{}; } };
|
||||
pc.ontrack=(ev)=>{ if(ev.track.kind==='audio'){ let a=document.getElementById('remoteAudio'); if(!a){a=document.createElement('audio');a.id='remoteAudio';a.autoplay=true;document.body.appendChild(a);} a.srcObject=ev.streams[0]; } };
|
||||
pc.onicecandidate=(ev)=>{if(ev.candidate)ws.send(JSON.stringify({type:'ice-candidate',sessionId,candidate:ev.candidate}));};
|
||||
pc.onconnectionstatechange=()=>{ if(!pc) return; if(pc.connectionState==='connected'){ clearTimeout(window.__connWatch); } if(pc.connectionState==='failed'){ clearTimeout(window.__connWatch); try{ws.send(JSON.stringify({type:'end-session',sessionId,reason:'customer-ended'}));}catch(_){} endShareSession("Couldn't connect on this network — it may be blocking screen sharing. Try a different network (e.g. mobile data / hotspot), then tap below for a new code."); } };
|
||||
@@ -207,6 +220,38 @@ function startCustTranscription(){
|
||||
}catch(e){}
|
||||
}
|
||||
function stopCustTranscription(){ crecogActive=false; if(crecog){ try{crecog.stop();}catch(_){} crecog=null; } }
|
||||
// ---- Remote control (SHARER side) ----
|
||||
// The agent (connect.html) streams mouse/keyboard events over the 'input' data channel. We forward them
|
||||
// to the Biz Connect DESKTOP shell (via the parent frame's native bridge) for OS injection — but ONLY
|
||||
// after the user explicitly taps "Allow control", and only inside the desktop app. In a plain browser
|
||||
// there's no OS injection possible, so it stays view-only. The user can stop control at any time.
|
||||
let rcDesktop=false, rcAllowed=false, rcPrompted=false;
|
||||
try{ window.addEventListener('message',(e)=>{ if(e.origin!==location.origin) return; const d=e.data||{}; if(d.type==='rc-pong') rcDesktop=!!d.desktop; }); }catch(_){}
|
||||
try{ if(window.parent && window.parent!==window) window.parent.postMessage({type:'rc-ping'}, location.origin); }catch(_){}
|
||||
function rcPost(msg){ try{ if(window.parent && window.parent!==window) window.parent.postMessage(msg, location.origin); }catch(_){} }
|
||||
function rcOnInput(data){
|
||||
let evt; try{ evt=JSON.parse(data); }catch(_){ return; }
|
||||
if(rcAllowed){ rcPost({type:'rc-input', evt}); return; }
|
||||
if(!rcDesktop) return; // browser sharer: cannot inject OS input — view-only
|
||||
// Ask for consent the first time the agent actually interacts (a click/keypress, not mere cursor moves).
|
||||
if(!rcPrompted && (evt.kind==='mousedown'||evt.kind==='keydown'||evt.kind==='dblclick')){ rcPrompted=true; showControlConsent(); }
|
||||
}
|
||||
function showControlConsent(){
|
||||
if(document.getElementById('rcConsent')) return;
|
||||
const el=document.createElement('div'); el.id='rcConsent'; el.className='rc-consent';
|
||||
el.innerHTML='<div class="rc-card"><div class="rc-h">Allow remote control?</div><p>Your agent is asking to control your mouse & keyboard to help you. You stay in charge — stop anytime.</p><div class="rc-btns"><button id="rcDeny" class="rc-deny">Not now</button><button id="rcAllow" class="rc-allow">Allow control</button></div></div>';
|
||||
document.body.appendChild(el);
|
||||
el.querySelector('#rcAllow').onclick=()=>{ rcAllowed=true; rcPost({type:'rc-arm', on:true}); el.remove(); showControlBanner(); };
|
||||
el.querySelector('#rcDeny').onclick=()=>{ el.remove(); };
|
||||
}
|
||||
function showControlBanner(){
|
||||
if(document.getElementById('rcBanner')) return;
|
||||
const el=document.createElement('div'); el.id='rcBanner'; el.className='rc-banner';
|
||||
el.innerHTML='<span class="rc-dot"></span><span>Your agent is controlling your screen</span><button id="rcStop">Stop control</button>';
|
||||
document.body.appendChild(el);
|
||||
el.querySelector('#rcStop').onclick=rcStopControl;
|
||||
}
|
||||
function rcStopControl(){ rcAllowed=false; rcPrompted=false; rcPost({type:'rc-arm', on:false}); const b=document.getElementById('rcBanner'); if(b) b.remove(); }
|
||||
let recTimerInt=null, recStartTs=0;
|
||||
function fmtElapsed(ms){const s=Math.max(0,Math.floor(ms/1000));return String(Math.floor(s/60)).padStart(2,'0')+':'+String(s%60).padStart(2,'0');}
|
||||
function recNotice(on){
|
||||
@@ -225,6 +270,7 @@ function recNotice(on){
|
||||
} else { clearInterval(recTimerInt); recTimerInt=null; if(n) n.remove(); }
|
||||
}
|
||||
function endShareSession(msgText){
|
||||
try{ rcStopControl(); }catch(_){} // release remote control when the session ends
|
||||
sessionOver=true; window.onbeforeunload=null; bzcSession(false); { const hl=document.getElementById('homeLink'); if(hl) hl.style.display=''; } try{recNotice(false);stopCustTranscription();}catch(_){}
|
||||
removeSessionUI();
|
||||
indicator.classList.remove('show');
|
||||
@@ -234,7 +280,7 @@ function endShareSession(msgText){
|
||||
var card=document.querySelector('.panelside .card');
|
||||
if(card){ card.innerHTML='<h1 style="color:var(--blue)">Session ended</h1><div class="sub">'+esc(msgText||'The session has ended.')+'</div><button onclick="location.reload()" style="width:100%;margin-top:.4rem">Get a new code</button>'; }
|
||||
}
|
||||
function teardown(){sessionOver=true;window.onbeforeunload=null;bzcSession(false);{const hl=document.getElementById('homeLink');if(hl)hl.style.display='';}try{recNotice(false);stopCustTranscription();}catch(_){}indicator.classList.remove('show');removeSessionUI();if(window.__mic){window.__mic.getTracks().forEach(t=>t.stop());window.__mic=null;}if(localStream){localStream.getTracks().forEach(t=>t.stop());localStream=null;}if(pc){pc.close();pc=null;}consentBox.innerHTML='';setStatus('Session ended. Refresh this page to get a new code.');}
|
||||
function teardown(){try{rcStopControl();}catch(_){}sessionOver=true;window.onbeforeunload=null;bzcSession(false);{const hl=document.getElementById('homeLink');if(hl)hl.style.display='';}try{recNotice(false);stopCustTranscription();}catch(_){}indicator.classList.remove('show');removeSessionUI();if(window.__mic){window.__mic.getTracks().forEach(t=>t.stop());window.__mic=null;}if(localStream){localStream.getTracks().forEach(t=>t.stop());localStream=null;}if(pc){pc.close();pc=null;}consentBox.innerHTML='';setStatus('Session ended. Refresh this page to get a new code.');}
|
||||
|
||||
let chatOpen=false;
|
||||
const SVG_MIC='<svg viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="2" width="6" height="11" rx="3"/><path d="M5 10a7 7 0 0 0 14 0"/><line x1="12" y1="19" x2="12" y2="22"/></svg>';
|
||||
|
||||
Reference in New Issue
Block a user