fix(agent): remote-control input now works (data channel + nut-js API)
Two latent bugs that broke input control on any setup: - Data channel was created by the viewer (the answerer), so the agent's offer had no SCTP m-line and the channel never opened -> no input reached the agent. The agent (offerer) now creates the 'input' channel; the viewer receives it. - inject.js used nut.screen.getResolution() which doesn't exist in this nut-js; switched to screen.width()/height() with per-session caching. Verified end-to-end locally: screen streams + mouse injection moves the remote cursor. Also commits desktop/ + mobile/ package-lock.json from client installs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+10
-2
@@ -41,10 +41,17 @@ function mapKey(key, code) {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cache the screen size (this nut-js exposes screen.width()/height(), not getResolution()).
|
||||||
|
// Recomputed once per session (cleared in releaseAll) so a resolution change is picked up.
|
||||||
|
let _screen = null;
|
||||||
|
async function screenSize() {
|
||||||
|
if (!_screen) _screen = { w: await nut.screen.width(), h: await nut.screen.height() };
|
||||||
|
return _screen;
|
||||||
|
}
|
||||||
async function moveTo(xNorm, yNorm) {
|
async function moveTo(xNorm, yNorm) {
|
||||||
if (!nut) return;
|
if (!nut) return;
|
||||||
const { width, height } = await nut.screen.getResolution();
|
const { w, h } = await screenSize();
|
||||||
await nut.mouse.setPosition(new nut.Point(Math.round(xNorm * width), Math.round(yNorm * height)));
|
await nut.mouse.setPosition(new nut.Point(Math.round(xNorm * w), Math.round(yNorm * h)));
|
||||||
}
|
}
|
||||||
|
|
||||||
function buttonEnum(b) {
|
function buttonEnum(b) {
|
||||||
@@ -98,6 +105,7 @@ async function releaseAll() {
|
|||||||
if (!nut) { pressed.clear(); return; }
|
if (!nut) { pressed.clear(); return; }
|
||||||
for (const k of pressed) { try { await nut.keyboard.releaseKey(k); } catch {} }
|
for (const k of pressed) { try { await nut.keyboard.releaseKey(k); } catch {} }
|
||||||
pressed.clear();
|
pressed.clear();
|
||||||
|
_screen = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { inject, releaseAll, available, mapKey };
|
module.exports = { inject, releaseAll, available, mapKey };
|
||||||
|
|||||||
@@ -104,13 +104,13 @@ async function startStreaming() {
|
|||||||
pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
||||||
localStream.getTracks().forEach((t) => pc.addTrack(t, localStream));
|
localStream.getTracks().forEach((t) => pc.addTrack(t, localStream));
|
||||||
|
|
||||||
// Viewer creates the input data channel; we receive it here.
|
// The agent is the OFFERER, so it must create the input data channel — otherwise the
|
||||||
pc.ondatachannel = (ev) => {
|
// SCTP m-line is absent from the offer and the channel never negotiates (viewer's stays
|
||||||
const ch = ev.channel;
|
// closed, so no input arrives). The viewer receives this channel via ondatachannel.
|
||||||
ch.onmessage = (msg) => {
|
const inputCh = pc.createDataChannel('input', { ordered: true });
|
||||||
let evt; try { evt = JSON.parse(msg.data); } catch { return; }
|
inputCh.onmessage = (msg) => {
|
||||||
window.agent.injectInput(evt); // -> main process -> OS injection
|
let evt; try { evt = JSON.parse(msg.data); } catch { return; }
|
||||||
};
|
window.agent.injectInput(evt); // -> main process -> OS injection
|
||||||
};
|
};
|
||||||
|
|
||||||
pc.onicecandidate = (ev) => {
|
pc.onicecandidate = (ev) => {
|
||||||
|
|||||||
Generated
+4112
File diff suppressed because it is too large
Load Diff
Generated
+4929
File diff suppressed because it is too large
Load Diff
@@ -75,7 +75,8 @@ ws.onmessage = async (e) => {
|
|||||||
|
|
||||||
function setupPeer() {
|
function setupPeer() {
|
||||||
pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
||||||
inputChannel = pc.createDataChannel('input', { ordered: true });
|
// The agent (offerer) creates the 'input' channel; we receive it here and send input on it.
|
||||||
|
pc.ondatachannel = (ev) => { if (ev.channel.label === 'input') inputChannel = ev.channel; };
|
||||||
pc.ontrack = (ev) => {
|
pc.ontrack = (ev) => {
|
||||||
video.srcObject = ev.streams[0];
|
video.srcObject = ev.streams[0];
|
||||||
setStatus(`Connected to ${machineName} — controlling. Click the screen to send input.`);
|
setStatus(`Connected to ${machineName} — controlling. Click the screen to send input.`);
|
||||||
|
|||||||
Reference in New Issue
Block a user