first commit

2026-06-05 17:29:09 +05:30
commit b984b55bc0
31 changed files with 5008 additions and 0 deletions
@@ -0,0 +1,103 @@
+// OS input injection layer.
+//
+// Cross-platform mouse/keyboard control via @nut-tree-fork/nut-js (optional
+// native dependency). If nut-js isn't installed (e.g. CI, or a sandbox without
+// a display), this module degrades to a logging no-op so the rest of the agent
+// still runs and can be tested. On Windows, nut-js drives the Win32 SendInput
+// API under the hood — the same mechanism TeamViewer/AnyDesk use.
+
+let nut = null;
+try {
+  // eslint-disable-next-line import/no-extraneous-dependencies
+  nut = require('@nut-tree-fork/nut-js');
+  nut.mouse.config.autoDelayMs = 0;
+  nut.keyboard.config.autoDelayMs = 0;
+} catch {
+  nut = null;
+}
+
+const available = !!nut;
+
+// Map browser KeyboardEvent.key values to nut-js Key enum names.
+function mapKey(key, code) {
+  if (!nut) return null;
+  const K = nut.Key;
+  const direct = {
+    'Enter': K.Enter, 'Backspace': K.Backspace, 'Tab': K.Tab, 'Escape': K.Escape,
+    ' ': K.Space, 'ArrowLeft': K.Left, 'ArrowRight': K.Right, 'ArrowUp': K.Up, 'ArrowDown': K.Down,
+    'Home': K.Home, 'End': K.End, 'PageUp': K.PageUp, 'PageDown': K.PageDown, 'Delete': K.Delete,
+    'Control': K.LeftControl, 'Shift': K.LeftShift, 'Alt': K.LeftAlt, 'Meta': K.LeftSuper,
+    'CapsLock': K.CapsLock,
+  };
+  if (direct[key] !== undefined) return [direct[key]];
+  if (/^F\d{1,2}$/.test(key) && K[key] !== undefined) return [K[key]];
+  if (key && key.length === 1) {
+    const upper = key.toUpperCase();
+    if (/[A-Z]/.test(upper) && K[upper] !== undefined) return [K[upper]];
+    if (/[0-9]/.test(key) && K['Num' + key] !== undefined) return [K['Num' + key]];
+    // Fall back to typing the literal character (handles symbols/shifted chars)
+    return { type: key };
+  }
+  return null;
+}
+
+async function moveTo(xNorm, yNorm) {
+  if (!nut) return;
+  const { width, height } = await nut.screen.getResolution();
+  await nut.mouse.setPosition(new nut.Point(Math.round(xNorm * width), Math.round(yNorm * height)));
+}
+
+function buttonEnum(b) {
+  if (!nut) return null;
+  return b === 2 ? nut.Button.RIGHT : b === 1 ? nut.Button.MIDDLE : nut.Button.LEFT;
+}
+
+const pressed = new Set();
+
+// Inject a single normalized input event coming from the viewer.
+async function inject(evt) {
+  if (!nut) {
+    if (evt.kind !== 'mousemove') console.log('[input:noop]', JSON.stringify(evt));
+    return;
+  }
+  try {
+    switch (evt.kind) {
+      case 'mousemove':
+        await moveTo(evt.x, evt.y); break;
+      case 'mousedown':
+        await moveTo(evt.x, evt.y); await nut.mouse.pressButton(buttonEnum(evt.button)); break;
+      case 'mouseup':
+        await nut.mouse.releaseButton(buttonEnum(evt.button)); break;
+      case 'dblclick':
+        await moveTo(evt.x, evt.y); await nut.mouse.doubleClick(nut.Button.LEFT); break;
+      case 'scroll':
+        if (evt.dy) await (evt.dy > 0 ? nut.mouse.scrollDown(Math.abs(evt.dy)) : nut.mouse.scrollUp(Math.abs(evt.dy)));
+        if (evt.dx) await (evt.dx > 0 ? nut.mouse.scrollRight(Math.abs(evt.dx)) : nut.mouse.scrollLeft(Math.abs(evt.dx)));
+        break;
+      case 'keydown': {
+        const m = mapKey(evt.key, evt.code);
+        if (!m) break;
+        if (m.type) { await nut.keyboard.type(m.type); break; }
+        await nut.keyboard.pressKey(...m); m.forEach((k) => pressed.add(k));
+        break;
+      }
+      case 'keyup': {
+        const m = mapKey(evt.key, evt.code);
+        if (!m || m.type) break;
+        await nut.keyboard.releaseKey(...m); m.forEach((k) => pressed.delete(k));
+        break;
+      }
+    }
+  } catch (e) {
+    console.error('[input] inject error:', e.message);
+  }
+}
+
+// Safety: release any stuck modifier keys when a session ends.
+async function releaseAll() {
+  if (!nut) { pressed.clear(); return; }
+  for (const k of pressed) { try { await nut.keyboard.releaseKey(k); } catch {} }
+  pressed.clear();
+}
+
+module.exports = { inject, releaseAll, available, mapKey };
@@ -0,0 +1,44 @@
+// Unit test for the input mapping logic — runs without a display or nut-js.
+// Verifies keymap returns sane shapes and inject() no-ops gracefully.
+const inject = require('./inject');
+const assert = require('assert');
+
+let pass = 0;
+function check(name, cond) {
+  assert.ok(cond, name);
+  console.log('  ok -', name);
+  pass++;
+}
+
+(async () => {
+  console.log('input layer tests:');
+  check('module exposes inject/releaseAll/available', typeof inject.inject === 'function' && typeof inject.releaseAll === 'function');
+  check('available is boolean', typeof inject.available === 'boolean');
+
+  // mapKey returns null when nut-js absent (sandbox) — that is expected & safe.
+  if (!inject.available) {
+    check('mapKey is null-safe without nut-js', inject.mapKey('a', 'KeyA') === null);
+  } else {
+    check('mapKey letter -> array', Array.isArray(inject.mapKey('a', 'KeyA')));
+    check('mapKey Enter -> array', Array.isArray(inject.mapKey('Enter', 'Enter')));
+    check('mapKey symbol -> type fallback', inject.mapKey('@', 'Digit2').type === '@');
+  }
+
+  // inject() must not throw on any event kind, even with no backend.
+  for (const evt of [
+    { kind: 'mousemove', x: 0.5, y: 0.5 },
+    { kind: 'mousedown', button: 0, x: 0.1, y: 0.2 },
+    { kind: 'mouseup', button: 0 },
+    { kind: 'dblclick', x: 0.3, y: 0.3 },
+    { kind: 'scroll', dx: 0, dy: 120 },
+    { kind: 'keydown', key: 'a', code: 'KeyA' },
+    { kind: 'keyup', key: 'a', code: 'KeyA' },
+  ]) {
+    await inject.inject(evt);
+  }
+  check('inject handled all event kinds without throwing', true);
+  await inject.releaseAll();
+  check('releaseAll clears state', true);
+
+  console.log(`\n${pass} checks passed.`);
+})();
@@ -0,0 +1,47 @@
+// OS input injection for Windows via user32 SendInput (koffi FFI).
+// On non-Windows platforms, exports a stub that logs instead of injecting,
+// so the agent can run in dev/test environments.
+
+const { toVirtualKey } = require('./keymap');
+
+const MOUSEEVENTF_MOVE = 0x0001;
+const MOUSEEVENTF_ABSOLUTE = 0x8000;
+const MOUSEEVENTF_LEFTDOWN = 0x0002;
+const MOUSEEVENTF_LEFTUP = 0x0004;
+const MOUSEEVENTF_RIGHTDOWN = 0x0008;
+const MOUSEEVENTF_RIGHTUP = 0x0010;
+const MOUSEEVENTF_MIDDLEDOWN = 0x0020;
+const MOUSEEVENTF_MIDDLEUP = 0x0040;
+const MOUSEEVENTF_WHEEL = 0x0800;
+const KEYEVENTF_KEYUP = 0x0002;
+const INPUT_MOUSE = 0;
+const INPUT_KEYBOARD = 1;
+
+const BUTTON_DOWN = { 0: MOUSEEVENTF_LEFTDOWN, 1: MOUSEEVENTF_MIDDLEDOWN, 2: MOUSEEVENTF_RIGHTDOWN };
+const BUTTON_UP = { 0: MOUSEEVENTF_LEFTUP, 1: MOUSEEVENTF_MIDDLEUP, 2: MOUSEEVENTF_RIGHTUP };
+
+function createWindowsInjector() {
+  const koffi = require('koffi');
+  const user32 = koffi.load('user32.dll');
+
+  const MOUSEINPUT = koffi.struct('MOUSEINPUT', {
+    dx: 'long', dy: 'long', mouseData: 'int32',
+    dwFlags: 'uint32', time: 'uint32', dwExtraInfo: 'uintptr_t',
+  });
+  const KEYBDINPUT = koffi.struct('KEYBDINPUT', {
+    wVk: 'uint16', wScan: 'uint16',
+    dwFlags: 'uint32', time: 'uint32', dwExtraInfo: 'uintptr_t',
+  });
+  const HARDWAREINPUT = koffi.struct('HARDWAREINPUT', {
+    uMsg: 'uint32', wParamL: 'uint16', wParamH: 'uint16',
+  });
+  const INPUT_UNION = koffi.union('INPUT_UNION', {
+    mi: MOUSEINPUT, ki: KEYBDINPUT, hi: HARDWAREINPUT,
+  });
+  const INPUT = koffi.struct('INPUT', { type: 'uint32', u: INPUT_UNION });
+
+  const SendInput = user32.func('uint32 SendInput(uint32 cInputs, INPUT *pInputs, int cbSize)');
+  const INPUT_SIZE = koffi.sizeof(INPUT);
+
+  function sendMouse(mi) {
+    SendInput(1, [{ type: INPUT_MOUSE, u: { mi:
@@ -0,0 +1,71 @@
+// Maps browser KeyboardEvent.code values to Windows virtual-key codes.
+// Reference: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+
+const VK = {};
+
+// Letters (KeyA..KeyZ -> 0x41..0x5A)
+for (let i = 0; i < 26; i++) VK['Key' + String.fromCharCode(65 + i)] = 0x41 + i;
+// Top-row digits (Digit0..Digit9 -> 0x30..0x39)
+for (let i = 0; i <= 9; i++) VK['Digit' + i] = 0x30 + i;
+// Numpad digits
+for (let i = 0; i <= 9; i++) VK['Numpad' + i] = 0x60 + i;
+// Function keys
+for (let i = 1; i <= 24; i++) VK['F' + i] = 0x70 + (i - 1);
+
+Object.assign(VK, {
+  Escape: 0x1b,
+  Tab: 0x09,
+  CapsLock: 0x14,
+  ShiftLeft: 0xa0,
+  ShiftRight: 0xa1,
+  ControlLeft: 0xa2,
+  ControlRight: 0xa3,
+  AltLeft: 0xa4,
+  AltRight: 0xa5,
+  MetaLeft: 0x5b,
+  MetaRight: 0x5c,
+  ContextMenu: 0x5d,
+  Space: 0x20,
+  Enter: 0x0d,
+  NumpadEnter: 0x0d,
+  Backspace: 0x08,
+  Delete: 0x2e,
+  Insert: 0x2d,
+  Home: 0x24,
+  End: 0x23,
+  PageUp: 0x21,
+  PageDown: 0x22,
+  ArrowUp: 0x26,
+  ArrowDown: 0x28,
+  ArrowLeft: 0x25,
+  ArrowRight: 0x27,
+  PrintScreen: 0x2c,
+  ScrollLock: 0x91,
+  Pause: 0x13,
+  NumLock: 0x90,
+  // OEM punctuation (US layout)
+  Semicolon: 0xba,
+  Equal: 0xbb,
+  Comma: 0xbc,
+  Minus: 0xbd,
+  Period: 0xbe,
+  Slash: 0xbf,
+  Backquote: 0xc0,
+  BracketLeft: 0xdb,
+  Backslash: 0xdc,
+  BracketRight: 0xdd,
+  Quote: 0xde,
+  // Numpad operators
+  NumpadMultiply: 0x6a,
+  NumpadAdd: 0x6b,
+  NumpadSubtract: 0x6d,
+  NumpadDecimal: 0x6e,
+  NumpadDivide: 0x6f,
+});
+
+/** @param {string} code KeyboardEvent.code @returns {number|undefined} Windows VK code */
+function toVirtualKey(code) {
+  return VK[code];
+}
+
+module.exports = { toVirtualKey, VK };