claude-code/packages/@ant/computer-use-input/src/backends/darwin.ts

/**
 * macOS backend for computer-use-input
 *
 * Uses AppleScript (osascript) and JXA (JavaScript for Automation) to control
 * mouse and keyboard via CoreGraphics events and System Events.
 */

import { execFile, execFileSync } from 'child_process'
import { promisify } from 'util'
import type { FrontmostAppInfo, InputBackend } from '../types.js'

const execFileAsync = promisify(execFile)

const KEY_MAP: Record<string, number> = {
  return: 36, enter: 36, tab: 48, space: 49, delete: 51, backspace: 51,
  escape: 53, esc: 53,
  left: 123, right: 124, down: 125, up: 126,
  f1: 122, f2: 120, f3: 99, f4: 118, f5: 96, f6: 97,
  f7: 98, f8: 100, f9: 101, f10: 109, f11: 103, f12: 111,
  home: 115, end: 119, pageup: 116, pagedown: 121,
}

const MODIFIER_MAP: Record<string, string> = {
  command: 'command down', cmd: 'command down', meta: 'command down', super: 'command down',
  shift: 'shift down',
  option: 'option down', alt: 'option down',
  control: 'control down', ctrl: 'control down',
}

async function osascript(script: string): Promise<string> {
  const { stdout } = await execFileAsync('osascript', ['-e', script], {
    encoding: 'utf-8',
  })
  return stdout.trim()
}

async function jxa(script: string): Promise<string> {
  const { stdout } = await execFileAsync('osascript', ['-l', 'JavaScript', '-e', script], {
    encoding: 'utf-8',
  })
  return stdout.trim()
}

function buildMouseJxa(eventType: string, x: number, y: number, btn: number, clickState?: number): string {
  let script = `ObjC.import("CoreGraphics"); var p = $.CGPointMake(${x},${y}); var e = $.CGEventCreateMouseEvent(null, $.${eventType}, p, ${btn});`
  if (clickState !== undefined) {
    script += ` $.CGEventSetIntegerValueField(e, $.kCGMouseEventClickState, ${clickState});`
  }
  script += ` $.CGEventPost($.kCGHIDEventTap, e);`
  return script
}

export const moveMouse: InputBackend['moveMouse'] = async (x, y, _animated) => {
  await jxa(buildMouseJxa('kCGEventMouseMoved', x, y, 0))
}

export const key: InputBackend['key'] = async (keyName, action) => {
  if (action === 'release') return
  const lower = keyName.toLowerCase()
  const keyCode = KEY_MAP[lower]
  if (keyCode !== undefined) {
    await osascript(`tell application "System Events" to key code ${keyCode}`)
  } else {
    await osascript(`tell application "System Events" to keystroke "${keyName.length === 1 ? keyName : lower}"`)
  }
}

export const keys: InputBackend['keys'] = async (parts) => {
  const modifiers: string[] = []
  let finalKey: string | null = null
  for (const part of parts) {
    const mod = MODIFIER_MAP[part.toLowerCase()]
    if (mod) modifiers.push(mod)
    else finalKey = part
  }
  if (!finalKey) return
  const lower = finalKey.toLowerCase()
  const keyCode = KEY_MAP[lower]
  const modStr = modifiers.length > 0 ? ` using {${modifiers.join(', ')}}` : ''
  if (keyCode !== undefined) {
    await osascript(`tell application "System Events" to key code ${keyCode}${modStr}`)
  } else {
    await osascript(`tell application "System Events" to keystroke "${finalKey.length === 1 ? finalKey : lower}"${modStr}`)
  }
}

export const mouseLocation: InputBackend['mouseLocation'] = async () => {
  const result = await jxa('ObjC.import("CoreGraphics"); var e = $.CGEventCreate(null); var p = $.CGEventGetLocation(e); p.x + "," + p.y')
  const [xStr, yStr] = result.split(',')
  return { x: Math.round(Number(xStr)), y: Math.round(Number(yStr)) }
}

export const mouseButton: InputBackend['mouseButton'] = async (button, action, count) => {
  const pos = await mouseLocation()
  const btn = button === 'left' ? 0 : button === 'right' ? 1 : 2
  const downType = btn === 0 ? 'kCGEventLeftMouseDown' : btn === 1 ? 'kCGEventRightMouseDown' : 'kCGEventOtherMouseDown'
  const upType = btn === 0 ? 'kCGEventLeftMouseUp' : btn === 1 ? 'kCGEventRightMouseUp' : 'kCGEventOtherMouseUp'

  if (action === 'click') {
    for (let i = 0; i < (count ?? 1); i++) {
      await jxa(buildMouseJxa(downType, pos.x, pos.y, btn, i + 1))
      await jxa(buildMouseJxa(upType, pos.x, pos.y, btn, i + 1))
    }
  } else if (action === 'press') {
    await jxa(buildMouseJxa(downType, pos.x, pos.y, btn))
  } else {
    await jxa(buildMouseJxa(upType, pos.x, pos.y, btn))
  }
}

export const mouseScroll: InputBackend['mouseScroll'] = async (amount, direction) => {
  const script = direction === 'vertical'
    ? `ObjC.import("CoreGraphics"); var e = $.CGEventCreateScrollWheelEvent(null, 0, 1, ${amount}); $.CGEventPost($.kCGHIDEventTap, e);`
    : `ObjC.import("CoreGraphics"); var e = $.CGEventCreateScrollWheelEvent(null, 0, 2, 0, ${amount}); $.CGEventPost($.kCGHIDEventTap, e);`
  await jxa(script)
}

export const typeText: InputBackend['typeText'] = async (text) => {
  const escaped = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
  await osascript(`tell application "System Events" to keystroke "${escaped}"`)
}

export const getFrontmostAppInfo: InputBackend['getFrontmostAppInfo'] = () => {
  try {
    const output = execFileSync('osascript', ['-e', `
      tell application "System Events"
        set frontApp to first application process whose frontmost is true
        set appName to name of frontApp
        set bundleId to bundle identifier of frontApp
        return bundleId & "|" & appName
      end tell
    `], { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'ignore'] }).trim()
    if (!output || !output.includes('|')) return null
    const [bundleId, appName] = output.split('|', 2)
    return { bundleId: bundleId!, appName: appName! }
  } catch {
    return null
  }
}