/** * macOS backend for computer-use-swift * * Uses AppleScript/JXA/screencapture for display info, app management, * and screenshots. */ import { readFileSync, unlinkSync } from 'fs' import { tmpdir } from 'os' import { join } from 'path' import type { AppInfo, AppsAPI, DisplayAPI, DisplayGeometry, InstalledApp, PrepareDisplayResult, RunningApp, ScreenshotAPI, ScreenshotResult, SwiftBackend, WindowDisplayInfo, } from '../types.js' // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function jxaSync(script: string): string { const result = Bun.spawnSync({ cmd: ['osascript', '-l', 'JavaScript', '-e', script], stdout: 'pipe', stderr: 'pipe', }) return new TextDecoder().decode(result.stdout).trim() } function osascriptSync(script: string): string { const result = Bun.spawnSync({ cmd: ['osascript', '-e', script], stdout: 'pipe', stderr: 'pipe', }) return new TextDecoder().decode(result.stdout).trim() } async function osascript(script: string): Promise { const proc = Bun.spawn(['osascript', '-e', script], { stdout: 'pipe', stderr: 'pipe', }) const text = await new Response(proc.stdout).text() await proc.exited return text.trim() } async function jxa(script: string): Promise { const proc = Bun.spawn(['osascript', '-l', 'JavaScript', '-e', script], { stdout: 'pipe', stderr: 'pipe', }) const text = await new Response(proc.stdout).text() await proc.exited return text.trim() } // --------------------------------------------------------------------------- // DisplayAPI // --------------------------------------------------------------------------- export const display: DisplayAPI = { getSize(displayId?: number): DisplayGeometry { const all = this.listAll() if (displayId !== undefined) { const found = all.find(d => d.displayId === displayId) if (found) return found } return all[0] ?? { width: 1920, height: 1080, scaleFactor: 2, displayId: 1 } }, listAll(): DisplayGeometry[] { try { const raw = jxaSync(` ObjC.import("CoreGraphics"); var displays = $.CGDisplayCopyAllDisplayModes ? [] : []; var active = $.CGGetActiveDisplayList(10, null, Ref()); var countRef = Ref(); $.CGGetActiveDisplayList(0, null, countRef); var count = countRef[0]; var idBuf = Ref(); $.CGGetActiveDisplayList(count, idBuf, countRef); var result = []; for (var i = 0; i < count; i++) { var did = idBuf[i]; var w = $.CGDisplayPixelsWide(did); var h = $.CGDisplayPixelsHigh(did); var mode = $.CGDisplayCopyDisplayMode(did); var pw = $.CGDisplayModeGetPixelWidth(mode); var sf = pw > 0 && w > 0 ? pw / w : 2; result.push({width: w, height: h, scaleFactor: sf, displayId: did}); } JSON.stringify(result); `) return (JSON.parse(raw) as DisplayGeometry[]).map(d => ({ width: Number(d.width), height: Number(d.height), scaleFactor: Number(d.scaleFactor), displayId: Number(d.displayId), })) } catch { try { const raw = jxaSync(` ObjC.import("AppKit"); var screens = $.NSScreen.screens; var result = []; for (var i = 0; i < screens.count; i++) { var s = screens.objectAtIndex(i); var frame = s.frame; var desc = s.deviceDescription; var screenNumber = desc.objectForKey($("NSScreenNumber")).intValue; var backingFactor = s.backingScaleFactor; result.push({ width: Math.round(frame.size.width), height: Math.round(frame.size.height), scaleFactor: backingFactor, displayId: screenNumber }); } JSON.stringify(result); `) return (JSON.parse(raw) as DisplayGeometry[]).map(d => ({ width: Number(d.width), height: Number(d.height), scaleFactor: Number(d.scaleFactor), displayId: Number(d.displayId), })) } catch { return [{ width: 1920, height: 1080, scaleFactor: 2, displayId: 1 }] } } }, } // --------------------------------------------------------------------------- // AppsAPI // --------------------------------------------------------------------------- export const apps: AppsAPI = { async prepareDisplay(_allowlistBundleIds, _surrogateHost, _displayId) { return { activated: '', hidden: [] } }, async previewHideSet(_bundleIds, _displayId) { return [] }, async findWindowDisplays(bundleIds) { return bundleIds.map(bundleId => ({ bundleId, displayIds: [1] })) }, async appUnderPoint(_x, _y) { try { const result = await jxa(` ObjC.import("CoreGraphics"); ObjC.import("AppKit"); var pt = $.CGPointMake(${_x}, ${_y}); var app = $.NSWorkspace.sharedWorkspace.frontmostApplication; JSON.stringify({bundleId: app.bundleIdentifier.js, displayName: app.localizedName.js}); `) return JSON.parse(result) } catch { return null } }, async listInstalled() { try { // Use Spotlight (mdfind) to enumerate .app bundles and mdls to get real bundle IDs. // Searches /Applications, /System/Applications, and /System/Applications/Utilities // so that system apps (Terminal, Chess, etc.) and core services (Finder) are found. const proc = Bun.spawn([ 'bash', '-c', `for dir in /Applications /System/Applications /System/Applications/Utilities /System/Library/CoreServices; do mdfind 'kMDItemContentType == "com.apple.application-bundle"' -onlyin "$dir" 2>/dev/null done | sort -u | while read -r appPath; do bundleId=$(mdls -raw -name kMDItemCFBundleIdentifier "$appPath" 2>/dev/null) if [ -n "$bundleId" ] && [ "$bundleId" != "(null)" ]; then displayName=$(basename "$appPath" .app) echo "$bundleId|$displayName|$appPath" fi done`, ], { stdout: 'pipe', stderr: 'pipe' }) const text = await new Response(proc.stdout).text() await proc.exited return text.split('\n').filter(Boolean).map(line => { const [bundleId, displayName, path] = line.split('|', 3) return { bundleId: bundleId ?? '', displayName: displayName ?? '', path: path ?? '', } }) } catch { return [] } }, iconDataUrl(_path) { return null }, listRunning() { try { const raw = jxaSync(` var apps = Application("System Events").applicationProcesses.whose({backgroundOnly: false}); var result = []; for (var i = 0; i < apps.length; i++) { try { var a = apps[i]; result.push({bundleId: a.bundleIdentifier(), displayName: a.name()}); } catch(e) {} } JSON.stringify(result); `) return JSON.parse(raw) } catch { return [] } }, async open(bundleId) { await osascript(`tell application id "${bundleId}" to activate`) }, async unhide(bundleIds) { for (const bundleId of bundleIds) { await osascript(` tell application "System Events" set visible of application process (name of application process whose bundle identifier is "${bundleId}") to true end tell `) } }, } // --------------------------------------------------------------------------- // ScreenshotAPI // --------------------------------------------------------------------------- async function captureScreenToBase64(args: string[]): Promise<{ base64: string; width: number; height: number }> { const tmpFile = join(tmpdir(), `cu-screenshot-${Date.now()}.png`) const proc = Bun.spawn(['screencapture', ...args, tmpFile], { stdout: 'pipe', stderr: 'pipe', }) await proc.exited try { const buf = readFileSync(tmpFile) const base64 = buf.toString('base64') const width = buf.readUInt32BE(16) const height = buf.readUInt32BE(20) return { base64, width, height } } finally { try { unlinkSync(tmpFile) } catch {} } } export const screenshot: ScreenshotAPI = { async captureExcluding(_allowedBundleIds, _quality, _targetW, _targetH, displayId) { const args = ['-x'] if (displayId !== undefined) args.push('-D', String(displayId)) return captureScreenToBase64(args) }, async captureRegion(_allowedBundleIds, x, y, w, h, _outW, _outH, _quality, displayId) { const args = ['-x', '-R', `${x},${y},${w},${h}`] if (displayId !== undefined) args.push('-D', String(displayId)) return captureScreenToBase64(args) }, }