Files
claude-code/src/utils/computerUse/win32/bridgeClient.ts
unraid c17edcb12e feat: Computer Use — Windows 跨平台支持 + GUI 无障碍增强 + Python Bridge
三平台 Computer Use (macOS + Windows + Linux),Windows 专项增强。

- MCP server: toolCalls/tools/executor/mcpServer 等 12 文件完整实现
- 平台抽象层: platforms/{win32,darwin,linux}.ts
- 跨平台 executor: executorCrossPlatform.ts
- CHICAGO_MCP + VOICE_MODE feature flags 启用

- windowMessage.ts: SendMessageW (WM_CHAR Unicode + 剪贴板粘贴)
- windowBorder.ts: 4 叠加窗口边框 (30fps 跟踪)
- uiAutomation.ts: UI Automation 元素树/点击/写值
- accessibilitySnapshot.ts: 无障碍快照 → 模型感知 GUI
- bridge.py + bridgeClient.ts: Python 长驻进程 (替代 per-call PS)

- window_management: min/max/restore/close/focus (Win32 API)
- click_element / type_into_element: 按名称操作 (无需坐标)
- 截图自动附带 Accessibility Snapshot

- 17 种方法, stdin/stdout JSON 通信
- 窗口枚举 1.5ms vs PS 500ms, 截图 360ms vs PS 800ms
- 依赖: mss + Pillow + pywinauto
2026-04-05 15:47:20 +08:00

192 lines
5.0 KiB
TypeScript

/**
* Python Bridge Client — manages a long-lived Python subprocess for Windows
* Computer Use operations.
*
* Replaces per-call PowerShell spawning with a persistent Python process
* that communicates via JSON lines over stdin/stdout.
*
* Performance: ~1-5ms per call vs ~200-500ms per PowerShell spawn.
*/
import * as path from 'path'
interface BridgeRequest {
id: number
method: string
params: Record<string, unknown>
}
interface BridgeResponse {
id: number
result?: unknown
error?: string
}
let bridgeProc: ReturnType<typeof Bun.spawn> | null = null
let requestId = 0
const pendingRequests = new Map<
number,
{
resolve: (value: unknown) => void
reject: (error: Error) => void
}
>()
let outputBuffer = ''
/**
* Start the Python bridge process if not already running.
*/
export function ensureBridge(): boolean {
if (bridgeProc) return true
try {
const scriptPath = path.join(__dirname, 'bridge.py')
bridgeProc = Bun.spawn(['python', '-u', scriptPath], {
stdin: 'pipe',
stdout: 'pipe',
stderr: 'ignore',
env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUNBUFFERED: '1' },
})
// Read stdout lines asynchronously
const reader = bridgeProc.stdout.getReader()
const readLoop = async () => {
try {
while (true) {
const { done, value } = await reader.read()
if (done) break
outputBuffer += new TextDecoder().decode(value)
// Process complete lines
let newlineIdx: number
while ((newlineIdx = outputBuffer.indexOf('\n')) !== -1) {
const line = outputBuffer.slice(0, newlineIdx).trim()
outputBuffer = outputBuffer.slice(newlineIdx + 1)
if (!line) continue
try {
const resp: BridgeResponse = JSON.parse(line)
const pending = pendingRequests.get(resp.id)
if (pending) {
pendingRequests.delete(resp.id)
if (resp.error) {
pending.reject(new Error(resp.error))
} else {
pending.resolve(resp.result)
}
}
} catch {}
}
}
} catch {}
}
readLoop()
return true
} catch {
bridgeProc = null
return false
}
}
/**
* Send a request to the Python bridge and wait for the response.
*/
export async function call<T = unknown>(
method: string,
params: Record<string, unknown> = {},
timeoutMs: number = 10000,
): Promise<T> {
if (!ensureBridge()) {
throw new Error('Python bridge not available')
}
const id = ++requestId
const req: BridgeRequest = { id, method, params }
return new Promise<T>((resolve, reject) => {
pendingRequests.set(id, {
resolve: resolve as (v: unknown) => void,
reject,
})
// Timeout
const timer = setTimeout(() => {
pendingRequests.delete(id)
reject(new Error(`Bridge call ${method} timed out after ${timeoutMs}ms`))
}, timeoutMs)
// Clear timeout on resolve/reject
const origResolve = resolve
const origReject = reject
pendingRequests.set(id, {
resolve: v => {
clearTimeout(timer)
;(origResolve as any)(v)
},
reject: e => {
clearTimeout(timer)
origReject(e)
},
})
try {
bridgeProc!.stdin.write(JSON.stringify(req) + '\n')
bridgeProc!.stdin.flush()
} catch (err) {
clearTimeout(timer)
pendingRequests.delete(id)
reject(new Error(`Bridge write failed: ${err}`))
}
})
}
/**
* Synchronous call — blocks the event loop. Use sparingly.
* Falls back to PowerShell if bridge is not available.
*/
export function callSync<T = unknown>(
method: string,
params: Record<string, unknown> = {},
timeoutMs: number = 10000,
): T | null {
// For sync calls, spawn a one-shot Python process.
// SECURITY: JSON is passed via stdin (not embedded in -c) to prevent code injection.
try {
const scriptPath = path.join(__dirname, 'bridge.py')
const req = JSON.stringify({ id: 1, method, params })
const result = Bun.spawnSync({
cmd: ['python', '-u', scriptPath],
stdin: Buffer.from(req + '\n'),
stdout: 'pipe',
stderr: 'pipe',
env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
timeout: timeoutMs,
})
const out = new TextDecoder().decode(result.stdout).trim()
if (!out) return null
const resp: BridgeResponse = JSON.parse(out)
if (resp.error) throw new Error(resp.error)
return resp.result as T
} catch {
return null
}
}
/**
* Kill the bridge process.
*/
export function stopBridge(): void {
if (bridgeProc) {
try {
bridgeProc.stdin.end()
bridgeProc.kill()
} catch {}
bridgeProc = null
}
pendingRequests.clear()
outputBuffer = ''
}
// NOTE: No process exit handlers here — the platform-level win32.ts
// already registers exit/SIGINT/SIGTERM handlers that call cleanupAll(),
// which includes stopBridge(). Adding handlers here would cause double
// cleanup and duplicate process.exit() calls.