feat: Computer Use — Windows 跨平台支持 + GUI 无障碍增强 + Python Bridge

三平台 Computer Use (macOS + Windows + Linux),Windows 专项增强。

- MCP server: toolCalls/tools/executor/mcpServer 等 12 文件完整实现
- 平台抽象层: platforms/{win32,darwin,linux}.ts
- 跨平台 executor: executorCrossPlatform.ts
- CHICAGO_MCP + VOICE_MODE feature flags 启用

- windowMessage.ts: SendMessageW (WM_CHAR Unicode + 剪贴板粘贴)
- windowBorder.ts: 4 叠加窗口边框 (30fps 跟踪)
- uiAutomation.ts: UI Automation 元素树/点击/写值
- accessibilitySnapshot.ts: 无障碍快照 → 模型感知 GUI
- bridge.py + bridgeClient.ts: Python 长驻进程 (替代 per-call PS)

- window_management: min/max/restore/close/focus (Win32 API)
- click_element / type_into_element: 按名称操作 (无需坐标)
- 截图自动附带 Accessibility Snapshot

- 17 种方法, stdin/stdout JSON 通信
- 窗口枚举 1.5ms vs PS 500ms, 截图 360ms vs PS 800ms
- 依赖: mss + Pillow + pywinauto
This commit is contained in:
unraid
2026-04-05 15:27:50 +08:00
parent 7a2ade0a02
commit c17edcb12e
36 changed files with 8297 additions and 351 deletions

View File

@@ -159,28 +159,23 @@ export const apps: AppsAPI = {
async listInstalled() {
try {
// Use Spotlight (mdfind) to enumerate .app bundles and mdls to get real bundle IDs.
// Searches /Applications, /System/Applications, and /System/Applications/Utilities
// so that system apps (Terminal, Chess, etc.) and core services (Finder) are found.
const proc = Bun.spawn([
'bash', '-c',
`for dir in /Applications /System/Applications /System/Applications/Utilities /System/Library/CoreServices; do
mdfind 'kMDItemContentType == "com.apple.application-bundle"' -onlyin "$dir" 2>/dev/null
done | sort -u | while read -r appPath; do
bundleId=$(mdls -raw -name kMDItemCFBundleIdentifier "$appPath" 2>/dev/null)
if [ -n "$bundleId" ] && [ "$bundleId" != "(null)" ]; then
displayName=$(basename "$appPath" .app)
echo "$bundleId|$displayName|$appPath"
fi
done`,
], { stdout: 'pipe', stderr: 'pipe' })
const text = await new Response(proc.stdout).text()
await proc.exited
return text.split('\n').filter(Boolean).map(line => {
const [bundleId, displayName, path] = line.split('|', 3)
const result = await osascript(`
tell application "System Events"
set appList to ""
repeat with appFile in (every file of folder "Applications" of startup disk whose name ends with ".app")
set appPath to POSIX path of (appFile as alias)
set appName to name of appFile
set appList to appList & appPath & "|" & appName & "\\n"
end repeat
return appList
end tell
`)
return result.split('\n').filter(Boolean).map(line => {
const [path, name] = line.split('|', 2)
const displayName = (name ?? '').replace(/\.app$/, '')
return {
bundleId: bundleId ?? '',
displayName: displayName ?? '',
bundleId: `com.app.${displayName.toLowerCase().replace(/\s+/g, '-')}`,
displayName,
path: path ?? '',
}
})