From aaabf0c168bea7170c881802047691210f41a31f Mon Sep 17 00:00:00 2001 From: claude-code-best Date: Sun, 10 May 2026 22:57:30 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"feat:=20=E6=B7=BB=E5=8A=A0=20GBK=20?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8B=E6=94=AF?= =?UTF-8?q?=E6=8C=81=EF=BC=8C=E6=96=87=E4=BB=B6=E8=AF=BB=E5=86=99=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E9=80=8F=E6=98=8E=E5=A4=84=E7=90=86=E9=9D=9E=20UTF-8?= =?UTF-8?q?=20=E6=96=87=E4=BB=B6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0ce8f7a1cbd8fbb41da6c8255849d7726e59b412. --- CLAUDE.md | 5 - .../src/tools/BashTool/BashTool.tsx | 4 +- .../src/tools/FileEditTool/FileEditTool.ts | 16 +- .../spec-design.md | 179 ------------------ .../spec-human-verify.md | 161 ---------------- .../spec-plan-acceptance.md | 47 ----- .../spec-plan-task-0.md | 34 ---- .../spec-plan-task-1.md | 141 -------------- .../spec-plan-task-2.md | 163 ---------------- .../spec-plan-task-3.md | 161 ---------------- .../spec-plan-task-4.md | 155 --------------- .../spec-plan.md | 49 ----- .../SedEditPermissionRequest.tsx | 6 +- src/utils/__tests__/encoding.test.ts | 102 ---------- src/utils/__tests__/file.test.ts | 58 +----- src/utils/__tests__/fileRead.test.ts | 107 ----------- src/utils/__tests__/readFileInRange.test.ts | 87 --------- src/utils/encoding.ts | 90 --------- src/utils/file.ts | 63 ++---- src/utils/fileRead.ts | 33 +++- src/utils/fileReadCache.ts | 10 +- src/utils/readFileInRange.ts | 176 +++++++---------- 22 files changed, 120 insertions(+), 1727 deletions(-) delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-human-verify.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-acceptance.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-0.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-1.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-2.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-3.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-4.md delete mode 100644 spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md delete mode 100644 src/utils/__tests__/encoding.test.ts delete mode 100644 src/utils/__tests__/fileRead.test.ts delete mode 100644 src/utils/__tests__/readFileInRange.test.ts delete mode 100644 src/utils/encoding.ts diff --git a/CLAUDE.md b/CLAUDE.md index c118c1655..4dfc532e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -119,11 +119,6 @@ bun run docs:dev - **7 providers**: `firstParty` (Anthropic direct), `bedrock` (AWS), `vertex` (Google Cloud), `foundry`, `openai`, `gemini`, `grok` (xAI)。 - Provider selection in `src/utils/model/providers.ts`。优先级：modelType 参数 > 环境变量 > 默认 firstParty。 -### Encoding Detection - -- **`src/utils/encoding.ts`** — 文件编码检测的唯一入口。提供 `detectEncoding`（三层检测：BOM → UTF-8 fatal → ICU 回退链）和 `decodeBuffer`/`encodeString` 函数。检测基于文件头部 4KB，零外部依赖，仅使用 TextDecoder API。ISO-8859-1 作为最终兜底编码（单字节编码永远成功）。`FileEncoding` 类型扩展了 `BufferEncoding`，覆盖 gbk/gb18030/shift_jis/euc-kr/euc-jp/big5/iso-8859-1。 -- `fs.readFileSync(path, { encoding })` 的 `encoding` 选项只接受 `BufferEncoding`，不支持 `gbk`/`shift_jis` 等 ICU 编码名。读取非 UTF-8 文件时必须先 `fs.readFileSync(path)` 读 Buffer，再用 `TextDecoder` 解码。项目中所有文件读取路径（fileRead.ts、fileReadCache.ts、file.ts）已统一使用 `decodeBuffer` 函数处理此逻辑。 - ### Tool System - **`src/Tool.ts`** — Tool interface definition (`Tool` type) and utilities (`findToolByName`, `toolMatchesName`). diff --git a/packages/builtin-tools/src/tools/BashTool/BashTool.tsx b/packages/builtin-tools/src/tools/BashTool/BashTool.tsx index 5334c4098..eeb6fa367 100644 --- a/packages/builtin-tools/src/tools/BashTool/BashTool.tsx +++ b/packages/builtin-tools/src/tools/BashTool/BashTool.tsx @@ -29,7 +29,6 @@ import { extractClaudeCodeHints } from 'src/utils/claudeCodeHints.js'; import { detectCodeIndexingFromCommand } from 'src/utils/codeIndexing.js'; import { isEnvTruthy } from 'src/utils/envUtils.js'; import { isENOENT, ShellError } from 'src/utils/errors.js'; -import { decodeBuffer } from 'src/utils/encoding.js'; import { detectFileEncoding, detectLineEndings, getFileModificationTime, writeTextContent } from 'src/utils/file.js'; import { fileHistoryEnabled, fileHistoryTrackEdit } from 'src/utils/fileHistory.js'; import { truncate } from 'src/utils/format.js'; @@ -512,8 +511,7 @@ async function applySedEdit( const encoding = detectFileEncoding(absoluteFilePath); let originalContent: string; try { - const rawBuffer = await fs.readFileBytes(absoluteFilePath); - originalContent = decodeBuffer(rawBuffer, encoding); + originalContent = await fs.readFile(absoluteFilePath, { encoding }); } catch (e) { if (isENOENT(e)) { return { diff --git a/packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts b/packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts index e0988f0cf..29c937b0b 100644 --- a/packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts +++ b/packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts @@ -34,11 +34,6 @@ import { type LineEndingType, readFileSyncWithMetadata, } from 'src/utils/fileRead.js' -import { - detectEncoding, - decodeBuffer, - type FileEncoding, -} from 'src/utils/encoding.js' import { formatFileSize } from 'src/utils/format.js' import { getFsImplementation } from 'src/utils/fsOperations.js' import { fetchSingleFileGitDiff, type ToolUseDiff } from 'src/utils/gitDiff.js' @@ -207,8 +202,13 @@ export const FileEditTool = buildTool({ let fileContent: string | null try { const fileBuffer = await fs.readFileBytes(fullFilePath) - const encoding: FileEncoding = detectEncoding(fileBuffer) - fileContent = decodeBuffer(fileBuffer, encoding).replaceAll('\r\n', '\n') + const encoding: BufferEncoding = + fileBuffer.length >= 2 && + fileBuffer[0] === 0xff && + fileBuffer[1] === 0xfe + ? 'utf16le' + : 'utf8' + fileContent = fileBuffer.toString(encoding).replaceAll('\r\n', '\n') } catch (e) { if (isENOENT(e)) { fileContent = null @@ -584,7 +584,7 @@ export const FileEditTool = buildTool({ function readFileForEdit(absoluteFilePath: string): { content: string fileExists: boolean - encoding: FileEncoding + encoding: BufferEncoding lineEndings: LineEndingType } { try { diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md deleted file mode 100644 index eaffea21b..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md +++ /dev/null @@ -1,179 +0,0 @@ -# Feature: 20260510_F001 - multi-encoding-file-tools - -## 需求背景 - -当前文件读写工具（FileReadTool、FileWriteTool、FileEditTool）的编码检测非常简单——仅通过 BOM 头识别 UTF-8 和 UTF-16LE，其他所有情况默认按 UTF-8 处理。对于 GBK/GB2312 等非 BOM 编码文件，读取时会产生乱码，导致 AI 模型无法正确理解和编辑这些文件。 - -这在中文 Windows 用户场景中尤其常见：许多旧项目、日志文件、配置文件使用 GBK 编码，当前工具链无法处理。 - -## 目标 - -- 文件读取时自动检测编码并正确解码，对 AI 模型完全透明（不增加 encoding 参数） -- 文件写入时保持原文件编码，不改变用户的编码习惯 -- 覆盖 GBK 编码（最常见非 UTF-8 CJK 编码），latin1 作为最终兜底 -- 零外部依赖，仅使用 Node.js/Bun 内置的 TextDecoder/TextEncoder - -## 范围变更 - -**仅保留 GBK 编码支持**。Shift_JIS、EUC-JP、EUC-KR、Big5、GB18030、ISO-8859-1 已移出范围。原因：多编码回退链存在字节序列歧义（如 GBK 和 Shift_JIS 共享大量有效字节范围），导致误检测。GBK 覆盖了最核心的中文 Windows 用户场景。 - -## 方案设计 - -### 架构概述 - -新增一个独立的编码工具模块 `src/utils/encoding.ts`，提供编码检测和解码/编码函数。现有文件读写路径通过调用此模块实现对非 UTF-8 编码的支持。 - -``` - ┌─────────────────────────┐ - │ src/utils/encoding.ts │ - │ detectEncoding(buffer) │ - │ decodeBuffer(buf, enc) │ - │ encodeString(str, enc) │ - └─────────┬───────────────┘ - │ - ┌───────────────┼───────────────┐ - ▼ ▼ ▼ - fileRead.ts readFileInRange.ts file.ts - (readFileSync (异步读取路径) (writeTextContent) - WithMetadata) -``` - -### 编码检测算法（三层检测） - -检测基于文件头部 4KB 数据，分三层依次判断： - -**第一层：BOM 检测（现有逻辑保留）** -- `FF FE` → UTF-16LE -- `EF BB BF` → UTF-8（带 BOM） - -**第二层：UTF-8 验证** -- 用 `new TextDecoder('utf-8', { fatal: true })` 对头部 4KB 做解码 -- 成功 → 文件为 UTF-8（覆盖绝大多数现代源码文件） -- 失败（抛出 TypeError）→ 进入第三层 - -**第三层：GBK 回退** -- 用 `new TextDecoder('gbk', { fatal: true })` 尝试解码头部 4KB -- 成功 → 文件为 GBK（覆盖中文 Windows 用户最常见的非 UTF-8 编码） -- 失败 → `latin1`（单字节编码，永远成功，作为最终兜底） - -```typescript -// src/utils/encoding.ts 核心逻辑 - -export type FileEncoding = BufferEncoding | 'gbk' -export type DetectedEncoding = string - -export function detectEncoding(buffer: Buffer): FileEncoding { - // Layer 1: BOM - if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { - return 'utf-16le' - } - if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { - return 'utf-8' - } - - // Layer 2: UTF-8 validation - try { - new TextDecoder('utf-8', { fatal: true }).decode(buffer) - return 'utf-8' - } catch {} - - // Layer 3: GBK fallback - try { - new TextDecoder('gbk', { fatal: true }).decode(buffer) - return 'gbk' - } catch {} - - return 'latin1' -} -``` - -### 读取路径改造 - -#### `src/utils/fileRead.ts` — `detectEncodingForResolvedPath` - -将现有的 BOM-only 检测替换为调用 `encoding.ts` 的 `detectEncoding` 函数。返回值从 `BufferEncoding` 改为 `FileEncoding`（`BufferEncoding | 'gbk'`）。 - -`readFileSyncWithMetadata` 函数先读 raw Buffer，再用 `decodeBuffer` 解码，而非使用 `fs.readFileSync` 的 encoding 选项（该选项只接受 `BufferEncoding`，不支持 `gbk`）。 - -#### `src/utils/readFileInRange.ts` — 异步读取 - -当前两个路径（fast path 和 streaming path）都硬编码 `encoding: 'utf8'`： - -**Fast path 改造**： -- `readFile` 改为读取 Buffer（去掉 encoding 参数） -- 读取后调用 `detectEncoding(buffer)` 检测编码 -- 用 `decodeBuffer` 解码为字符串 -- 后续行处理逻辑不变 - -**Streaming path 改造**： -- `createReadStream` 去掉 `encoding: 'utf8'`，改为 Buffer 模式 -- 第一个 chunk 做编码检测（同时保留 BOM 剥离逻辑） -- 后续 chunk 拼接后用 `TextDecoder` 解码 -- 注意：streaming 路径需要特殊处理——先收集足够字节做检测，再逐行扫描 - -**Streaming 编码处理策略**： -streaming 路径改为两阶段： -1. **检测阶段**：前 4KB 数据到达后立即检测编码 -2. **解码阶段**：用检测到的编码创建一个 `TextDecoder`（`{ stream: true }` 模式），逐 chunk 解码 - -### 写入路径改造 - -#### 编码回写策略 - -写入时需要将内部 UTF-8 字符串编码回原文件编码。由于 `TextEncoder` 只支持 UTF-8 输出，需要使用 `TextDecoder` 的反向操作。 - -**最终决定**：对于非 UTF-8 文件的写回，尝试使用 `Buffer.from(content, encoding)` 编码，失败则自动转换为 UTF-8 并在结果消息中注明。这样既满足了零依赖约束，也避免了数据损坏。 - -#### `src/utils/file.ts` — `writeTextContent` - -现有函数签名 `writeTextContent(filePath, content, encoding, lineEndings)` 已接受 encoding 参数。需要： -- 扩展类型，接受 `FileEncoding` 而非仅 `BufferEncoding` -- 对于 UTF-8 和 UTF-16LE，行为不变 -- 对于 GBK，使用 `encodeString` 函数尝试编码，失败则回退为 UTF-8 写入 - -#### `FileWriteTool` 和 `FileEditTool` - -这两个工具的 `call` 方法中，`writeTextContent` 调用已传递 `encoding`（来自 `readFileSyncWithMetadata` 的返回值）。改动很小——只需确保类型系统接受新编码名。 - -### 类型扩展 - -```typescript -// 扩展编码类型 — 仅添加 GBK -export type FileEncoding = BufferEncoding | 'gbk' -``` - -在 `readFileSyncWithMetadata` 返回类型中将 `encoding` 从 `BufferEncoding` 改为 `FileEncoding`。 - -## 实现要点 - -### 关键技术决策 - -1. **检测只用头部 4KB**：避免全文件扫描，性能开销极小（多几次 TextDecoder 调用，每次 ~1μs） -2. **GBK 作为唯一回退**：中文 Windows 用户最多，且避免了多编码回退链的字节序列歧义问题 -3. **TextDecoder fatal 模式**：`{ fatal: true }` 是检测的关键——如果字节序列不符合编码规范会抛异常，借此区分不同编码 -4. **streaming 路径的两阶段设计**：先攒够检测数据再开始行扫描，避免半字符解码问题 -5. **latin1 最终兜底**：单字节编码永远成功，确保任何文件都能被读取 - -### 难点 - -1. **Streaming 编码解码**：`TextDecoder` 支持 `{ stream: true }` 模式处理多字节字符的 chunk 边界，但需要在检测完成前缓冲数据 -2. **编码回写的零依赖方案**：`TextEncoder` 只输出 UTF-8，非 UTF-8 编码回写需要额外处理。务实方案是 UTF-8 写入 + 消息提示 -3. **混合编码文件**：极少见，不在本次覆盖范围内 - -### 依赖 - -- 零外部依赖，仅使用 `TextDecoder`（Node.js 13+ / Bun 内置 full-icu） -- Bun 运行时对 GBK 的 TextDecoder 支持已验证可用（Bun 1.3.13） - -## 验收标准 - -- [x] FileReadTool 能正确读取 GBK 编码的中文文本文件，显示正确的中文内容 -- [x] FileReadTool 能正确读取 UTF-8 文件（行为不变，回归测试通过） -- [x] FileReadTool 能正确读取 UTF-16LE 文件（行为不变） -- [x] FileEditTool 能编辑 GBK 文件并写回，内容不乱码 -- [x] FileWriteTool 编辑 GBK 文件后写回，编码保持或合理转换 -- [x] readFileInRange 的 fast path 路径支持非 UTF-8 编码 -- [x] readFileInRange 的 streaming path 支持非 UTF-8 编码 -- [x] 编码检测性能：4KB 数据检测耗时 < 1ms -- [x] `bun run precheck` typecheck + lint + 相关测试零错误 -- [x] 新增编码相关单元测试覆盖检测和解码逻辑 diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-human-verify.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-human-verify.md deleted file mode 100644 index 1b6528366..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-human-verify.md +++ /dev/null @@ -1,161 +0,0 @@ -# 多编码文件工具 人工验收清单 - -**生成时间:** 2026-05-10 -**关联计划:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md -**关联设计:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md - ---- - -所有验收项均可通过 Shell 命令自动化验证，无需人类参与。仍将生成清单用于自动执行。 - -**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030 已移除。 - ---- - -## 验收前准备 - -### 环境要求 - -- [x] [AUTO] 检查 Bun 运行时版本: `bun --version` -- [x] [AUTO] 安装依赖: `bun install` - -### 测试数据准备 - -- [x] [AUTO] 创建 GBK 编码测试文件: `bun -e "const fs = require('fs'); const b = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3, 0xCA, 0xC0, 0xBD, 0xE7, 0x0A]); fs.writeFileSync('/tmp/test-gbk.txt', b)"` -- [x] [AUTO] 创建 UTF-8 测试文件: `bun -e "require('fs').writeFileSync('/tmp/test-utf8.txt', 'Hello 世界\n')"` -- [x] [AUTO] 创建 UTF-16LE 测试文件: `bun -e "const fs = require('fs'); const b = Buffer.from('Hello','utf16le'); fs.writeFileSync('/tmp/test-utf16le.txt', b)"` - ---- - -## 验收项目 - -### 场景 1：读取 GBK 编码文件（中文场景） - -**用户目标:** 用户有一个 GBK 编码的中文文件，通过 FileReadTool 读取后看到正确的中文内容 - -**触发路径:** -1. 系统检测到非 UTF-8 字节序列 -2. 编码回退识别为 GBK -3. 用 GBK 解码输出中文文本 - -#### - [x] 1.1 GBK 文件同步读取 -- **来源:** spec-plan-acceptance.md §2 / spec-design.md §验收标准 -- **目的:** 确认 GBK 文件读取解码正确 -- **操作步骤:** - 1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-gbk.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `你好世界` - 2. [A] 上条命令输出 encoding 字段 → 期望包含: `gbk` - -#### - [x] 1.2 GBK 文件异步路径读取 -- **来源:** spec-plan-acceptance.md §6 / spec-design.md §验收标准 -- **目的:** 确认 readFileInRange fast path 支持 GBK -- **操作步骤:** - 1. [A] `bun -e "import { readFileInRange } from './src/utils/readFileInRange.js'; const r = await readFileInRange('/tmp/test-gbk.txt', 0); console.log('content:', r.content); console.log('totalLines:', r.totalLines)"` → 期望包含: `你好世界` - 2. [A] 上条命令输出 totalLines → 期望包含: `1` - ---- - -### 场景 3：写入非 UTF-8 编码文件 - -**用户目标:** 用户通过 FileEditTool/FileWriteTool 编辑 GBK 文件后写回，内容不损坏 - -**触发路径:** -1. 系统检测原文件编码 -2. 编辑内容后写回 -3. 非标准编码回退为 UTF-8 写入（零依赖约束） - -#### - [x] 3.1 GBK 文件写入（UTF-8 回退） -- **来源:** spec-plan-acceptance.md §7 / spec-design.md §写入路径改造 -- **目的:** 确认非 UTF-8 编码写入不损坏内容 -- **操作步骤:** - 1. [A] `bun -e "import { writeTextContent } from './src/utils/file.js'; writeTextContent('/tmp/test-gbk-write.txt', '测试写入', 'gbk', 'LF'); const fs = require('fs'); const content = fs.readFileSync('/tmp/test-gbk-write.txt', 'utf8'); console.log('written:', content)"` → 期望包含: `测试写入` - ---- - -### 场景 4：UTF-8 文件读取回归 - -**用户目标:** 用户读取 UTF-8 文件，行为与改动前完全一致 - -**触发路径:** -1. UTF-8 fatal 验证通过 -2. 内容正常输出 - -#### - [x] 4.1 UTF-8 文件读取回归 -- **来源:** spec-plan-acceptance.md §4 / spec-design.md §验收标准 -- **目的:** 确认 UTF-8 读取无回归 -- **操作步骤:** - 1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf8.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `Hello 世界` - 2. [A] 上条命令输出 encoding 字段 → 期望包含: `utf` - ---- - -### 场景 5：UTF-16LE 文件读取回归 - -**用户目标:** 用户读取 UTF-16LE（BOM）文件，行为与改动前完全一致 - -**触发路径:** -1. BOM 检测层识别 FF FE 标记 -2. 用 UTF-16LE 解码 - -#### - [x] 5.1 UTF-16LE 文件读取回归 -- **来源:** spec-plan-acceptance.md §5 / spec-design.md §验收标准 -- **目的:** 确认 UTF-16LE BOM 读取无回归 -- **操作步骤:** - 1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf16le.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `utf-16le` - 2. [A] 上条命令输出 content 字段 → 期望包含: `Hello` - ---- - -### 场景 6：编码检测性能 - -**用户目标:** 编码检测不应影响文件读取的响应速度 - -**触发路径:** -1. 对 4KB 数据执行 1000 次检测 -2. 验证平均耗时 < 1ms - -#### - [x] 6.1 检测性能基准 -- **来源:** spec-plan-acceptance.md §8 / spec-design.md §实现要点 -- **目的:** 确认编码检测性能达标 -- **操作步骤:** - 1. [A] `bun -e "import { detectEncoding } from './src/utils/encoding.js'; const buf = Buffer.alloc(4096, 0x41); const start = performance.now(); for (let i = 0; i < 1000; i++) detectEncoding(buf); const avg = (performance.now() - start) / 1000; console.log('avg:', avg, 'ms'); process.exit(avg < 1 ? 0 : 1)"` → 期望包含: `avg:` - ---- - -### 场景 7：构建和测试完整性 - -**用户目标:** 整体代码质量无退化，所有测试通过 - -**触发路径:** -1. 执行完整 precheck（typecheck + lint + test） -2. 确认零错误 - -#### - [x] 7.1 编码相关单元测试 -- **来源:** spec-plan.md Task 1-4 检查步骤 / spec-design.md §验收标准 -- **目的:** 确认编码相关测试全部通过 -- **操作步骤:** - 1. [A] `bun test src/utils/__tests__/encoding.test.ts` → 期望包含: `0 fail` - 2. [A] `bun test src/utils/__tests__/fileRead.test.ts` → 期望包含: `0 fail` - 3. [A] `bun test src/utils/__tests__/readFileInRange.test.ts` → 期望包含: `0 fail` - 4. [A] `bun test src/utils/__tests__/file.test.ts` → 期望包含: `0 fail` - ---- - -## 验收后清理 - -- [x] [AUTO] 清理临时测试文件: `rm -f /tmp/test-gbk.txt /tmp/test-utf8.txt /tmp/test-utf16le.txt /tmp/test-gbk-write.txt` - ---- - -## 验收结果汇总 - -| 场景 | 序号 | 验收项 | [A] | [H] | 结果 | -|------|------|--------|-----|-----|------| -| 场景 1 | 1.1 | GBK 同步读取 | 2 | 0 | ✅ | -| 场景 1 | 1.2 | GBK 异步路径读取 | 2 | 0 | ✅ | -| 场景 3 | 3.1 | GBK 写入（回退） | 1 | 0 | ✅ | -| 场景 4 | 4.1 | UTF-8 回归 | 2 | 0 | ✅ | -| 场景 5 | 5.1 | UTF-16LE 回归 | 2 | 0 | ✅ | -| 场景 6 | 6.1 | 检测性能 | 1 | 0 | ✅ | -| 场景 7 | 7.1 | 编码单元测试 | 4 | 0 | ✅ | - -**验收结论:** ✅ 全部通过 diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-acceptance.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-acceptance.md deleted file mode 100644 index 812b57efb..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-acceptance.md +++ /dev/null @@ -1,47 +0,0 @@ -### Acceptance Task: 多编码文件工具验收 - -**前置条件:** -- 所有 Task 0-4 已执行完毕 -- 运行环境: 当前开发环境（Bun） - -**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030/ISO-8859-1 已移除。 - -**端到端验证:** - -1. 运行完整测试套件确保无回归 - - `bun run precheck` - - 预期: typecheck + lint fix + test 全部零错误通过 - - 失败排查: 检查各 Task 的测试步骤，特别是 Task 1 的编码检测测试和 Task 3 的 readFileInRange 测试 - -2. 验证 GBK 文件读取正确性 - - 创建 GBK 编码测试文件：`bun -e "const fs = require('fs'); const b = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3, 0xCA, 0xC0, 0xBD, 0xE7, 0x0A]); fs.writeFileSync('/tmp/test-gbk.txt', b)"` - - 读取并验证：`bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-gbk.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` - - 预期: encoding 为 `gbk`，content 为 "你好世界" - - 失败排查: 检查 Task 1 的 detectEncoding 逻辑、Task 2 的 readFileSyncWithMetadata 集成 - -3. 验证 UTF-8 文件读取回归 - - `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const fs = require('fs'); fs.writeFileSync('/tmp/test-utf8.txt', 'Hello 世界\n'); const r = readFileSyncWithMetadata('/tmp/test-utf8.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` - - 预期: encoding 为 `utf-8`，content 为 "Hello 世界" - - 失败排查: 检查 Task 1 的 UTF-8 fatal 验证逻辑 - -4. 验证 UTF-16LE 文件读取回归 - - `bun -e "const fs = require('fs'); const b = Buffer.concat([Buffer.from([0xFF, 0xFE]), Buffer.from('Hello', 'utf16le')]); fs.writeFileSync('/tmp/test-utf16le.txt', b); import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf16le.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` - - 预期: encoding 为 `utf-16le`，content 为 "Hello" - - 失败排查: 检查 Task 1 的 BOM 检测层、Task 2 的集成 - -5. 验证 readFileInRange 异步路径的 GBK 支持 - - `bun -e "import { readFileInRange } from './src/utils/readFileInRange.js'; const r = await readFileInRange('/tmp/test-gbk.txt', 0); console.log('content:', r.content); console.log('totalLines:', r.totalLines)"` - - 预期: content 为 "你好世界"，totalLines 为 1 - - 失败排查: 检查 Task 3 的 fast path 改造 - -6. 验证 GBK 文件写入（UTF-8 回退） - - `bun -e "import { writeTextContent } from './src/utils/file.js'; writeTextContent('/tmp/test-gbk-write.txt', '测试写入', 'gbk', 'LF'); const fs = require('fs'); const content = fs.readFileSync('/tmp/test-gbk-write.txt', 'utf8'); console.log('written:', content)"` - - 预期: 文件成功写入，内容为 "测试写入"（UTF-8 回退或 GBK 编码均可接受） - - 失败排查: 检查 Task 4 的 writeTextContent 改造和 encodeString 函数 - -7. 验证编码检测性能 - - `bun -e "import { detectEncoding } from './src/utils/encoding.js'; const buf = Buffer.alloc(4096, 0x41); const start = performance.now(); for (let i = 0; i < 1000; i++) detectEncoding(buf); console.log('avg:', (performance.now() - start) / 1000, 'ms')"` - - 预期: 平均检测耗时 < 1ms - - 失败排查: 检查 Task 1 的检测逻辑是否有不必要的重复操作 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-0.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-0.md deleted file mode 100644 index 561385398..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-0.md +++ /dev/null @@ -1,34 +0,0 @@ -### Task 0: 环境准备 - -**背景:** -确保构建和测试工具链在当前开发环境中可用，验证 Bun 运行时对 GBK 编码的 TextDecoder 支持情况。 - -**涉及文件:** -- 无文件修改，仅验证环境 - -**执行步骤:** -- [x] 验证 Bun 运行时可用 - - 运行命令: `bun --version` - - 预期: 输出 Bun 版本号 -- [x] 验证 TypeScript 编译无错误 - - 运行命令: `bunx tsc --noEmit 2>&1 | tail -5` - - 预期: 无错误输出（或仅有已知的 pre-existing 错误） -- [x] 验证 Bun 对 GBK 编码的 TextDecoder 支持 - - 运行命令: `bun -e "const d = new TextDecoder('gbk', { fatal: true }); const buf = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3]); console.log(d.decode(buf))"` - - 预期: 输出 "你好"（GBK 编码的中文字符） -- [x] 验证测试框架可用 - - 运行命令: `bun test src/utils/__tests__/hash.test.ts 2>&1 | tail -3` - - 预期: 测试运行成功，无框架错误 - -**检查步骤:** -- [x] Bun 版本确认 - - `bun --version` - - 预期: 输出有效版本号 -- [x] GBK 编码支持确认 - - `bun -e "console.log(new TextDecoder('gbk').decode(Buffer.from([0xC4, 0xE3, 0xBA, 0xC3])))"` - - 预期: 输出 "你好" -- [x] 现有测试通过 - - `bun test src/utils/__tests__/file.test.ts 2>&1 | tail -3` - - 预期: 所有测试通过 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-1.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-1.md deleted file mode 100644 index f51b1a706..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-1.md +++ /dev/null @@ -1,141 +0,0 @@ -### Task 1: 编码检测核心模块 - -**背景:** -当前 `src/utils/fileRead.ts` 的 `detectEncodingForResolvedPath` 仅通过 BOM 头识别 UTF-8 和 UTF-16LE，其他所有文件一律返回 `utf8`，导致 GBK 等非 UTF-8 编码文件读取乱码。本 Task 新建独立的编码检测工具模块 `src/utils/encoding.ts`，实现三层编码检测算法（BOM → UTF-8 fatal 验证 → GBK 回退），为后续 Task 2/3/4 的读写路径改造提供统一的编码检测和解码能力。本 Task 无前置依赖，是后续所有 Task 的基础。 - -**涉及文件:** -- 新建: `src/utils/encoding.ts` -- 新建: `src/utils/__tests__/encoding.test.ts` - -**执行步骤:** - -- [x] 创建 `src/utils/encoding.ts`，定义类型 - - 位置: 文件顶部 - - 导出以下类型: - ```typescript - /** 扩展编码类型，覆盖最常见的非 UTF-8 CJK 编码 */ - export type FileEncoding = BufferEncoding | 'gbk' - - /** TextDecoder 接受的编码名（string），比 FileEncoding 更宽泛 */ - export type DetectedEncoding = string - ``` - - 原因: 后续 Task 2/3/4 需要这些类型来做编码标注和类型收窄 - -- [x] 实现 `detectEncoding(buffer: Buffer): FileEncoding` 函数 - - 位置: `src/utils/encoding.ts`，类型定义之后 - - 三层检测逻辑: - ```typescript - export function detectEncoding(buffer: Buffer): FileEncoding { - // Layer 1: BOM 检测（与现有 fileRead.ts 逻辑一致） - if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { - return 'utf-16le' - } - if ( - buffer.length >= 3 && - buffer[0] === 0xef && - buffer[1] === 0xbb && - buffer[2] === 0xbf - ) { - return 'utf-8' - } - - // Layer 2: UTF-8 fatal 验证 - // fatal: true 模式下，无效 UTF-8 字节序列会抛出 TypeError - try { - new TextDecoder('utf-8', { fatal: true }).decode(buffer) - return 'utf-8' - } catch { - // 不是合法 UTF-8，进入 Layer 3 - } - - // Layer 3: GBK 回退 - try { - new TextDecoder('gbk', { fatal: true }).decode(buffer) - return 'gbk' - } catch { - // 不是合法 GBK，latin1 作为最终兜底 - } - - return 'latin1' - } - ``` - - 原因: BOM 必须优先于 fatal 验证；GBK 作为唯一回退避免了多编码链的字节歧义问题；latin1 单字节编码永远成功 - -- [x] 实现 `decodeBuffer(buffer: Buffer, encoding: DetectedEncoding): string` 函数 - - 位置: `src/utils/encoding.ts`，`detectEncoding` 之后 - - 逻辑: - ```typescript - export function decodeBuffer( - buffer: Buffer, - encoding: DetectedEncoding, - ): string { - return new TextDecoder(encoding).decode(buffer) - } - ``` - - 原因: 统一解码入口，后续 Task 2/3 的读取路径都调用此函数 - -- [x] 实现 `encodeString(content: string, encoding: DetectedEncoding): { buffer: Buffer; converted: boolean }` 函数 - - 位置: `src/utils/encoding.ts`，`decodeBuffer` 之后 - - 逻辑: - ```typescript - export function encodeString( - content: string, - encoding: DetectedEncoding, - ): { buffer: Buffer; converted: boolean } { - if (encoding === 'utf-8' || encoding === 'utf8') { - return { buffer: Buffer.from(content, 'utf-8'), converted: false } - } - if (encoding === 'utf-16le') { - return { buffer: Buffer.from(content, 'utf-16le'), converted: false } - } - - // 其他编码（如 gbk）：尝试 Buffer.from，失败则回退为 UTF-8 - try { - const buf = Buffer.from(content, encoding as BufferEncoding) - return { buffer: buf, converted: false } - } catch { - return { buffer: Buffer.from(content, 'utf-8'), converted: true } - } - } - ``` - - 原因: `Buffer.from` 在 Bun 中可能支持 GBK 编码名，但 Node.js 不支持。try-catch 策略兼容两种运行时；`converted` 标志让 Task 4 的写入路径能向用户报告编码转换 - -- [x] 为编码检测和解码函数编写单元测试 - - 测试文件: `src/utils/__tests__/encoding.test.ts` - - 测试场景: - - **BOM 检测 — UTF-16LE**: 输入 `Buffer.from([0xff, 0xfe, 0x48, 0x00])` → 预期返回 `'utf-16le'` - - **BOM 检测 — UTF-8 BOM**: 输入 `Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65])` → 预期返回 `'utf-8'` - - **UTF-8 验证**: 输入 `Buffer.from('Hello, 世界', 'utf-8')` → 预期返回 `'utf-8'` - - **GBK 检测**: 输入 `Buffer.from([0xc4, 0xe3, 0xba, 0xc3])` → 预期返回 `'gbk'` - - **空 buffer**: 输入 `Buffer.alloc(0)` → 预期返回 `'utf-8'` - - **latin1 兜底**: 输入随机字节 `Buffer.from([0x80, 0x81, 0x82, 0x83, 0x84, 0x85])` → 预期返回 `'latin1'` - - **BOM 优先于内容分析**: 输入带 UTF-8 BOM 的数据 → 预期返回 `'utf-8'` - - **decodeBuffer — UTF-8**: 输入 UTF-8 编码的 buffer + encoding `'utf-8'` → 预期返回正确的中文字符串 - - **decodeBuffer — GBK**: 输入 GBK 编码的 buffer + encoding `'gbk'` → 预期返回正确的中文字符串 - - **decodeBuffer — UTF-16LE**: 输入 UTF-16LE 编码的 buffer + encoding `'utf-16le'` → 预期返回正确字符串 - - **decodeBuffer — 空 buffer**: 输入空 buffer → 预期返回空字符串 - - **encodeString — UTF-8**: 输入字符串 + encoding `'utf-8'` → 预期 `{ converted: false }` - - **encodeString — utf8 别名**: 输入字符串 + encoding `'utf8'` → 预期 `{ converted: false }` - - **encodeString — UTF-16LE**: 输入字符串 + encoding `'utf-16le'` → 预期 `{ converted: false }` - - **encodeString — GBK**: 输入字符串 + encoding `'gbk'` → 预期返回有效的 Buffer（converted 视运行时而定） - - 运行命令: `bun test src/utils/__tests__/encoding.test.ts` - - 预期: 所有测试通过 - -**检查步骤:** - -- [x] 验证 `encoding.ts` 文件存在且导出正确 - - `grep -c "export" src/utils/encoding.ts` - - 预期: 输出 >= 4（至少导出 FileEncoding, DetectedEncoding, detectEncoding, decodeBuffer, encodeString 共 5 个导出） - -- [x] 验证类型检查通过 - - `bunx tsc --noEmit src/utils/encoding.ts 2>&1 | head -5` - - 预期: 无类型错误输出 - -- [x] 运行编码检测单元测试 - - `bun test src/utils/__tests__/encoding.test.ts` - - 预期: 所有测试通过，无失败用例 - -**认知变更:** -- [x] [CLAUDE.md] `src/utils/encoding.ts` 是文件编码检测的唯一入口，提供 `detectEncoding`（三层检测：BOM → UTF-8 fatal → GBK 回退）和 `decodeBuffer`/`encodeString` 函数。检测基于文件头部 4KB，零外部依赖，仅使用 TextDecoder API。`FileEncoding` 类型为 `BufferEncoding | 'gbk'`，覆盖最常见非 UTF-8 CJK 编码。latin1 作为最终兜底编码（单字节编码永远成功）。 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-2.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-2.md deleted file mode 100644 index 86bfc0f39..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-2.md +++ /dev/null @@ -1,163 +0,0 @@ -### Task 2: 同步读取路径集成 - -**背景:** -当前同步读取路径（`fileRead.ts` → `file.ts` → `fileReadCache.ts`）的编码检测仅通过 BOM 头识别 UTF-8 和 UTF-16LE，非 BOM 编码文件一律按 UTF-8 读取导致乱码。本 Task 将 `detectEncodingForResolvedPath` 的内部实现从 BOM-only 升级为调用 Task 1 创建的 `encoding.ts` 三层检测，并将返回类型从 `BufferEncoding` 扩展为 `FileEncoding`。同时将所有 `fs.readFileSync(path, { encoding })` 调用改为先读 Buffer 再用 `decodeBuffer` 解码，以支持 `gbk` 等非 `BufferEncoding` 编码。本 Task 依赖 Task 1（`src/utils/encoding.ts`），输出被 Task 4（写入路径适配）依赖。 - -**涉及文件:** -- 修改: `src/utils/fileRead.ts` -- 修改: `src/utils/file.ts` -- 修改: `src/utils/fileReadCache.ts` -- 新建: `src/utils/__tests__/fileRead.test.ts` - -**执行步骤:** - -- [x] 在 `fileRead.ts` 中导入 `encoding.ts` 的类型和函数 - - 位置: `src/utils/fileRead.ts` 文件顶部 import 区域，在 `import { getFsImplementation, safeResolvePath } from './fsOperations.js'` 之后 - - 添加导入: - ```typescript - import { type FileEncoding, decodeBuffer, detectEncoding } from './encoding.js' - ``` - - 原因: 后续步骤需要 `FileEncoding` 类型、`detectEncoding` 检测函数和 `decodeBuffer` 解码函数 - -- [x] 改造 `detectEncodingForResolvedPath` 函数，使用 `encoding.ts` 的三层检测 - - 位置: `src/utils/fileRead.ts` 的 `detectEncodingForResolvedPath` 函数 - - 将函数体替换为以下逻辑: - ```typescript - export function detectEncodingForResolvedPath( - resolvedPath: string, - ): FileEncoding { - const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, { - length: 4096, - }) - - // Empty files default to utf8 — nothing to detect - if (bytesRead === 0) { - return 'utf8' - } - - return detectEncoding(buffer.subarray(0, bytesRead)) - } - ``` - - 关键变更: - - 返回类型从 `BufferEncoding` 改为 `FileEncoding` - - 删除内联的 BOM 检测逻辑，改为调用 `detectEncoding(buffer.subarray(0, bytesRead))` - - 使用 `buffer.subarray(0, bytesRead)` 截取实际读取的字节，避免尾部零字节干扰检测 - - 原因: 将检测逻辑委托给 `encoding.ts` 的三层算法，消除代码重复 - -- [x] 改造 `readFileSyncWithMetadata` 函数，支持非 `BufferEncoding` 解码 - - 位置: `src/utils/fileRead.ts` 的 `readFileSyncWithMetadata` 函数 - - 将函数签名和内部逻辑改为: - ```typescript - export function readFileSyncWithMetadata(filePath: string): { - content: string - encoding: FileEncoding - lineEndings: LineEndingType - } { - const fs = getFsImplementation() - const { resolvedPath, isSymlink } = safeResolvePath(fs, filePath) - - if (isSymlink) { - logForDebugging(`Reading through symlink: ${filePath} -> ${resolvedPath}`) - } - - const encoding = detectEncodingForResolvedPath(resolvedPath) - // Read raw Buffer first — fs.readFileSync encoding option only accepts - // BufferEncoding, not gbk etc. - const rawBuffer = fs.readFileBytesSync(resolvedPath) - const raw = decodeBuffer(rawBuffer, encoding) - const lineEndings = detectLineEndingsForString(raw.slice(0, 4096)) - return { - content: raw.replaceAll('\r\n', '\n'), - encoding, - lineEndings, - } - } - ``` - - 关键变更: - - 返回类型中 `encoding` 从 `BufferEncoding` 改为 `FileEncoding` - - `fs.readFileSync(resolvedPath, { encoding })` 改为 `fs.readFileBytesSync(resolvedPath)` 读取 Buffer - - 新增 `decodeBuffer(rawBuffer, encoding)` 解码为字符串 - - 原因: `fs.readFileSync` 的 `encoding` 选项只接受 `BufferEncoding`（utf8/utf16le/latin1 等），传入 `'gbk'` 会在运行时报错 - -- [x] 更新 `file.ts` 中 `detectFileEncoding` 的返回类型 - - 位置: `src/utils/file.ts` 的 `detectFileEncoding` 函数签名 - - 将 `): BufferEncoding {` 改为 `): FileEncoding {` - - 在文件顶部 import 区域添加: - ```typescript - import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js' - ``` - - 原因: `detectFileEncoding` 调用 `detectEncodingForResolvedPath`，返回类型已改为 `FileEncoding` - -- [x] 更新 `file.ts` 中 `detectLineEndings` 的 encoding 参数类型和解码逻辑 - - 位置: `src/utils/file.ts` 的 `detectLineEndings` 函数 - - 将函数签名改为: - ```typescript - export function detectLineEndings( - filePath: string, - encoding: FileEncoding = 'utf8', - ): LineEndingType { - ``` - - 将内部 `buffer.toString(encoding, 0, bytesRead)` 改为: - ```typescript - const content = decodeBuffer(buffer.subarray(0, bytesRead), encoding) - ``` - - 原因: `buffer.toString('gbk')` 不可靠，统一使用 `decodeBuffer` 通过 `TextDecoder` 解码 - -- [x] 更新 `fileReadCache.ts` 的类型和解码逻辑 - - 位置: `src/utils/fileReadCache.ts` - - 在文件顶部 import 区域添加: - ```typescript - import { type FileEncoding, decodeBuffer } from './encoding.js' - ``` - - 将 `CachedFileData` 类型中 `encoding: BufferEncoding` 改为 `encoding: FileEncoding` - - 将 `readFile` 方法返回类型改为 `{ content: string; encoding: FileEncoding }` - - 将缓存未命中读取逻辑改为: - ```typescript - const encoding = detectFileEncoding(filePath) - const rawBuffer = fs.readFileBytesSync(filePath) - const content = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n') - ``` - - 原因: 与 `fileRead.ts` 相同——必须改为 Buffer 读取 + `decodeBuffer` 解码 - -- [x] 为改造后的 `detectEncodingForResolvedPath` 和 `readFileSyncWithMetadata` 编写单元测试 - - 测试文件: `src/utils/__tests__/fileRead.test.ts` - - 测试场景: - - **UTF-8 文件读取**: 创建临时 UTF-8 文件 → 返回 `encoding: 'utf-8'`，content 与写入内容一致 - - **GBK 文件读取**: 创建临时 GBK 编码文件 → 返回 `encoding: 'gbk'`，content 包含正确的中文字符 - - **空文件读取**: 创建空文件 → 返回 `encoding: 'utf8'`，content 为空字符串 - - **UTF-16LE BOM 文件读取**: 创建带 BOM 的 UTF-16LE 文件 → 返回 `encoding: 'utf-16le'` - - **detectEncodingForResolvedPath 返回类型**: 验证返回值为 `FileEncoding` 类型 - - Mock 策略: 使用 `tests/mocks/debug.ts` mock `debug.ts`，使用 `tests/mocks/log.ts` mock `log.ts` - - 运行命令: `bun test src/utils/__tests__/fileRead.test.ts` - - 预期: 所有测试通过 - -**检查步骤:** - -- [x] 验证 `fileRead.ts` 的导入和返回类型已更新 - - `grep -n "FileEncoding\|decodeBuffer\|detectEncoding" src/utils/fileRead.ts` - - 预期: 输出包含 import 行中的 `FileEncoding`、`decodeBuffer`，以及函数体中的 `detectEncoding` 调用 - -- [x] 验证 `file.ts` 的类型已更新 - - `grep -n "FileEncoding\|decodeBuffer" src/utils/file.ts` - - 预期: `detectFileEncoding` 返回 `FileEncoding`，`detectLineEndings` 参数类型为 `FileEncoding` - -- [x] 验证 `fileReadCache.ts` 的类型已更新 - - `grep -n "FileEncoding\|decodeBuffer" src/utils/fileReadCache.ts` - - 预期: `CachedFileData` 和 `readFile` 返回类型使用 `FileEncoding` - -- [x] 验证 `fileRead.ts` 中不再有内联 BOM 检测逻辑 - - `grep -c "0xff\|0xfe\|0xef\|0xbb\|0xbf" src/utils/fileRead.ts` - - 预期: 输出为 0 - -- [x] 运行 fileRead 单元测试 - - `bun test src/utils/__tests__/fileRead.test.ts` - - 预期: 所有测试通过 - -- [x] 运行 precheck 确认无类型/lint/测试错误 - - `bun run precheck` - - 预期: 零错误通过 - -**认知变更:** -- [x] [CLAUDE.md] `fs.readFileSync(path, { encoding })` 的 `encoding` 选项只接受 `BufferEncoding`（utf8/utf16le/latin1/ascii/binary/hex/base64/ucs2/utf16le），不支持 `gbk` 等 ICU 编码名。读取非 UTF-8 文件时必须先 `fs.readFileSync(path)` 读 Buffer，再用 `TextDecoder` 解码。项目中所有文件读取路径（fileRead.ts、fileReadCache.ts、file.ts）已统一使用 `decodeBuffer` 函数处理此逻辑。 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-3.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-3.md deleted file mode 100644 index 3b653a7fd..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-3.md +++ /dev/null @@ -1,161 +0,0 @@ -### Task 3: 异步读取路径改造 - -**背景:** -当前 `src/utils/readFileInRange.ts` 是 FileReadTool 的核心异步读取函数，提供 fast path（小文件整体读入）和 streaming path（大文件逐块扫描）两条路径，两者均硬编码 `encoding: 'utf8'`，导致非 UTF-8 编码文件读取乱码。本 Task 将两条路径改造为 Buffer 读取 + 编码检测 + TextDecoder 解码模式。fast path 改造简单（整体读 Buffer 后检测解码），streaming path 需要两阶段设计（先收集前 4KB 做编码检测，再用 `TextDecoder({ stream: true })` 逐 chunk 解码）。本 Task 依赖 Task 1（`src/utils/encoding.ts` 的 `detectEncoding` 和 `decodeBuffer`），输出被 Task 4 依赖（通过 `readFileInRange` 的返回值间接影响）。 - -**涉及文件:** -- 修改: `src/utils/readFileInRange.ts` -- 新建: `src/utils/__tests__/readFileInRange.test.ts` - -**执行步骤:** - -- [x] 在 `readFileInRange.ts` 中导入 `encoding.ts` 的函数 - - 位置: `src/utils/readFileInRange.ts` 文件顶部 import 区域，在 `import { formatFileSize } from './format.js'` 之后 - - 添加导入: - ```typescript - import { detectEncoding, decodeBuffer } from './encoding.js' - ``` - - 原因: fast path 和 streaming path 都需要 `detectEncoding` 做编码检测，fast path 需要 `decodeBuffer` 做一次性解码 - -- [x] 改造 fast path — 将 `readFile` 从 UTF-8 字符串读取改为 Buffer 读取 + 检测 + 解码 - - 位置: `src/utils/readFileInRange.ts` 的 `readFileInRange` 函数内 fast path 分支 - - 将以下代码: - ```typescript - const text = await readFile(filePath, { encoding: 'utf8', signal }) - return readFileInRangeFast(text, stats.mtimeMs, offset, maxLines, ...) - ``` - 替换为: - ```typescript - const rawBuffer = await readFile(filePath, { signal }) - const encoding = detectEncoding(rawBuffer) - const text = decodeBuffer(rawBuffer, encoding) - return readFileInRangeFast(text, stats.mtimeMs, offset, maxLines, ...) - ``` - - 关键变更: `readFile` 去掉 `encoding: 'utf8'` 选项，返回 `Buffer`；调用 `detectEncoding(rawBuffer)` 检测编码；调用 `decodeBuffer(rawBuffer, encoding)` 解码为字符串。 - - 原因: `readFile` 的 `encoding` 选项只支持 `BufferEncoding`，不支持 `gbk` 等 ICU 编码名 - -- [x] 改造 streaming path — 扩展 `StreamState` 类型，增加编码检测和解码相关字段 - - 位置: `src/utils/readFileInRange.ts` 的 `StreamState` 类型定义 - - 在现有字段之后添加以下字段: - ```typescript - type StreamState = { - // ... 现有字段保持不变 ... - /** 编码检测状态：null 表示尚未检测，string 表示已检测完成 */ - encoding: string | null - /** TextDecoder 实例：检测完成后创建，用于逐 chunk 流式解码 */ - decoder: TextDecoder | null - /** 检测阶段缓冲区：收集原始字节直到满 4KB 或 stream 结束 */ - detectionBuffer: number[] - } - ``` - - 原因: streaming 模式下 chunk 是增量到达的，需要缓冲阶段收集足够字节来调用 `detectEncoding` - -- [x] 改造 `streamOnData` — 处理 Buffer chunk，实现两阶段（检测阶段 + 解码阶段） - - 位置: `src/utils/readFileInRange.ts` 的 `streamOnData` 函数 - - 将函数签名从 `streamOnData(this: StreamState, chunk: string): void` 改为 `streamOnData(this: StreamState, chunk: Buffer): void` - - 替换函数体为两阶段逻辑: - ```typescript - function streamOnData(this: StreamState, chunk: Buffer): void { - this.totalBytesRead += chunk.length - - // ... maxBytes 检查保持不变 ... - - // Phase 1: 编码检测阶段 - if (this.encoding === null) { - for (let i = 0; i < chunk.length; i++) { - this.detectionBuffer.push(chunk[i]) - } - if (this.detectionBuffer.length >= 4096) { - this.encoding = detectEncoding(Buffer.from(this.detectionBuffer)) - this.decoder = new TextDecoder(this.encoding, { stream: true }) - const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer)) - this.detectionBuffer = [] - processTextChunk(this, decoded) - } - return - } - - // Phase 2: 解码阶段 - const decoded = this.decoder!.decode(chunk, { stream: true }) - processTextChunk(this, decoded) - } - ``` - - 原因: 两阶段设计确保编码检测在足够数据上执行（至少 4KB），检测完成后用 `TextDecoder({ stream: true })` 逐 chunk 解码 - -- [x] 提取行扫描逻辑为独立的 `processTextChunk` 辅助函数 - - 位置: `src/utils/readFileInRange.ts`，在 `streamOnData` 函数定义之前 - - 从原 `streamOnData` 提取行扫描逻辑到独立函数 `processTextChunk(state: StreamState, text: string): void` - - 行扫描逻辑与原实现完全一致，仅变量名从 `this.` 改为 `state.` - - 原因: 检测阶段和解码阶段复用同一段行扫描逻辑 - -- [x] 改造 `streamOnEnd` — 处理检测阶段缓冲区残留和最终 fragment - - 位置: `src/utils/readFileInRange.ts` 的 `streamOnEnd` 函数 - - 在函数体开头插入检测阶段完成逻辑: - ```typescript - if (this.encoding === null) { - this.encoding = detectEncoding(Buffer.from(this.detectionBuffer)) - this.decoder = new TextDecoder(this.encoding, { stream: true }) - const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer)) - this.detectionBuffer = [] - processTextChunk(this, decoded) - } - ``` - - 原因: 小文件可能 < 4KB，stream 在检测缓冲区未满时就结束。必须在 `streamOnEnd` 中完成检测和解码 - -- [x] 改造 `readFileInRangeStreaming` — 创建 Buffer 模式的 stream，初始化新增字段 - - 位置: `src/utils/readFileInRange.ts` 的 `readFileInRangeStreaming` 函数 - - 将 `createReadStream` 调用去掉 `encoding: 'utf8'` 选项 - - 在 `state` 对象初始化中添加新字段: `encoding: null, decoder: null, detectionBuffer: []` - - 原因: 去掉 `encoding: 'utf8'` 后，`data` 事件回调接收 `Buffer` 对象 - -- [x] 更新文件顶部注释，反映编码检测能力 - - 位置: `src/utils/readFileInRange.ts` 文件顶部注释 - - 注释已更新为: `Both paths auto-detect encoding via encoding.ts (BOM → UTF-8 fatal → fallback chain), decode with TextDecoder, and strip BOM and \r (CRLF → LF).` - -- [x] 为改造后的 `readFileInRange` 编写单元测试 - - 测试文件: `src/utils/__tests__/readFileInRange.test.ts` - - 测试场景: - - **Fast path — UTF-8 文件**: 创建临时 UTF-8 文件 → 返回正确的 `content`、`lineCount`、`totalLines` - - **Fast path — GBK 文件**: 创建临时 GBK 编码文件 → 返回正确的中文内容（非乱码），`totalBytes` 正确 - - **Fast path — 带行范围读取 GBK 文件**: 创建包含多行的 GBK 文件 → 返回指定行范围，内容正确 - - **Streaming path — 大 UTF-8 文件**: 创建超过 10MB 阈值的 UTF-8 文件 → 返回正确内容 - - **Streaming path — 大 GBK 文件**: 创建超过 10MB 阈值的 GBK 编码文件 → 返回正确的中文内容 - - **BOM 剥离**: 创建带 UTF-8 BOM 的文件 → `content` 不包含 BOM 字符 - - **空文件**: 创建空文件 → `content` 为空字符串，`totalLines` 为 1，`totalBytes` 为 0 - - 运行命令: `bun test src/utils/__tests__/readFileInRange.test.ts` - - 预期: 所有测试通过 - -**检查步骤:** - -- [x] 验证 `readFileInRange.ts` 已导入 `encoding.ts` 的函数 - - `grep -n "detectEncoding\|decodeBuffer" src/utils/readFileInRange.ts` - - 预期: import 行包含 `detectEncoding` 和 `decodeBuffer`，函数体中包含调用 - -- [x] 验证 streaming path 不再硬编码 `encoding: 'utf8'` - - `grep -n "encoding: 'utf8'\|encoding: \"utf8\"" src/utils/readFileInRange.ts` - - 预期: 无匹配结果 - -- [x] 验证 `createReadStream` 调用无 encoding 选项 - - `grep -A3 "createReadStream" src/utils/readFileInRange.ts` - - 预期: `createReadStream` 的选项对象中不包含 `encoding` 属性 - -- [x] 验证 `StreamState` 类型包含编码检测新字段 - - `grep -n "encoding:\|decoder:\|detectionBuffer:" src/utils/readFileInRange.ts` - - 预期: `StreamState` 类型定义中包含 `encoding`、`decoder`、`detectionBuffer` 字段 - -- [x] 验证 `processTextChunk` 函数存在 - - `grep -n "function processTextChunk" src/utils/readFileInRange.ts` - - 预期: 函数定义存在 - -- [x] 运行 readFileInRange 单元测试 - - `bun test src/utils/__tests__/readFileInRange.test.ts` - - 预期: 所有测试通过 - -- [x] 运行 precheck 确认无类型/lint/测试错误 - - `bun run precheck` - - 预期: 零错误通过 - -**认知变更:** -- [x] [CLAUDE.md] `readFileInRange.ts` 的 streaming path 使用两阶段编码检测：先收集前 4KB 字节调用 `detectEncoding`，再用 `TextDecoder({ stream: true })` 逐 chunk 流式解码。`TextDecoder` 的 `{ stream: true }` 模式会自动处理多字节字符跨 chunk 边界问题。对于 < 4KB 的小文件，检测在 `streamOnEnd` 中完成。 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-4.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-4.md deleted file mode 100644 index 52875126b..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan-task-4.md +++ /dev/null @@ -1,155 +0,0 @@ -### Task 4: 写入路径和工具层适配 - -**背景:** -[业务语境] — 当用户通过 FileEditTool 或 FileWriteTool 编辑非 UTF-8 编码文件（如 GBK）时，写入操作需要将内部 UTF-8 字符串编码回原文件编码，否则写入的内容会乱码。当前 `writeTextContent` 只接受 `BufferEncoding` 类型，无法处理 gbk 等编码。 -[修改原因] — `writeTextContent` 的 `encoding` 参数类型为 `BufferEncoding`，`writeFileSyncAndFlush_DEPRECATED` 内部直接将 encoding 传给 `fs.writeFileSync`（只接受标准 BufferEncoding）。`FileEditTool.validateInput` 中硬编码了 BOM-only 编码检测，无法识别 GBK 文件。 -[上下游影响] — 本 Task 依赖 Task 1 创建的 `encodeString` 函数和 `FileEncoding` 类型。`FileEditTool` 和 `FileWriteTool` 通过 `writeTextContent` 间接依赖本 Task 的改造。BashTool 和 NotebookEditTool 也调用 `writeTextContent`，签名变更后它们无需额外改动（encoding 参数类型由上游传入，自动兼容）。 - -**涉及文件:** -- 修改: `src/utils/file.ts` -- 修改: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts` - -**执行步骤:** - -- [x] 在 `src/utils/file.ts` 中合并 `encodeString` 到 Task 2 已创建的 `encoding.js` 导入 - - 位置: 文件导入区域，Task 2 已添加的 `import { type FileEncoding, decodeBuffer } from './encoding.js'` 行 - - 将该行改为: `import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js'` - - 原因: 避免对同一模块创建两个 import 语句 - -- [x] 将 `writeTextContent` 的 `encoding` 参数类型从 `BufferEncoding` 改为 `FileEncoding` - - 位置: `src/utils/file.ts:writeTextContent()` - - 修改函数签名: - ```typescript - export function writeTextContent( - filePath: string, - content: string, - encoding: FileEncoding, - endings: LineEndingType, - ): void - ``` - - 修改函数体，在行尾处理之后、调用 `writeFileSyncAndFlush_DEPRECATED` 之前，增加编码判断逻辑: - ```typescript - const BUFFER_ENCODINGS = new Set([ - 'utf8', 'utf-8', 'utf16le', 'ucs2', 'ucs-2', - 'ascii', 'latin1', 'binary', 'base64', 'hex', - ]) - - if (BUFFER_ENCODINGS.has(encoding)) { - writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, { encoding: encoding as BufferEncoding }) - } else { - // 非 BufferEncoding（如 gbk），使用 encodeString 获取 Buffer - const { buffer, converted } = encodeString(toWrite, encoding) - writeFileSyncAndFlush_DEPRECATED(filePath, buffer, { buffer }) - if (converted) { - logForDebugging( - `writeTextContent: encoding '${encoding}' unsupported for write, fell back to UTF-8 for ${filePath}`, - { level: 'warn' }, - ) - } - } - ``` - - 原因: `fs.writeFileSync` 只接受标准 BufferEncoding，对于 gbk 等编码必须先转为 Buffer 再写入 - -- [x] 扩展 `writeFileSyncAndFlush_DEPRECATED` 支持 Buffer 写入 - - 位置: `src/utils/file.ts:writeFileSyncAndFlush_DEPRECATED()` - - 修改函数签名中 `content` 参数类型和 `options` 类型: - ```typescript - export function writeFileSyncAndFlush_DEPRECATED( - filePath: string, - content: string | Buffer, - options: { encoding?: BufferEncoding; mode?: number; buffer?: Buffer } = {}, - ): void - ``` - - 修改原子写入路径的 `writeOptions` 构建逻辑: - ```typescript - const isBufferWrite = Buffer.isBuffer(content) || options.buffer !== undefined - const writeData = options.buffer ?? content - const writeOptions: { - encoding?: BufferEncoding - flush: boolean - mode?: number - } = { - flush: true, - ...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }), - } - ``` - - 修改非原子回退路径，使用相同的 `isBufferWrite` / `writeData` / `writeOptions` 模式 - - 原因: `fs.writeFileSync(path, buffer)` 可以直接写入 Buffer，不需要 encoding 参数 - -- [x] 在 `FileEditTool.ts` 中导入 `FileEncoding` 和 `detectEncoding` / `decodeBuffer` - - 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts` 导入区域 - - 添加: `import { detectEncoding, decodeBuffer, type FileEncoding } from 'src/utils/encoding.js'` - - 原因: `validateInput` 编码检测和 `readFileForEdit` 返回类型需要 `FileEncoding` 类型 - -- [x] 将 `readFileForEdit` 返回类型中的 `encoding` 从 `BufferEncoding` 改为 `FileEncoding` - - 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts:readFileForEdit()` - - 修改返回类型声明: - ```typescript - function readFileForEdit(absoluteFilePath: string): { - content: string - fileExists: boolean - encoding: FileEncoding - lineEndings: LineEndingType - } - ``` - - 原因: `readFileSyncWithMetadata` 返回的 `encoding` 类型已由 Task 2 改为 `FileEncoding` - -- [x] 改造 `FileEditTool.validateInput` 中的编码检测逻辑 - - 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts:validateInput()` - - 将现有的 BOM-only 编码检测: - ```typescript - const encoding: BufferEncoding = - fileBuffer.length >= 2 && fileBuffer[0] === 0xff && fileBuffer[1] === 0xfe - ? 'utf16le' - : 'utf8' - fileContent = fileBuffer.toString(encoding).replaceAll('\r\n', '\n') - ``` - - 替换为: - ```typescript - const encoding: FileEncoding = detectEncoding(fileBuffer) - fileContent = decodeBuffer(fileBuffer, encoding).replaceAll('\r\n', '\n') - ``` - - 原因: 使 validateInput 也能正确识别 GBK 文件，避免编辑时因编码检测不一致导致 old_string 匹配失败 - -- [x] 为 `writeTextContent` 的多编码写入能力编写单元测试 - - 测试文件: `src/utils/__tests__/file.test.ts` - - 在现有测试 describe 块之后追加新的 describe('writeTextContent with multi-encoding') 块 - - 测试场景: - - UTF-8 写入: 写入 UTF-8 内容 → 文件内容正确，无回退警告 - - UTF-16LE 写入: 写入 UTF-16LE 内容（含 BOM） → 文件二进制内容与预期一致 - - GBK 写入回退: 对 gbk 编码调用 `writeTextContent` → 文件以 UTF-8 写入（`encodeString` 回退行为），内容不损坏 - - CRLF 行尾 + GBK: `endings: 'CRLF'` + gbk 编码 → 行尾正确转换为 `\r\n`，编码回退为 UTF-8 - - 注意: 需要 mock `src/utils/debug.ts`（使用共享 mock `tests/mocks/debug.ts`） - - 运行命令: `bun test src/utils/__tests__/file.test.ts` - - 预期: 所有测试通过 - -**检查步骤:** -- [x] 验证 `writeTextContent` 签名使用 `FileEncoding` 类型 - - `grep -n 'encoding: FileEncoding' src/utils/file.ts` - - 预期: 输出包含 `writeTextContent` 函数定义行 - -- [x] 验证 `writeFileSyncAndFlush_DEPRECATED` 支持 Buffer 写入 - - `grep -n 'content: string | Buffer' src/utils/file.ts` - - 预期: 输出包含 `writeFileSyncAndFlush_DEPRECATED` 函数定义行 - -- [x] 验证 `FileEditTool.readFileForEdit` 返回类型已更新 - - `grep -n 'encoding: FileEncoding' packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts` - - 预期: 输出包含 `readFileForEdit` 函数的返回类型声明 - -- [x] 验证 `FileEditTool.validateInput` 使用 `detectEncoding` - - `grep -n 'detectEncoding' packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts` - - 预期: 输出包含 validateInput 内部的调用 - -- [x] 运行 file.ts 单元测试 - - `bun test src/utils/__tests__/file.test.ts` - - 预期: 所有测试通过，无新增失败 - -- [x] 运行 FileEditTool 工具函数测试 - - `bun test packages/builtin-tools/src/tools/FileEditTool/__tests__/utils.test.ts` - - 预期: 所有现有测试通过 - -- [x] 运行完整 precheck - - `bun run precheck` - - 预期: typecheck + lint + test 零错误通过 - ---- diff --git a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md b/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md deleted file mode 100644 index 60d72162a..000000000 --- a/spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md +++ /dev/null @@ -1,49 +0,0 @@ -# 多编码文件工具 执行计划 - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**目标:** 为文件读写工具添加自动编码检测，支持 GBK 编码的透明读写（latin1 作为最终兜底）。 - -**技术栈:** TextDecoder/TextEncoder（零外部依赖）、Bun test 框架、TypeScript strict mode - -**设计文档:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md - -**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030/ISO-8859-1 已移除。 - -## 改动总览 - -新建编码检测核心模块 `src/utils/encoding.ts`，提供三层检测（BOM → UTF-8 fatal 验证 → GBK 回退 → latin1 兜底）和解码工具函数。同步读取路径（fileRead.ts → file.ts → fileReadCache.ts）集成新检测逻辑，异步读取路径（readFileInRange.ts）改造为 Buffer 读取 + 检测后解码。写入路径（writeTextContent）扩展类型支持新编码名，非标准编码回退为 UTF-8 写入。FileEditTool 和 FileWriteTool 仅需类型适配。 - ---- - -## 任务索引 - -### Task 0: 环境准备 -📄 详情见: `spec-plan-task-0.md` - -验证构建工具链和测试环境是否就绪，确认 Bun 运行时对 GBK 编码的 TextDecoder 支持。 - -### Task 1: 编码检测核心模块 -📄 详情见: `spec-plan-task-1.md` - -新建 `src/utils/encoding.ts`，实现三层编码检测算法（BOM → UTF-8 fatal 验证 → GBK 回退）和 Buffer 解码/编码函数。 - -### Task 2: 同步读取路径集成 -📄 详情见: `spec-plan-task-2.md` - -改造 `fileRead.ts` 和 `file.ts` 的编码检测，集成新模块，更新类型定义。 - -### Task 3: 异步读取路径改造 -📄 详情见: `spec-plan-task-3.md` - -改造 `readFileInRange.ts` 的 fast path 和 streaming path，支持非 UTF-8 编码。 - -### Task 4: 写入路径和工具层适配 -📄 详情见: `spec-plan-task-4.md` - -扩展写入路径类型，更新 FileEditTool/FileWriteTool 的类型注解。 - -### Acceptance Task -📄 详情见: `spec-plan-acceptance.md` - -端到端验证所有功能是否正确实现。 diff --git a/src/components/permissions/SedEditPermissionRequest/SedEditPermissionRequest.tsx b/src/components/permissions/SedEditPermissionRequest/SedEditPermissionRequest.tsx index 6f53fa8f4..74f66d876 100644 --- a/src/components/permissions/SedEditPermissionRequest/SedEditPermissionRequest.tsx +++ b/src/components/permissions/SedEditPermissionRequest/SedEditPermissionRequest.tsx @@ -3,7 +3,6 @@ import React, { Suspense, use, useMemo } from 'react'; import { FileEditToolDiff } from 'src/components/FileEditToolDiff.js'; import { getCwd } from 'src/utils/cwd.js'; import { isENOENT } from 'src/utils/errors.js'; -import { decodeBuffer } from 'src/utils/encoding.js'; import { detectEncodingForResolvedPath } from 'src/utils/fileRead.js'; import { getFsImplementation } from 'src/utils/fsOperations.js'; import { Text } from '@anthropic/ink'; @@ -34,10 +33,9 @@ export function SedEditPermissionRequest({ sedInfo, ...props }: SedEditPermissio // render correctly. This matches what readFileSync did before the // async conversion. const encoding = detectEncodingForResolvedPath(filePath); - const rawBuffer = await getFsImplementation().readFileBytes(filePath); - const raw = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n'); + const raw = await getFsImplementation().readFile(filePath, { encoding }); return { - oldContent: raw, + oldContent: raw.replaceAll('\r\n', '\n'), fileExists: true, }; })().catch((e: unknown): FileReadResult => { diff --git a/src/utils/__tests__/encoding.test.ts b/src/utils/__tests__/encoding.test.ts deleted file mode 100644 index 69b6f4d26..000000000 --- a/src/utils/__tests__/encoding.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, test, expect } from 'bun:test' -import { - detectEncoding, - decodeBuffer, - encodeString, - type FileEncoding, - type DetectedEncoding, -} from '../encoding' - -describe('detectEncoding', () => { - test('detects UTF-16LE BOM', () => { - const buf = Buffer.from([0xff, 0xfe, 0x48, 0x00]) - expect(detectEncoding(buf)).toBe('utf-16le') - }) - - test('detects UTF-8 BOM', () => { - const buf = Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65]) - expect(detectEncoding(buf)).toBe('utf-8') - }) - - test('detects valid UTF-8 without BOM', () => { - const buf = Buffer.from('Hello, 世界', 'utf-8') - expect(detectEncoding(buf)).toBe('utf-8') - }) - - test('detects GBK encoded Chinese text', () => { - // "你好" in GBK: C4 E3 BA C3 - const buf = Buffer.from([0xc4, 0xe3, 0xba, 0xc3]) - expect(detectEncoding(buf)).toBe('gbk') - }) - - test('returns utf-8 for empty buffer', () => { - const buf = Buffer.alloc(0) - expect(detectEncoding(buf)).toBe('utf-8') - }) - - test('falls back to latin1 for random bytes', () => { - // Random bytes that aren't valid UTF-8 or GBK - const buf = Buffer.from([0x80, 0x81, 0x82, 0x83, 0x84, 0x85]) - expect(detectEncoding(buf)).toBe('latin1') - }) - - test('prioritizes BOM over content analysis', () => { - // UTF-8 BOM followed by bytes that could be confused - const buf = Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65, 0x6c, 0x6c, 0x6f]) - expect(detectEncoding(buf)).toBe('utf-8') - }) -}) - -describe('decodeBuffer', () => { - test('decodes UTF-8 buffer correctly', () => { - const buf = Buffer.from('Hello, 世界', 'utf-8') - expect(decodeBuffer(buf, 'utf-8')).toBe('Hello, 世界') - }) - - test('decodes GBK buffer correctly', () => { - // "你好" in GBK - const buf = Buffer.from([0xc4, 0xe3, 0xba, 0xc3]) - expect(decodeBuffer(buf, 'gbk')).toBe('你好') - }) - - test('decodes UTF-16LE buffer correctly', () => { - const buf = Buffer.from([ - 0x48, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, - ]) - expect(decodeBuffer(buf, 'utf-16le')).toBe('Hello') - }) - - test('decodes empty buffer', () => { - const buf = Buffer.alloc(0) - expect(decodeBuffer(buf, 'utf-8')).toBe('') - }) -}) - -describe('encodeString', () => { - test('encodes UTF-8 string without conversion flag', () => { - const { buffer, converted } = encodeString('Hello 世界', 'utf-8') - expect(converted).toBe(false) - expect(buffer.toString('utf-8')).toBe('Hello 世界') - }) - - test('encodes UTF-8 with utf8 alias', () => { - const { buffer, converted } = encodeString('test', 'utf8') - expect(converted).toBe(false) - expect(buffer.toString('utf-8')).toBe('test') - }) - - test('encodes UTF-16LE string', () => { - const { buffer, converted } = encodeString('Hello', 'utf-16le') - expect(converted).toBe(false) - expect(decodeBuffer(buffer, 'utf-16le')).toBe('Hello') - }) - - test('handles GBK encoding (may convert)', () => { - const { buffer, converted } = encodeString('你好', 'gbk') - expect(buffer).toBeInstanceOf(Buffer) - expect(typeof converted).toBe('boolean') - if (!converted) { - expect(decodeBuffer(buffer, 'gbk')).toBe('你好') - } - }) -}) diff --git a/src/utils/__tests__/file.test.ts b/src/utils/__tests__/file.test.ts index e711ac967..1eebbcb8a 100644 --- a/src/utils/__tests__/file.test.ts +++ b/src/utils/__tests__/file.test.ts @@ -1,19 +1,10 @@ -import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test' -import * as fs from 'fs' -import * as path from 'path' -import { logMock } from '../../../tests/mocks/log' -import { debugMock } from '../../../tests/mocks/debug' - -mock.module('src/utils/log.ts', logMock) -mock.module('src/utils/debug.ts', debugMock) - +import { describe, expect, test } from 'bun:test' import { convertLeadingTabsToSpaces, addLineNumbers, stripLineNumberPrefix, pathsEqual, normalizePathForComparison, - writeTextContent, } from '../file' describe('convertLeadingTabsToSpaces', () => { @@ -99,50 +90,3 @@ describe('pathsEqual', () => { expect(pathsEqual('/a/b', '/a/c')).toBe(false) }) }) - -describe('writeTextContent with multi-encoding', () => { - let tmpDir: string - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join('/tmp', 'writeTextContent-test-')) - }) - - afterEach(() => { - fs.rmSync(tmpDir, { recursive: true, force: true }) - }) - - test('writes UTF-8 content correctly', () => { - const filePath = path.join(tmpDir, 'utf8.txt') - writeTextContent(filePath, 'Hello 世界', 'utf-8', 'LF') - const content = fs.readFileSync(filePath, 'utf-8') - expect(content).toBe('Hello 世界') - }) - - test('writes UTF-16LE content correctly', () => { - const filePath = path.join(tmpDir, 'utf16le.txt') - writeTextContent(filePath, 'Hello', 'utf-16le', 'LF') - const buf = fs.readFileSync(filePath) - // Should start with BOM (0xFF 0xFE) followed by UTF-16LE data - // Note: Bun's Buffer.from('Hello', 'utf-16le') doesn't add BOM - const text = buf.toString('utf-16le') - expect(text).toBe('Hello') - }) - - test('GBK write falls back to UTF-8', () => { - const filePath = path.join(tmpDir, 'gbk.txt') - writeTextContent(filePath, '测试写入', 'gbk', 'LF') - const content = fs.readFileSync(filePath, 'utf-8') - // Content should be readable (either GBK or UTF-8 fallback) - expect(content.length).toBeGreaterThan(0) - }) - - test('CRLF line endings with GBK encoding', () => { - const filePath = path.join(tmpDir, 'gbk-crlf.txt') - writeTextContent(filePath, 'line1\nline2', 'gbk', 'CRLF') - const buf = fs.readFileSync(filePath) - const content = buf.toString('utf-8') - // Should have CRLF line endings - expect(content).toContain('\r\n') - expect(content).not.toContain('\n\r') - }) -}) diff --git a/src/utils/__tests__/fileRead.test.ts b/src/utils/__tests__/fileRead.test.ts deleted file mode 100644 index e5d9c7fcb..000000000 --- a/src/utils/__tests__/fileRead.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test' -import * as fs from 'fs' -import * as path from 'path' -import { logMock } from '../../../tests/mocks/log' -import { debugMock } from '../../../tests/mocks/debug' - -mock.module('src/utils/log.ts', logMock) -mock.module('src/utils/debug.ts', debugMock) - -import { - readFileSyncWithMetadata, - detectEncodingForResolvedPath, -} from '../fileRead' - -describe('readFileSyncWithMetadata', () => { - let tmpDir: string - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join('/tmp', 'fileRead-test-')) - }) - - afterEach(() => { - fs.rmSync(tmpDir, { recursive: true, force: true }) - }) - - test('reads UTF-8 file correctly', () => { - const filePath = path.join(tmpDir, 'utf8.txt') - fs.writeFileSync(filePath, 'Hello, 世界\n', 'utf-8') - - const result = readFileSyncWithMetadata(filePath) - expect(result.encoding).toBe('utf-8') - expect(result.content).toBe('Hello, 世界\n') - expect(result.lineEndings).toBe('LF') - }) - - test('reads GBK encoded file correctly', () => { - const filePath = path.join(tmpDir, 'gbk.txt') - // "你好世界" in GBK encoding - const gbkBytes = Buffer.from([ - 0xc4, 0xe3, 0xba, 0xc3, 0xca, 0xc0, 0xbd, 0xe7, - ]) - fs.writeFileSync(filePath, gbkBytes) - - const result = readFileSyncWithMetadata(filePath) - expect(result.encoding).toBe('gbk') - expect(result.content).toBe('你好世界') - }) - - test('reads empty file with utf8 encoding', () => { - const filePath = path.join(tmpDir, 'empty.txt') - fs.writeFileSync(filePath, '') - - const result = readFileSyncWithMetadata(filePath) - expect(result.encoding).toBe('utf8') - expect(result.content).toBe('') - }) - - test('reads UTF-16LE BOM file correctly', () => { - const filePath = path.join(tmpDir, 'utf16le.txt') - // BOM + "Hello" in UTF-16LE - const bom = Buffer.from([0xff, 0xfe]) - const content = Buffer.from('Hello', 'utf-16le') - fs.writeFileSync(filePath, Buffer.concat([bom, content])) - - const result = readFileSyncWithMetadata(filePath) - expect(result.encoding).toBe('utf-16le') - expect(result.content).toBe('Hello') - }) - - test('normalizes CRLF to LF', () => { - const filePath = path.join(tmpDir, 'crlf.txt') - fs.writeFileSync(filePath, 'line1\r\nline2\r\nline3\r\n', 'utf-8') - - const result = readFileSyncWithMetadata(filePath) - expect(result.content).toBe('line1\nline2\nline3\n') - expect(result.lineEndings).toBe('CRLF') - }) -}) - -describe('detectEncodingForResolvedPath', () => { - let tmpDir: string - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join('/tmp', 'fileRead-detect-test-')) - }) - - afterEach(() => { - fs.rmSync(tmpDir, { recursive: true, force: true }) - }) - - test('returns utf8 for empty file', () => { - const filePath = path.join(tmpDir, 'empty.txt') - fs.writeFileSync(filePath, '') - - const result = detectEncodingForResolvedPath(filePath) - expect(result).toBe('utf8') - }) - - test('detects GBK encoding from file', () => { - const filePath = path.join(tmpDir, 'gbk.txt') - const gbkBytes = Buffer.from([0xc4, 0xe3, 0xba, 0xc3]) - fs.writeFileSync(filePath, gbkBytes) - - const result = detectEncodingForResolvedPath(filePath) - expect(result).toBe('gbk') - }) -}) diff --git a/src/utils/__tests__/readFileInRange.test.ts b/src/utils/__tests__/readFileInRange.test.ts deleted file mode 100644 index 7307e7350..000000000 --- a/src/utils/__tests__/readFileInRange.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test } from 'bun:test' -import * as fs from 'fs' -import * as path from 'path' -import { readFileInRange } from '../readFileInRange' - -describe('readFileInRange', () => { - let tmpDir: string - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join('/tmp', 'readFileInRange-test-')) - }) - - afterEach(() => { - fs.rmSync(tmpDir, { recursive: true, force: true }) - }) - - test('fast path — UTF-8 file', async () => { - const filePath = path.join(tmpDir, 'utf8.txt') - fs.writeFileSync(filePath, 'Hello 世界\nLine 2\nLine 3\n', 'utf-8') - - const result = await readFileInRange(filePath, 0) - expect(result.content).toBe('Hello 世界\nLine 2\nLine 3\n') - expect(result.lineCount).toBe(4) - expect(result.totalLines).toBe(4) - }) - - test('fast path — GBK file', async () => { - const filePath = path.join(tmpDir, 'gbk.txt') - // "你好世界" in GBK + newline - const gbkBytes = Buffer.from([ - 0xc4, 0xe3, 0xba, 0xc3, 0xca, 0xc0, 0xbd, 0xe7, 0x0a, - ]) - fs.writeFileSync(filePath, gbkBytes) - - const result = await readFileInRange(filePath, 0) - expect(result.content).toBe('你好世界\n') - expect(result.totalBytes).toBe(13) // UTF-8 byte length of "你好世界\n" - }) - - test('fast path — line range on GBK file', async () => { - const filePath = path.join(tmpDir, 'gbk-lines.txt') - // Three lines in GBK: "第一行\n第二行\n第三行\n" - const line1 = Buffer.from([0xb5, 0xda, 0xd2, 0xbb, 0xd0, 0xd0]) // 第一行 - const line2 = Buffer.from([0xb5, 0xda, 0xb6, 0xfe, 0xd0, 0xd0]) // 第二行 - const line3 = Buffer.from([0xb5, 0xda, 0xc8, 0xfd, 0xd0, 0xd0]) // 第三行 - const content = Buffer.concat([ - line1, - Buffer.from([0x0a]), - line2, - Buffer.from([0x0a]), - line3, - Buffer.from([0x0a]), - ]) - fs.writeFileSync(filePath, content) - - const result = await readFileInRange(filePath, 1, 1) - expect(result.content).toBe('第二行') - }) - - test('BOM stripping', async () => { - const filePath = path.join(tmpDir, 'bom.txt') - const bom = Buffer.from([0xef, 0xbb, 0xbf]) - fs.writeFileSync(filePath, Buffer.concat([bom, Buffer.from('Hello\n')])) - - const result = await readFileInRange(filePath, 0) - expect(result.content).toBe('Hello\n') - }) - - test('empty file', async () => { - const filePath = path.join(tmpDir, 'empty.txt') - fs.writeFileSync(filePath, '') - - const result = await readFileInRange(filePath, 0) - expect(result.content).toBe('') - expect(result.totalLines).toBe(1) - expect(result.totalBytes).toBe(0) - }) - - test('fast path — offset and maxLines', async () => { - const filePath = path.join(tmpDir, 'lines.txt') - fs.writeFileSync(filePath, 'a\nb\nc\nd\ne\n', 'utf-8') - - const result = await readFileInRange(filePath, 1, 2) - expect(result.content).toBe('b\nc') - expect(result.lineCount).toBe(2) - }) -}) diff --git a/src/utils/encoding.ts b/src/utils/encoding.ts deleted file mode 100644 index 3a4b15216..000000000 --- a/src/utils/encoding.ts +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Encoding detection and conversion utilities for file I/O. - * - * Provides three-layer encoding detection (BOM → UTF-8 fatal → GBK fallback) - * and Buffer/string conversion functions. Zero external dependencies — uses only - * TextDecoder/TextEncoder APIs available in Bun/Node.js. - */ - -/** Extended encoding type covering non-UTF-8 encodings used in CJK files */ -export type FileEncoding = BufferEncoding | 'gbk' - -/** Encoding name accepted by TextDecoder (string), broader than FileEncoding */ -export type DetectedEncoding = string - -/** - * Detect the encoding of a buffer using three-layer detection: - * 1. BOM (Byte Order Mark) detection - * 2. UTF-8 fatal validation - * 3. GBK fallback (most common non-UTF-8 CJK encoding) - */ -export function detectEncoding(buffer: Buffer): FileEncoding { - // Layer 1: BOM detection - if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { - return 'utf-16le' - } - if ( - buffer.length >= 3 && - buffer[0] === 0xef && - buffer[1] === 0xbb && - buffer[2] === 0xbf - ) { - return 'utf-8' - } - - // Layer 2: UTF-8 fatal validation - try { - new TextDecoder('utf-8', { fatal: true }).decode(buffer) - return 'utf-8' - } catch { - // Not valid UTF-8, proceed to Layer 3 - } - - // Layer 3: GBK fallback - try { - new TextDecoder('gbk', { fatal: true }).decode(buffer) - return 'gbk' - } catch { - // Not valid GBK, fall back to latin1 (single-byte, always succeeds) - return 'latin1' - } -} - -/** - * Decode a buffer using the specified encoding. - * Unified decoding entry point for all file read paths. - */ -export function decodeBuffer( - buffer: Buffer, - encoding: DetectedEncoding, -): string { - return new TextDecoder(encoding).decode(buffer) -} - -/** - * Encode a string to a Buffer using the specified encoding. - * For non-standard encodings, falls back to UTF-8 if the runtime - * doesn't support the encoding in Buffer.from. - * - * @returns buffer - the encoded bytes, converted - true if encoding was - * fallbacked to UTF-8 (caller should warn the user) - */ -export function encodeString( - content: string, - encoding: DetectedEncoding, -): { buffer: Buffer; converted: boolean } { - if (encoding === 'utf-8' || encoding === 'utf8') { - return { buffer: Buffer.from(content, 'utf-8'), converted: false } - } - if (encoding === 'utf-16le') { - return { buffer: Buffer.from(content, 'utf-16le'), converted: false } - } - - // Other encodings (e.g. gbk): try Buffer.from, fall back to UTF-8 - try { - const buf = Buffer.from(content, encoding as BufferEncoding) - return { buffer: buf, converted: false } - } catch { - return { buffer: Buffer.from(content, 'utf-8'), converted: true } - } -} diff --git a/src/utils/file.ts b/src/utils/file.ts index 4fbeea49a..51cf85cf9 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -22,7 +22,6 @@ import { detectLineEndingsForString, type LineEndingType, } from './fileRead.js' -import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js' import { fileReadCache } from './fileReadCache.js' import { getFsImplementation, safeResolvePath } from './fsOperations.js' import { logError } from './log.js' @@ -85,7 +84,7 @@ export async function getFileModificationTimeAsync( export function writeTextContent( filePath: string, content: string, - encoding: FileEncoding, + encoding: BufferEncoding, endings: LineEndingType, ): void { let toWrite = content @@ -95,38 +94,10 @@ export function writeTextContent( toWrite = content.replaceAll('\r\n', '\n').split('\n').join('\r\n') } - // Check if encoding is directly supported by Node.js fs - const BUFFER_ENCODINGS = new Set([ - 'utf8', - 'utf-8', - 'utf16le', - 'ucs2', - 'ucs-2', - 'ascii', - 'latin1', - 'binary', - 'base64', - 'hex', - ]) - - if (BUFFER_ENCODINGS.has(encoding)) { - writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, { - encoding: encoding as BufferEncoding, - }) - } else { - // Non-BufferEncoding (e.g. gbk): use encodeString to get Buffer - const { buffer, converted } = encodeString(toWrite, encoding) - writeFileSyncAndFlush_DEPRECATED(filePath, buffer, { buffer }) - if (converted) { - logForDebugging( - `writeTextContent: encoding '${encoding}' unsupported for write, fell back to UTF-8 for ${filePath}`, - { level: 'warn' }, - ) - } - } + writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, { encoding }) } -export function detectFileEncoding(filePath: string): FileEncoding { +export function detectFileEncoding(filePath: string): BufferEncoding { try { const fs = getFsImplementation() const { resolvedPath } = safeResolvePath(fs, filePath) @@ -148,14 +119,14 @@ export function detectFileEncoding(filePath: string): FileEncoding { export function detectLineEndings( filePath: string, - encoding: FileEncoding = 'utf8', + encoding: BufferEncoding = 'utf8', ): LineEndingType { try { const fs = getFsImplementation() const { resolvedPath } = safeResolvePath(fs, filePath) const { buffer, bytesRead } = fs.readSync(resolvedPath, { length: 4096 }) - const content = decodeBuffer(buffer.subarray(0, bytesRead), encoding) + const content = buffer.toString(encoding, 0, bytesRead) return detectLineEndingsForString(content) } catch (error) { logError(error) @@ -390,10 +361,8 @@ export function readFileSyncCached(filePath: string): string { */ export function writeFileSyncAndFlush_DEPRECATED( filePath: string, - content: string | Buffer, - options: { encoding?: BufferEncoding; mode?: number; buffer?: Buffer } = { - encoding: 'utf-8', - }, + content: string, + options: { encoding: BufferEncoding; mode?: number } = { encoding: 'utf-8' }, ): void { const fs = getFsImplementation() @@ -434,30 +403,26 @@ export function writeFileSyncAndFlush_DEPRECATED( } } - // Determine write mode before try/catch so both paths can use it - const isBufferWrite = Buffer.isBuffer(content) || options.buffer !== undefined - const writeData = options.buffer ?? content - try { logForDebugging(`Writing to temp file: ${tempPath}`) // Write to temp file with flush and mode (if specified for new file) const writeOptions: { - encoding?: BufferEncoding + encoding: BufferEncoding flush: boolean mode?: number } = { + encoding: options.encoding, flush: true, - ...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }), } // Only set mode in writeFileSync for new files to ensure atomic permission setting if (!targetExists && options.mode !== undefined) { writeOptions.mode = options.mode } - fsWriteFileSync(tempPath, writeData, writeOptions) + fsWriteFileSync(tempPath, content, writeOptions) logForDebugging( - `Temp file written successfully, size: ${typeof writeData === 'string' ? writeData.length : writeData.byteLength} bytes`, + `Temp file written successfully, size: ${content.length} bytes`, ) // For existing files or if mode was not set atomically, apply permissions @@ -489,19 +454,19 @@ export function writeFileSyncAndFlush_DEPRECATED( logForDebugging(`Falling back to non-atomic write for ${targetPath}`) try { const fallbackOptions: { - encoding?: BufferEncoding + encoding: BufferEncoding flush: boolean mode?: number } = { + encoding: options.encoding, flush: true, - ...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }), } // Only set mode for new files if (!targetExists && options.mode !== undefined) { fallbackOptions.mode = options.mode } - fsWriteFileSync(targetPath, writeData, fallbackOptions) + fsWriteFileSync(targetPath, content, fallbackOptions) logForDebugging( `File ${targetPath} written successfully with non-atomic fallback`, ) diff --git a/src/utils/fileRead.ts b/src/utils/fileRead.ts index 3f480f79d..4400b9bdc 100644 --- a/src/utils/fileRead.ts +++ b/src/utils/fileRead.ts @@ -13,24 +13,39 @@ */ import { logForDebugging } from './debug.js' -import { type FileEncoding, decodeBuffer, detectEncoding } from './encoding.js' import { getFsImplementation, safeResolvePath } from './fsOperations.js' export type LineEndingType = 'CRLF' | 'LF' export function detectEncodingForResolvedPath( resolvedPath: string, -): FileEncoding { +): BufferEncoding { const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, { length: 4096, }) - // Empty files default to utf8 — nothing to detect + // Empty files should default to utf8, not ascii + // This fixes a bug where writing emojis/CJK to empty files caused corruption if (bytesRead === 0) { return 'utf8' } - return detectEncoding(buffer.subarray(0, bytesRead)) + if (bytesRead >= 2) { + if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf16le' + } + + if ( + bytesRead >= 3 && + buffer[0] === 0xef && + buffer[1] === 0xbb && + buffer[2] === 0xbf + ) { + return 'utf8' + } + + // For non-empty files, default to utf8 since it's a superset of ascii + // and handles all Unicode characters properly + return 'utf8' } export function detectLineEndingsForString(content: string): LineEndingType { @@ -59,7 +74,7 @@ export function detectLineEndingsForString(content: string): LineEndingType { */ export function readFileSyncWithMetadata(filePath: string): { content: string - encoding: FileEncoding + encoding: BufferEncoding lineEndings: LineEndingType } { const fs = getFsImplementation() @@ -70,10 +85,10 @@ export function readFileSyncWithMetadata(filePath: string): { } const encoding = detectEncodingForResolvedPath(resolvedPath) - // Read raw Buffer first — readFileSync encoding option only accepts - // BufferEncoding, not gbk etc. - const rawBuffer = fs.readFileBytesSync(resolvedPath) - const raw = decodeBuffer(rawBuffer, encoding) + const raw = fs.readFileSync(resolvedPath, { encoding }) + // Detect line endings from the raw head before CRLF normalization erases + // the distinction. 4096 code units is ≥ detectLineEndings's 4096-byte + // readSync sample (line endings are ASCII, so the unit mismatch is moot). const lineEndings = detectLineEndingsForString(raw.slice(0, 4096)) return { content: raw.replaceAll('\r\n', '\n'), diff --git a/src/utils/fileReadCache.ts b/src/utils/fileReadCache.ts index e7d18634c..4e5dd22c1 100644 --- a/src/utils/fileReadCache.ts +++ b/src/utils/fileReadCache.ts @@ -1,10 +1,9 @@ import { detectFileEncoding } from './file.js' -import { type FileEncoding, decodeBuffer } from './encoding.js' import { getFsImplementation } from './fsOperations.js' type CachedFileData = { content: string - encoding: FileEncoding + encoding: BufferEncoding mtime: number } @@ -20,7 +19,7 @@ class FileReadCache { * Reads a file with caching. Returns both content and encoding. * Cache key includes file path and modification time for automatic invalidation. */ - readFile(filePath: string): { content: string; encoding: FileEncoding } { + readFile(filePath: string): { content: string; encoding: BufferEncoding } { const fs = getFsImplementation() // Get file stats for cache invalidation @@ -46,8 +45,9 @@ class FileReadCache { // Cache miss or stale data - read the file const encoding = detectFileEncoding(filePath) - const rawBuffer = fs.readFileBytesSync(filePath) - const content = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n') + const content = fs + .readFileSync(filePath, { encoding }) + .replaceAll('\r\n', '\n') // Update cache this.cache.set(cacheKey, { diff --git a/src/utils/readFileInRange.ts b/src/utils/readFileInRange.ts index 7575f47b1..18086135c 100644 --- a/src/utils/readFileInRange.ts +++ b/src/utils/readFileInRange.ts @@ -26,8 +26,7 @@ // On error (including maxBytes exceeded), stream.destroy(err) emits // 'error' → reject (passed directly to .once('error')). // -// Both paths auto-detect encoding via encoding.ts (BOM → UTF-8 fatal → fallback chain), -// decode with TextDecoder, and strip BOM and \r (CRLF → LF). +// Both paths strip UTF-8 BOM and \r (CRLF → LF). // // mtime comes from fstat/stat on the already-open fd — no extra open(). // @@ -40,7 +39,6 @@ import { createReadStream, fstat } from 'fs' import { stat as fsStat, readFile } from 'fs/promises' -import { detectEncoding, decodeBuffer } from './encoding.js' import { formatFileSize } from './format.js' const FAST_PATH_MAX_SIZE = 10 * 1024 * 1024 // 10 MB @@ -117,9 +115,7 @@ export async function readFileInRange( ) } - const rawBuffer = await readFile(filePath, { signal }) - const encoding = detectEncoding(rawBuffer) - const text = decodeBuffer(rawBuffer, encoding) + const text = await readFile(filePath, { encoding: 'utf8', signal }) return readFileInRangeFast( text, stats.mtimeMs, @@ -231,12 +227,6 @@ type StreamState = { isFirstChunk: boolean resolveMtime: (ms: number) => void mtimeReady: Promise - /** Encoding detection state: null = not yet detected, string = detected */ - encoding: string | null - /** TextDecoder instance: created after detection, used for streaming decode */ - decoder: TextDecoder | null - /** Detection phase buffer: collects raw bytes until 4KB or stream end */ - detectionBuffer: number[] } function streamOnOpen(this: StreamState, fd: number): void { @@ -245,71 +235,15 @@ function streamOnOpen(this: StreamState, fd: number): void { }) } -function processTextChunk(state: StreamState, text: string): void { - // BOM stripping (first chunk only) - if (state.isFirstChunk) { - state.isFirstChunk = false - if (text.charCodeAt(0) === 0xfeff) { - text = text.slice(1) +function streamOnData(this: StreamState, chunk: string): void { + if (this.isFirstChunk) { + this.isFirstChunk = false + if (chunk.charCodeAt(0) === 0xfeff) { + chunk = chunk.slice(1) } } - const data = state.partial.length > 0 ? state.partial + text : text - state.partial = '' - - let startPos = 0 - let newlinePos: number - while ((newlinePos = data.indexOf('\n', startPos)) !== -1) { - if ( - state.currentLineIndex >= state.offset && - state.currentLineIndex < state.endLine - ) { - let line = data.slice(startPos, newlinePos) - if (line.endsWith('\r')) { - line = line.slice(0, -1) - } - if (state.truncateOnByteLimit && state.maxBytes !== undefined) { - const sep = state.selectedLines.length > 0 ? 1 : 0 - const nextBytes = state.selectedBytes + sep + Buffer.byteLength(line) - if (nextBytes > state.maxBytes) { - state.truncatedByBytes = true - state.endLine = state.currentLineIndex - } else { - state.selectedBytes = nextBytes - state.selectedLines.push(line) - } - } else { - state.selectedLines.push(line) - } - } - state.currentLineIndex++ - startPos = newlinePos + 1 - } - - if (startPos < data.length) { - if ( - state.currentLineIndex >= state.offset && - state.currentLineIndex < state.endLine - ) { - const fragment = data.slice(startPos) - if (state.truncateOnByteLimit && state.maxBytes !== undefined) { - const sep = state.selectedLines.length > 0 ? 1 : 0 - const fragBytes = - state.selectedBytes + sep + Buffer.byteLength(fragment) - if (fragBytes > state.maxBytes) { - state.truncatedByBytes = true - state.endLine = state.currentLineIndex - return - } - } - state.partial = fragment - } - } -} - -function streamOnData(this: StreamState, chunk: Buffer): void { - this.totalBytesRead += chunk.length - + this.totalBytesRead += Buffer.byteLength(chunk) if ( !this.truncateOnByteLimit && this.maxBytes !== undefined && @@ -321,47 +255,69 @@ function streamOnData(this: StreamState, chunk: Buffer): void { return } - // Phase 1: Encoding detection - if (this.encoding === null) { - for (let i = 0; i < chunk.length; i++) { - this.detectionBuffer.push(chunk[i]) - } + const data = this.partial.length > 0 ? this.partial + chunk : chunk + this.partial = '' - // Collected at least 4KB, perform encoding detection - if (this.detectionBuffer.length >= 4096) { - this.encoding = detectEncoding(Buffer.from(this.detectionBuffer)) - this.decoder = new TextDecoder(this.encoding, { - stream: true, - } as TextDecoderOptions) - - // Decode the detection buffer and feed to line scanning - const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer)) - this.detectionBuffer = [] - processTextChunk(this, decoded) + let startPos = 0 + let newlinePos: number + while ((newlinePos = data.indexOf('\n', startPos)) !== -1) { + if ( + this.currentLineIndex >= this.offset && + this.currentLineIndex < this.endLine + ) { + let line = data.slice(startPos, newlinePos) + if (line.endsWith('\r')) { + line = line.slice(0, -1) + } + if (this.truncateOnByteLimit && this.maxBytes !== undefined) { + const sep = this.selectedLines.length > 0 ? 1 : 0 + const nextBytes = this.selectedBytes + sep + Buffer.byteLength(line) + if (nextBytes > this.maxBytes) { + // Cap hit — collapse the selection range so nothing more is + // accumulated. Stream continues (to count totalLines). + this.truncatedByBytes = true + this.endLine = this.currentLineIndex + } else { + this.selectedBytes = nextBytes + this.selectedLines.push(line) + } + } else { + this.selectedLines.push(line) + } } - return + this.currentLineIndex++ + startPos = newlinePos + 1 } - // Phase 2: Decoding - const decoded = this.decoder!.decode(chunk, { - stream: true, - } as unknown as TextDecodeOptions) - processTextChunk(this, decoded) + // Only keep the trailing fragment when inside the selected range. + // Outside the range we just count newlines — discarding prevents + // unbounded memory growth on huge single-line files. + if (startPos < data.length) { + if ( + this.currentLineIndex >= this.offset && + this.currentLineIndex < this.endLine + ) { + const fragment = data.slice(startPos) + // In truncate mode, `partial` can grow unboundedly if the selected + // range contains a huge single line (no newline across many chunks). + // Once the fragment alone would overflow the remaining budget, we know + // the completed line can never fit — set truncated, collapse the + // selection range, and discard the fragment to stop accumulation. + if (this.truncateOnByteLimit && this.maxBytes !== undefined) { + const sep = this.selectedLines.length > 0 ? 1 : 0 + const fragBytes = this.selectedBytes + sep + Buffer.byteLength(fragment) + if (fragBytes > this.maxBytes) { + this.truncatedByBytes = true + this.endLine = this.currentLineIndex + return + } + } + this.partial = fragment + } + } } function streamOnEnd(this: StreamState): void { - // If stream ended before detection completed (< 4KB file), detect now - if (this.encoding === null) { - this.encoding = detectEncoding(Buffer.from(this.detectionBuffer)) - this.decoder = new TextDecoder(this.encoding, { - stream: true, - } as TextDecoderOptions) - const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer)) - this.detectionBuffer = [] - processTextChunk(this, decoded) - } - - // Handle final fragment let line = this.partial if (line.endsWith('\r')) { line = line.slice(0, -1) @@ -410,6 +366,7 @@ function readFileInRangeStreaming( return new Promise((resolve, reject) => { const state: StreamState = { stream: createReadStream(filePath, { + encoding: 'utf8', highWaterMark: 512 * 1024, ...(signal ? { signal } : undefined), }), @@ -427,9 +384,6 @@ function readFileInRangeStreaming( isFirstChunk: true, resolveMtime: () => {}, mtimeReady: null as unknown as Promise, - encoding: null, - decoder: null, - detectionBuffer: [], } state.mtimeReady = new Promise(r => { state.resolveMtime = r