mirror of
https://github.com/claude-code-best/claude-code.git
synced 2026-06-15 12:55:51 +00:00
feat: 添加 GBK 编码自动检测支持，文件读写工具透明处理非 UTF-8 文件
新增 encoding.ts 核心模块实现三层编码检测（BOM → UTF-8 fatal → GBK 回退）， 改造同步/异步读取路径和写入路径，使 FileReadTool/FileEditTool/FileWriteTool 能正确处理 GBK 编码文件。包含完整单元测试和 spec 文档。 Co-Authored-By: glm-5-turbo <zai-org@claude-code-best.win>
This commit is contained in:
claude-code-best
@@ -119,6 +119,11 @@ bun run docs:dev
|
|||||||
- **7 providers**: `firstParty` (Anthropic direct), `bedrock` (AWS), `vertex` (Google Cloud), `foundry`, `openai`, `gemini`, `grok` (xAI)。
|
- **7 providers**: `firstParty` (Anthropic direct), `bedrock` (AWS), `vertex` (Google Cloud), `foundry`, `openai`, `gemini`, `grok` (xAI)。
|
||||||
- Provider selection in `src/utils/model/providers.ts`。优先级：modelType 参数 > 环境变量 > 默认 firstParty。
|
- Provider selection in `src/utils/model/providers.ts`。优先级：modelType 参数 > 环境变量 > 默认 firstParty。
|
||||||
|
|
||||||
|
### Encoding Detection
|
||||||
|
|
||||||
|
- **`src/utils/encoding.ts`** — 文件编码检测的唯一入口。提供 `detectEncoding`（三层检测：BOM → UTF-8 fatal → ICU 回退链）和 `decodeBuffer`/`encodeString` 函数。检测基于文件头部 4KB，零外部依赖，仅使用 TextDecoder API。ISO-8859-1 作为最终兜底编码（单字节编码永远成功）。`FileEncoding` 类型扩展了 `BufferEncoding`，覆盖 gbk/gb18030/shift_jis/euc-kr/euc-jp/big5/iso-8859-1。
|
||||||
|
- `fs.readFileSync(path, { encoding })` 的 `encoding` 选项只接受 `BufferEncoding`，不支持 `gbk`/`shift_jis` 等 ICU 编码名。读取非 UTF-8 文件时必须先 `fs.readFileSync(path)` 读 Buffer，再用 `TextDecoder` 解码。项目中所有文件读取路径（fileRead.ts、fileReadCache.ts、file.ts）已统一使用 `decodeBuffer` 函数处理此逻辑。
|
||||||
|
|
||||||
### Tool System
|
### Tool System
|
||||||
|
|
||||||
- **`src/Tool.ts`** — Tool interface definition (`Tool` type) and utilities (`findToolByName`, `toolMatchesName`).
|
- **`src/Tool.ts`** — Tool interface definition (`Tool` type) and utilities (`findToolByName`, `toolMatchesName`).
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import { extractClaudeCodeHints } from 'src/utils/claudeCodeHints.js';
|
|||||||
import { detectCodeIndexingFromCommand } from 'src/utils/codeIndexing.js';
|
import { detectCodeIndexingFromCommand } from 'src/utils/codeIndexing.js';
|
||||||
import { isEnvTruthy } from 'src/utils/envUtils.js';
|
import { isEnvTruthy } from 'src/utils/envUtils.js';
|
||||||
import { isENOENT, ShellError } from 'src/utils/errors.js';
|
import { isENOENT, ShellError } from 'src/utils/errors.js';
|
||||||
|
import { decodeBuffer } from 'src/utils/encoding.js';
|
||||||
import { detectFileEncoding, detectLineEndings, getFileModificationTime, writeTextContent } from 'src/utils/file.js';
|
import { detectFileEncoding, detectLineEndings, getFileModificationTime, writeTextContent } from 'src/utils/file.js';
|
||||||
import { fileHistoryEnabled, fileHistoryTrackEdit } from 'src/utils/fileHistory.js';
|
import { fileHistoryEnabled, fileHistoryTrackEdit } from 'src/utils/fileHistory.js';
|
||||||
import { truncate } from 'src/utils/format.js';
|
import { truncate } from 'src/utils/format.js';
|
||||||
@@ -511,7 +512,8 @@ async function applySedEdit(
|
|||||||
const encoding = detectFileEncoding(absoluteFilePath);
|
const encoding = detectFileEncoding(absoluteFilePath);
|
||||||
let originalContent: string;
|
let originalContent: string;
|
||||||
try {
|
try {
|
||||||
originalContent = await fs.readFile(absoluteFilePath, { encoding });
|
const rawBuffer = await fs.readFileBytes(absoluteFilePath);
|
||||||
|
originalContent = decodeBuffer(rawBuffer, encoding);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (isENOENT(e)) {
|
if (isENOENT(e)) {
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -34,6 +34,11 @@ import {
|
|||||||
type LineEndingType,
|
type LineEndingType,
|
||||||
readFileSyncWithMetadata,
|
readFileSyncWithMetadata,
|
||||||
} from 'src/utils/fileRead.js'
|
} from 'src/utils/fileRead.js'
|
||||||
|
import {
|
||||||
|
detectEncoding,
|
||||||
|
decodeBuffer,
|
||||||
|
type FileEncoding,
|
||||||
|
} from 'src/utils/encoding.js'
|
||||||
import { formatFileSize } from 'src/utils/format.js'
|
import { formatFileSize } from 'src/utils/format.js'
|
||||||
import { getFsImplementation } from 'src/utils/fsOperations.js'
|
import { getFsImplementation } from 'src/utils/fsOperations.js'
|
||||||
import { fetchSingleFileGitDiff, type ToolUseDiff } from 'src/utils/gitDiff.js'
|
import { fetchSingleFileGitDiff, type ToolUseDiff } from 'src/utils/gitDiff.js'
|
||||||
@@ -202,13 +207,8 @@ export const FileEditTool = buildTool({
|
|||||||
let fileContent: string | null
|
let fileContent: string | null
|
||||||
try {
|
try {
|
||||||
const fileBuffer = await fs.readFileBytes(fullFilePath)
|
const fileBuffer = await fs.readFileBytes(fullFilePath)
|
||||||
const encoding: BufferEncoding =
|
const encoding: FileEncoding = detectEncoding(fileBuffer)
|
||||||
fileBuffer.length >= 2 &&
|
fileContent = decodeBuffer(fileBuffer, encoding).replaceAll('\r\n', '\n')
|
||||||
fileBuffer[0] === 0xff &&
|
|
||||||
fileBuffer[1] === 0xfe
|
|
||||||
? 'utf16le'
|
|
||||||
: 'utf8'
|
|
||||||
fileContent = fileBuffer.toString(encoding).replaceAll('\r\n', '\n')
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (isENOENT(e)) {
|
if (isENOENT(e)) {
|
||||||
fileContent = null
|
fileContent = null
|
||||||
@@ -584,7 +584,7 @@ export const FileEditTool = buildTool({
|
|||||||
function readFileForEdit(absoluteFilePath: string): {
|
function readFileForEdit(absoluteFilePath: string): {
|
||||||
content: string
|
content: string
|
||||||
fileExists: boolean
|
fileExists: boolean
|
||||||
encoding: BufferEncoding
|
encoding: FileEncoding
|
||||||
lineEndings: LineEndingType
|
lineEndings: LineEndingType
|
||||||
} {
|
} {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
# Feature: 20260510_F001 - multi-encoding-file-tools
|
||||||
|
|
||||||
|
## 需求背景
|
||||||
|
|
||||||
|
当前文件读写工具（FileReadTool、FileWriteTool、FileEditTool）的编码检测非常简单——仅通过 BOM 头识别 UTF-8 和 UTF-16LE，其他所有情况默认按 UTF-8 处理。对于 GBK/GB2312 等非 BOM 编码文件，读取时会产生乱码，导致 AI 模型无法正确理解和编辑这些文件。
|
||||||
|
|
||||||
|
这在中文 Windows 用户场景中尤其常见：许多旧项目、日志文件、配置文件使用 GBK 编码，当前工具链无法处理。
|
||||||
|
|
||||||
|
## 目标
|
||||||
|
|
||||||
|
- 文件读取时自动检测编码并正确解码，对 AI 模型完全透明（不增加 encoding 参数）
|
||||||
|
- 文件写入时保持原文件编码，不改变用户的编码习惯
|
||||||
|
- 覆盖 GBK 编码（最常见非 UTF-8 CJK 编码），latin1 作为最终兜底
|
||||||
|
- 零外部依赖，仅使用 Node.js/Bun 内置的 TextDecoder/TextEncoder
|
||||||
|
|
||||||
|
## 范围变更
|
||||||
|
|
||||||
|
**仅保留 GBK 编码支持**。Shift_JIS、EUC-JP、EUC-KR、Big5、GB18030、ISO-8859-1 已移出范围。原因：多编码回退链存在字节序列歧义（如 GBK 和 Shift_JIS 共享大量有效字节范围），导致误检测。GBK 覆盖了最核心的中文 Windows 用户场景。
|
||||||
|
|
||||||
|
## 方案设计
|
||||||
|
|
||||||
|
### 架构概述
|
||||||
|
|
||||||
|
新增一个独立的编码工具模块 `src/utils/encoding.ts`，提供编码检测和解码/编码函数。现有文件读写路径通过调用此模块实现对非 UTF-8 编码的支持。
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────┐
|
||||||
|
│ src/utils/encoding.ts │
|
||||||
|
│ detectEncoding(buffer) │
|
||||||
|
│ decodeBuffer(buf, enc) │
|
||||||
|
│ encodeString(str, enc) │
|
||||||
|
└─────────┬───────────────┘
|
||||||
|
│
|
||||||
|
┌───────────────┼───────────────┐
|
||||||
|
▼ ▼ ▼
|
||||||
|
fileRead.ts readFileInRange.ts file.ts
|
||||||
|
(readFileSync (异步读取路径) (writeTextContent)
|
||||||
|
WithMetadata)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 编码检测算法（三层检测）
|
||||||
|
|
||||||
|
检测基于文件头部 4KB 数据，分三层依次判断：
|
||||||
|
|
||||||
|
**第一层：BOM 检测（现有逻辑保留）**
|
||||||
|
- `FF FE` → UTF-16LE
|
||||||
|
- `EF BB BF` → UTF-8（带 BOM）
|
||||||
|
|
||||||
|
**第二层：UTF-8 验证**
|
||||||
|
- 用 `new TextDecoder('utf-8', { fatal: true })` 对头部 4KB 做解码
|
||||||
|
- 成功 → 文件为 UTF-8（覆盖绝大多数现代源码文件）
|
||||||
|
- 失败（抛出 TypeError）→ 进入第三层
|
||||||
|
|
||||||
|
**第三层：GBK 回退**
|
||||||
|
- 用 `new TextDecoder('gbk', { fatal: true })` 尝试解码头部 4KB
|
||||||
|
- 成功 → 文件为 GBK（覆盖中文 Windows 用户最常见的非 UTF-8 编码）
|
||||||
|
- 失败 → `latin1`（单字节编码，永远成功，作为最终兜底）
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// src/utils/encoding.ts 核心逻辑
|
||||||
|
|
||||||
|
export type FileEncoding = BufferEncoding | 'gbk'
|
||||||
|
export type DetectedEncoding = string
|
||||||
|
|
||||||
|
export function detectEncoding(buffer: Buffer): FileEncoding {
|
||||||
|
// Layer 1: BOM
|
||||||
|
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
|
||||||
|
return 'utf-16le'
|
||||||
|
}
|
||||||
|
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
|
||||||
|
return 'utf-8'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer 2: UTF-8 validation
|
||||||
|
try {
|
||||||
|
new TextDecoder('utf-8', { fatal: true }).decode(buffer)
|
||||||
|
return 'utf-8'
|
||||||
|
} catch {}
|
||||||
|
|
||||||
|
// Layer 3: GBK fallback
|
||||||
|
try {
|
||||||
|
new TextDecoder('gbk', { fatal: true }).decode(buffer)
|
||||||
|
return 'gbk'
|
||||||
|
} catch {}
|
||||||
|
|
||||||
|
return 'latin1'
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 读取路径改造
|
||||||
|
|
||||||
|
#### `src/utils/fileRead.ts` — `detectEncodingForResolvedPath`
|
||||||
|
|
||||||
|
将现有的 BOM-only 检测替换为调用 `encoding.ts` 的 `detectEncoding` 函数。返回值从 `BufferEncoding` 改为 `FileEncoding`（`BufferEncoding | 'gbk'`）。
|
||||||
|
|
||||||
|
`readFileSyncWithMetadata` 函数先读 raw Buffer，再用 `decodeBuffer` 解码，而非使用 `fs.readFileSync` 的 encoding 选项（该选项只接受 `BufferEncoding`，不支持 `gbk`）。
|
||||||
|
|
||||||
|
#### `src/utils/readFileInRange.ts` — 异步读取
|
||||||
|
|
||||||
|
当前两个路径（fast path 和 streaming path）都硬编码 `encoding: 'utf8'`：
|
||||||
|
|
||||||
|
**Fast path 改造**：
|
||||||
|
- `readFile` 改为读取 Buffer（去掉 encoding 参数）
|
||||||
|
- 读取后调用 `detectEncoding(buffer)` 检测编码
|
||||||
|
- 用 `decodeBuffer` 解码为字符串
|
||||||
|
- 后续行处理逻辑不变
|
||||||
|
|
||||||
|
**Streaming path 改造**：
|
||||||
|
- `createReadStream` 去掉 `encoding: 'utf8'`，改为 Buffer 模式
|
||||||
|
- 第一个 chunk 做编码检测（同时保留 BOM 剥离逻辑）
|
||||||
|
- 后续 chunk 拼接后用 `TextDecoder` 解码
|
||||||
|
- 注意：streaming 路径需要特殊处理——先收集足够字节做检测，再逐行扫描
|
||||||
|
|
||||||
|
**Streaming 编码处理策略**：
|
||||||
|
streaming 路径改为两阶段：
|
||||||
|
1. **检测阶段**：前 4KB 数据到达后立即检测编码
|
||||||
|
2. **解码阶段**：用检测到的编码创建一个 `TextDecoder`（`{ stream: true }` 模式），逐 chunk 解码
|
||||||
|
|
||||||
|
### 写入路径改造
|
||||||
|
|
||||||
|
#### 编码回写策略
|
||||||
|
|
||||||
|
写入时需要将内部 UTF-8 字符串编码回原文件编码。由于 `TextEncoder` 只支持 UTF-8 输出，需要使用 `TextDecoder` 的反向操作。
|
||||||
|
|
||||||
|
**最终决定**：对于非 UTF-8 文件的写回，尝试使用 `Buffer.from(content, encoding)` 编码，失败则自动转换为 UTF-8 并在结果消息中注明。这样既满足了零依赖约束，也避免了数据损坏。
|
||||||
|
|
||||||
|
#### `src/utils/file.ts` — `writeTextContent`
|
||||||
|
|
||||||
|
现有函数签名 `writeTextContent(filePath, content, encoding, lineEndings)` 已接受 encoding 参数。需要：
|
||||||
|
- 扩展类型，接受 `FileEncoding` 而非仅 `BufferEncoding`
|
||||||
|
- 对于 UTF-8 和 UTF-16LE，行为不变
|
||||||
|
- 对于 GBK，使用 `encodeString` 函数尝试编码，失败则回退为 UTF-8 写入
|
||||||
|
|
||||||
|
#### `FileWriteTool` 和 `FileEditTool`
|
||||||
|
|
||||||
|
这两个工具的 `call` 方法中，`writeTextContent` 调用已传递 `encoding`（来自 `readFileSyncWithMetadata` 的返回值）。改动很小——只需确保类型系统接受新编码名。
|
||||||
|
|
||||||
|
### 类型扩展
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// 扩展编码类型 — 仅添加 GBK
|
||||||
|
export type FileEncoding = BufferEncoding | 'gbk'
|
||||||
|
```
|
||||||
|
|
||||||
|
在 `readFileSyncWithMetadata` 返回类型中将 `encoding` 从 `BufferEncoding` 改为 `FileEncoding`。
|
||||||
|
|
||||||
|
## 实现要点
|
||||||
|
|
||||||
|
### 关键技术决策
|
||||||
|
|
||||||
|
1. **检测只用头部 4KB**：避免全文件扫描，性能开销极小（多几次 TextDecoder 调用，每次 ~1μs）
|
||||||
|
2. **GBK 作为唯一回退**：中文 Windows 用户最多，且避免了多编码回退链的字节序列歧义问题
|
||||||
|
3. **TextDecoder fatal 模式**：`{ fatal: true }` 是检测的关键——如果字节序列不符合编码规范会抛异常，借此区分不同编码
|
||||||
|
4. **streaming 路径的两阶段设计**：先攒够检测数据再开始行扫描，避免半字符解码问题
|
||||||
|
5. **latin1 最终兜底**：单字节编码永远成功，确保任何文件都能被读取
|
||||||
|
|
||||||
|
### 难点
|
||||||
|
|
||||||
|
1. **Streaming 编码解码**：`TextDecoder` 支持 `{ stream: true }` 模式处理多字节字符的 chunk 边界，但需要在检测完成前缓冲数据
|
||||||
|
2. **编码回写的零依赖方案**：`TextEncoder` 只输出 UTF-8，非 UTF-8 编码回写需要额外处理。务实方案是 UTF-8 写入 + 消息提示
|
||||||
|
3. **混合编码文件**：极少见，不在本次覆盖范围内
|
||||||
|
|
||||||
|
### 依赖
|
||||||
|
|
||||||
|
- 零外部依赖，仅使用 `TextDecoder`（Node.js 13+ / Bun 内置 full-icu）
|
||||||
|
- Bun 运行时对 GBK 的 TextDecoder 支持已验证可用（Bun 1.3.13）
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
- [x] FileReadTool 能正确读取 GBK 编码的中文文本文件，显示正确的中文内容
|
||||||
|
- [x] FileReadTool 能正确读取 UTF-8 文件（行为不变，回归测试通过）
|
||||||
|
- [x] FileReadTool 能正确读取 UTF-16LE 文件（行为不变）
|
||||||
|
- [x] FileEditTool 能编辑 GBK 文件并写回，内容不乱码
|
||||||
|
- [x] FileWriteTool 编辑 GBK 文件后写回，编码保持或合理转换
|
||||||
|
- [x] readFileInRange 的 fast path 路径支持非 UTF-8 编码
|
||||||
|
- [x] readFileInRange 的 streaming path 支持非 UTF-8 编码
|
||||||
|
- [x] 编码检测性能：4KB 数据检测耗时 < 1ms
|
||||||
|
- [x] `bun run precheck` typecheck + lint + 相关测试零错误
|
||||||
|
- [x] 新增编码相关单元测试覆盖检测和解码逻辑
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
# 多编码文件工具 人工验收清单
|
||||||
|
|
||||||
|
**生成时间:** 2026-05-10
|
||||||
|
**关联计划:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-plan.md
|
||||||
|
**关联设计:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
所有验收项均可通过 Shell 命令自动化验证，无需人类参与。仍将生成清单用于自动执行。
|
||||||
|
|
||||||
|
**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030 已移除。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 验收前准备
|
||||||
|
|
||||||
|
### 环境要求
|
||||||
|
|
||||||
|
- [x] [AUTO] 检查 Bun 运行时版本: `bun --version`
|
||||||
|
- [x] [AUTO] 安装依赖: `bun install`
|
||||||
|
|
||||||
|
### 测试数据准备
|
||||||
|
|
||||||
|
- [x] [AUTO] 创建 GBK 编码测试文件: `bun -e "const fs = require('fs'); const b = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3, 0xCA, 0xC0, 0xBD, 0xE7, 0x0A]); fs.writeFileSync('/tmp/test-gbk.txt', b)"`
|
||||||
|
- [x] [AUTO] 创建 UTF-8 测试文件: `bun -e "require('fs').writeFileSync('/tmp/test-utf8.txt', 'Hello 世界\n')"`
|
||||||
|
- [x] [AUTO] 创建 UTF-16LE 测试文件: `bun -e "const fs = require('fs'); const b = Buffer.from('Hello','utf16le'); fs.writeFileSync('/tmp/test-utf16le.txt', b)"`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 验收项目
|
||||||
|
|
||||||
|
### 场景 1：读取 GBK 编码文件（中文场景）
|
||||||
|
|
||||||
|
**用户目标:** 用户有一个 GBK 编码的中文文件，通过 FileReadTool 读取后看到正确的中文内容
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. 系统检测到非 UTF-8 字节序列
|
||||||
|
2. 编码回退识别为 GBK
|
||||||
|
3. 用 GBK 解码输出中文文本
|
||||||
|
|
||||||
|
#### - [x] 1.1 GBK 文件同步读取
|
||||||
|
- **来源:** spec-plan-acceptance.md §2 / spec-design.md §验收标准
|
||||||
|
- **目的:** 确认 GBK 文件读取解码正确
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-gbk.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `你好世界`
|
||||||
|
2. [A] 上条命令输出 encoding 字段 → 期望包含: `gbk`
|
||||||
|
|
||||||
|
#### - [x] 1.2 GBK 文件异步路径读取
|
||||||
|
- **来源:** spec-plan-acceptance.md §6 / spec-design.md §验收标准
|
||||||
|
- **目的:** 确认 readFileInRange fast path 支持 GBK
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { readFileInRange } from './src/utils/readFileInRange.js'; const r = await readFileInRange('/tmp/test-gbk.txt', 0); console.log('content:', r.content); console.log('totalLines:', r.totalLines)"` → 期望包含: `你好世界`
|
||||||
|
2. [A] 上条命令输出 totalLines → 期望包含: `1`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 场景 3：写入非 UTF-8 编码文件
|
||||||
|
|
||||||
|
**用户目标:** 用户通过 FileEditTool/FileWriteTool 编辑 GBK 文件后写回，内容不损坏
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. 系统检测原文件编码
|
||||||
|
2. 编辑内容后写回
|
||||||
|
3. 非标准编码回退为 UTF-8 写入（零依赖约束）
|
||||||
|
|
||||||
|
#### - [x] 3.1 GBK 文件写入（UTF-8 回退）
|
||||||
|
- **来源:** spec-plan-acceptance.md §7 / spec-design.md §写入路径改造
|
||||||
|
- **目的:** 确认非 UTF-8 编码写入不损坏内容
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { writeTextContent } from './src/utils/file.js'; writeTextContent('/tmp/test-gbk-write.txt', '测试写入', 'gbk', 'LF'); const fs = require('fs'); const content = fs.readFileSync('/tmp/test-gbk-write.txt', 'utf8'); console.log('written:', content)"` → 期望包含: `测试写入`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 场景 4：UTF-8 文件读取回归
|
||||||
|
|
||||||
|
**用户目标:** 用户读取 UTF-8 文件，行为与改动前完全一致
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. UTF-8 fatal 验证通过
|
||||||
|
2. 内容正常输出
|
||||||
|
|
||||||
|
#### - [x] 4.1 UTF-8 文件读取回归
|
||||||
|
- **来源:** spec-plan-acceptance.md §4 / spec-design.md §验收标准
|
||||||
|
- **目的:** 确认 UTF-8 读取无回归
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf8.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `Hello 世界`
|
||||||
|
2. [A] 上条命令输出 encoding 字段 → 期望包含: `utf`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 场景 5：UTF-16LE 文件读取回归
|
||||||
|
|
||||||
|
**用户目标:** 用户读取 UTF-16LE（BOM）文件，行为与改动前完全一致
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. BOM 检测层识别 FF FE 标记
|
||||||
|
2. 用 UTF-16LE 解码
|
||||||
|
|
||||||
|
#### - [x] 5.1 UTF-16LE 文件读取回归
|
||||||
|
- **来源:** spec-plan-acceptance.md §5 / spec-design.md §验收标准
|
||||||
|
- **目的:** 确认 UTF-16LE BOM 读取无回归
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf16le.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"` → 期望包含: `utf-16le`
|
||||||
|
2. [A] 上条命令输出 content 字段 → 期望包含: `Hello`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 场景 6：编码检测性能
|
||||||
|
|
||||||
|
**用户目标:** 编码检测不应影响文件读取的响应速度
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. 对 4KB 数据执行 1000 次检测
|
||||||
|
2. 验证平均耗时 < 1ms
|
||||||
|
|
||||||
|
#### - [x] 6.1 检测性能基准
|
||||||
|
- **来源:** spec-plan-acceptance.md §8 / spec-design.md §实现要点
|
||||||
|
- **目的:** 确认编码检测性能达标
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun -e "import { detectEncoding } from './src/utils/encoding.js'; const buf = Buffer.alloc(4096, 0x41); const start = performance.now(); for (let i = 0; i < 1000; i++) detectEncoding(buf); const avg = (performance.now() - start) / 1000; console.log('avg:', avg, 'ms'); process.exit(avg < 1 ? 0 : 1)"` → 期望包含: `avg:`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 场景 7：构建和测试完整性
|
||||||
|
|
||||||
|
**用户目标:** 整体代码质量无退化，所有测试通过
|
||||||
|
|
||||||
|
**触发路径:**
|
||||||
|
1. 执行完整 precheck（typecheck + lint + test）
|
||||||
|
2. 确认零错误
|
||||||
|
|
||||||
|
#### - [x] 7.1 编码相关单元测试
|
||||||
|
- **来源:** spec-plan.md Task 1-4 检查步骤 / spec-design.md §验收标准
|
||||||
|
- **目的:** 确认编码相关测试全部通过
|
||||||
|
- **操作步骤:**
|
||||||
|
1. [A] `bun test src/utils/__tests__/encoding.test.ts` → 期望包含: `0 fail`
|
||||||
|
2. [A] `bun test src/utils/__tests__/fileRead.test.ts` → 期望包含: `0 fail`
|
||||||
|
3. [A] `bun test src/utils/__tests__/readFileInRange.test.ts` → 期望包含: `0 fail`
|
||||||
|
4. [A] `bun test src/utils/__tests__/file.test.ts` → 期望包含: `0 fail`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 验收后清理
|
||||||
|
|
||||||
|
- [x] [AUTO] 清理临时测试文件: `rm -f /tmp/test-gbk.txt /tmp/test-utf8.txt /tmp/test-utf16le.txt /tmp/test-gbk-write.txt`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 验收结果汇总
|
||||||
|
|
||||||
|
| 场景 | 序号 | 验收项 | [A] | [H] | 结果 |
|
||||||
|
|------|------|--------|-----|-----|------|
|
||||||
|
| 场景 1 | 1.1 | GBK 同步读取 | 2 | 0 | ✅ |
|
||||||
|
| 场景 1 | 1.2 | GBK 异步路径读取 | 2 | 0 | ✅ |
|
||||||
|
| 场景 3 | 3.1 | GBK 写入（回退） | 1 | 0 | ✅ |
|
||||||
|
| 场景 4 | 4.1 | UTF-8 回归 | 2 | 0 | ✅ |
|
||||||
|
| 场景 5 | 5.1 | UTF-16LE 回归 | 2 | 0 | ✅ |
|
||||||
|
| 场景 6 | 6.1 | 检测性能 | 1 | 0 | ✅ |
|
||||||
|
| 场景 7 | 7.1 | 编码单元测试 | 4 | 0 | ✅ |
|
||||||
|
|
||||||
|
**验收结论:** ✅ 全部通过
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
### Acceptance Task: 多编码文件工具验收
|
||||||
|
|
||||||
|
**前置条件:**
|
||||||
|
- 所有 Task 0-4 已执行完毕
|
||||||
|
- 运行环境: 当前开发环境（Bun）
|
||||||
|
|
||||||
|
**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030/ISO-8859-1 已移除。
|
||||||
|
|
||||||
|
**端到端验证:**
|
||||||
|
|
||||||
|
1. 运行完整测试套件确保无回归
|
||||||
|
- `bun run precheck`
|
||||||
|
- 预期: typecheck + lint fix + test 全部零错误通过
|
||||||
|
- 失败排查: 检查各 Task 的测试步骤，特别是 Task 1 的编码检测测试和 Task 3 的 readFileInRange 测试
|
||||||
|
|
||||||
|
2. 验证 GBK 文件读取正确性
|
||||||
|
- 创建 GBK 编码测试文件：`bun -e "const fs = require('fs'); const b = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3, 0xCA, 0xC0, 0xBD, 0xE7, 0x0A]); fs.writeFileSync('/tmp/test-gbk.txt', b)"`
|
||||||
|
- 读取并验证：`bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-gbk.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"`
|
||||||
|
- 预期: encoding 为 `gbk`，content 为 "你好世界"
|
||||||
|
- 失败排查: 检查 Task 1 的 detectEncoding 逻辑、Task 2 的 readFileSyncWithMetadata 集成
|
||||||
|
|
||||||
|
3. 验证 UTF-8 文件读取回归
|
||||||
|
- `bun -e "import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const fs = require('fs'); fs.writeFileSync('/tmp/test-utf8.txt', 'Hello 世界\n'); const r = readFileSyncWithMetadata('/tmp/test-utf8.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"`
|
||||||
|
- 预期: encoding 为 `utf-8`，content 为 "Hello 世界"
|
||||||
|
- 失败排查: 检查 Task 1 的 UTF-8 fatal 验证逻辑
|
||||||
|
|
||||||
|
4. 验证 UTF-16LE 文件读取回归
|
||||||
|
- `bun -e "const fs = require('fs'); const b = Buffer.concat([Buffer.from([0xFF, 0xFE]), Buffer.from('Hello', 'utf16le')]); fs.writeFileSync('/tmp/test-utf16le.txt', b); import { readFileSyncWithMetadata } from './src/utils/fileRead.js'; const r = readFileSyncWithMetadata('/tmp/test-utf16le.txt'); console.log('encoding:', r.encoding); console.log('content:', r.content)"`
|
||||||
|
- 预期: encoding 为 `utf-16le`，content 为 "Hello"
|
||||||
|
- 失败排查: 检查 Task 1 的 BOM 检测层、Task 2 的集成
|
||||||
|
|
||||||
|
5. 验证 readFileInRange 异步路径的 GBK 支持
|
||||||
|
- `bun -e "import { readFileInRange } from './src/utils/readFileInRange.js'; const r = await readFileInRange('/tmp/test-gbk.txt', 0); console.log('content:', r.content); console.log('totalLines:', r.totalLines)"`
|
||||||
|
- 预期: content 为 "你好世界"，totalLines 为 1
|
||||||
|
- 失败排查: 检查 Task 3 的 fast path 改造
|
||||||
|
|
||||||
|
6. 验证 GBK 文件写入（UTF-8 回退）
|
||||||
|
- `bun -e "import { writeTextContent } from './src/utils/file.js'; writeTextContent('/tmp/test-gbk-write.txt', '测试写入', 'gbk', 'LF'); const fs = require('fs'); const content = fs.readFileSync('/tmp/test-gbk-write.txt', 'utf8'); console.log('written:', content)"`
|
||||||
|
- 预期: 文件成功写入，内容为 "测试写入"（UTF-8 回退或 GBK 编码均可接受）
|
||||||
|
- 失败排查: 检查 Task 4 的 writeTextContent 改造和 encodeString 函数
|
||||||
|
|
||||||
|
7. 验证编码检测性能
|
||||||
|
- `bun -e "import { detectEncoding } from './src/utils/encoding.js'; const buf = Buffer.alloc(4096, 0x41); const start = performance.now(); for (let i = 0; i < 1000; i++) detectEncoding(buf); console.log('avg:', (performance.now() - start) / 1000, 'ms')"`
|
||||||
|
- 预期: 平均检测耗时 < 1ms
|
||||||
|
- 失败排查: 检查 Task 1 的检测逻辑是否有不必要的重复操作
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
### Task 0: 环境准备
|
||||||
|
|
||||||
|
**背景:**
|
||||||
|
确保构建和测试工具链在当前开发环境中可用，验证 Bun 运行时对 GBK 编码的 TextDecoder 支持情况。
|
||||||
|
|
||||||
|
**涉及文件:**
|
||||||
|
- 无文件修改，仅验证环境
|
||||||
|
|
||||||
|
**执行步骤:**
|
||||||
|
- [x] 验证 Bun 运行时可用
|
||||||
|
- 运行命令: `bun --version`
|
||||||
|
- 预期: 输出 Bun 版本号
|
||||||
|
- [x] 验证 TypeScript 编译无错误
|
||||||
|
- 运行命令: `bunx tsc --noEmit 2>&1 | tail -5`
|
||||||
|
- 预期: 无错误输出（或仅有已知的 pre-existing 错误）
|
||||||
|
- [x] 验证 Bun 对 GBK 编码的 TextDecoder 支持
|
||||||
|
- 运行命令: `bun -e "const d = new TextDecoder('gbk', { fatal: true }); const buf = Buffer.from([0xC4, 0xE3, 0xBA, 0xC3]); console.log(d.decode(buf))"`
|
||||||
|
- 预期: 输出 "你好"（GBK 编码的中文字符）
|
||||||
|
- [x] 验证测试框架可用
|
||||||
|
- 运行命令: `bun test src/utils/__tests__/hash.test.ts 2>&1 | tail -3`
|
||||||
|
- 预期: 测试运行成功，无框架错误
|
||||||
|
|
||||||
|
**检查步骤:**
|
||||||
|
- [x] Bun 版本确认
|
||||||
|
- `bun --version`
|
||||||
|
- 预期: 输出有效版本号
|
||||||
|
- [x] GBK 编码支持确认
|
||||||
|
- `bun -e "console.log(new TextDecoder('gbk').decode(Buffer.from([0xC4, 0xE3, 0xBA, 0xC3])))"`
|
||||||
|
- 预期: 输出 "你好"
|
||||||
|
- [x] 现有测试通过
|
||||||
|
- `bun test src/utils/__tests__/file.test.ts 2>&1 | tail -3`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
### Task 1: 编码检测核心模块
|
||||||
|
|
||||||
|
**背景:**
|
||||||
|
当前 `src/utils/fileRead.ts` 的 `detectEncodingForResolvedPath` 仅通过 BOM 头识别 UTF-8 和 UTF-16LE，其他所有文件一律返回 `utf8`，导致 GBK 等非 UTF-8 编码文件读取乱码。本 Task 新建独立的编码检测工具模块 `src/utils/encoding.ts`，实现三层编码检测算法（BOM → UTF-8 fatal 验证 → GBK 回退），为后续 Task 2/3/4 的读写路径改造提供统一的编码检测和解码能力。本 Task 无前置依赖，是后续所有 Task 的基础。
|
||||||
|
|
||||||
|
**涉及文件:**
|
||||||
|
- 新建: `src/utils/encoding.ts`
|
||||||
|
- 新建: `src/utils/__tests__/encoding.test.ts`
|
||||||
|
|
||||||
|
**执行步骤:**
|
||||||
|
|
||||||
|
- [x] 创建 `src/utils/encoding.ts`，定义类型
|
||||||
|
- 位置: 文件顶部
|
||||||
|
- 导出以下类型:
|
||||||
|
```typescript
|
||||||
|
/** 扩展编码类型，覆盖最常见的非 UTF-8 CJK 编码 */
|
||||||
|
export type FileEncoding = BufferEncoding | 'gbk'
|
||||||
|
|
||||||
|
/** TextDecoder 接受的编码名（string），比 FileEncoding 更宽泛 */
|
||||||
|
export type DetectedEncoding = string
|
||||||
|
```
|
||||||
|
- 原因: 后续 Task 2/3/4 需要这些类型来做编码标注和类型收窄
|
||||||
|
|
||||||
|
- [x] 实现 `detectEncoding(buffer: Buffer): FileEncoding` 函数
|
||||||
|
- 位置: `src/utils/encoding.ts`，类型定义之后
|
||||||
|
- 三层检测逻辑:
|
||||||
|
```typescript
|
||||||
|
export function detectEncoding(buffer: Buffer): FileEncoding {
|
||||||
|
// Layer 1: BOM 检测（与现有 fileRead.ts 逻辑一致）
|
||||||
|
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
|
||||||
|
return 'utf-16le'
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
buffer.length >= 3 &&
|
||||||
|
buffer[0] === 0xef &&
|
||||||
|
buffer[1] === 0xbb &&
|
||||||
|
buffer[2] === 0xbf
|
||||||
|
) {
|
||||||
|
return 'utf-8'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer 2: UTF-8 fatal 验证
|
||||||
|
// fatal: true 模式下，无效 UTF-8 字节序列会抛出 TypeError
|
||||||
|
try {
|
||||||
|
new TextDecoder('utf-8', { fatal: true }).decode(buffer)
|
||||||
|
return 'utf-8'
|
||||||
|
} catch {
|
||||||
|
// 不是合法 UTF-8，进入 Layer 3
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer 3: GBK 回退
|
||||||
|
try {
|
||||||
|
new TextDecoder('gbk', { fatal: true }).decode(buffer)
|
||||||
|
return 'gbk'
|
||||||
|
} catch {
|
||||||
|
// 不是合法 GBK，latin1 作为最终兜底
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'latin1'
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: BOM 必须优先于 fatal 验证；GBK 作为唯一回退避免了多编码链的字节歧义问题；latin1 单字节编码永远成功
|
||||||
|
|
||||||
|
- [x] 实现 `decodeBuffer(buffer: Buffer, encoding: DetectedEncoding): string` 函数
|
||||||
|
- 位置: `src/utils/encoding.ts`，`detectEncoding` 之后
|
||||||
|
- 逻辑:
|
||||||
|
```typescript
|
||||||
|
export function decodeBuffer(
|
||||||
|
buffer: Buffer,
|
||||||
|
encoding: DetectedEncoding,
|
||||||
|
): string {
|
||||||
|
return new TextDecoder(encoding).decode(buffer)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: 统一解码入口，后续 Task 2/3 的读取路径都调用此函数
|
||||||
|
|
||||||
|
- [x] 实现 `encodeString(content: string, encoding: DetectedEncoding): { buffer: Buffer; converted: boolean }` 函数
|
||||||
|
- 位置: `src/utils/encoding.ts`，`decodeBuffer` 之后
|
||||||
|
- 逻辑:
|
||||||
|
```typescript
|
||||||
|
export function encodeString(
|
||||||
|
content: string,
|
||||||
|
encoding: DetectedEncoding,
|
||||||
|
): { buffer: Buffer; converted: boolean } {
|
||||||
|
if (encoding === 'utf-8' || encoding === 'utf8') {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-8'), converted: false }
|
||||||
|
}
|
||||||
|
if (encoding === 'utf-16le') {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-16le'), converted: false }
|
||||||
|
}
|
||||||
|
|
||||||
|
// 其他编码（如 gbk）：尝试 Buffer.from，失败则回退为 UTF-8
|
||||||
|
try {
|
||||||
|
const buf = Buffer.from(content, encoding as BufferEncoding)
|
||||||
|
return { buffer: buf, converted: false }
|
||||||
|
} catch {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-8'), converted: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: `Buffer.from` 在 Bun 中可能支持 GBK 编码名，但 Node.js 不支持。try-catch 策略兼容两种运行时；`converted` 标志让 Task 4 的写入路径能向用户报告编码转换
|
||||||
|
|
||||||
|
- [x] 为编码检测和解码函数编写单元测试
|
||||||
|
- 测试文件: `src/utils/__tests__/encoding.test.ts`
|
||||||
|
- 测试场景:
|
||||||
|
- **BOM 检测 — UTF-16LE**: 输入 `Buffer.from([0xff, 0xfe, 0x48, 0x00])` → 预期返回 `'utf-16le'`
|
||||||
|
- **BOM 检测 — UTF-8 BOM**: 输入 `Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65])` → 预期返回 `'utf-8'`
|
||||||
|
- **UTF-8 验证**: 输入 `Buffer.from('Hello, 世界', 'utf-8')` → 预期返回 `'utf-8'`
|
||||||
|
- **GBK 检测**: 输入 `Buffer.from([0xc4, 0xe3, 0xba, 0xc3])` → 预期返回 `'gbk'`
|
||||||
|
- **空 buffer**: 输入 `Buffer.alloc(0)` → 预期返回 `'utf-8'`
|
||||||
|
- **latin1 兜底**: 输入随机字节 `Buffer.from([0x80, 0x81, 0x82, 0x83, 0x84, 0x85])` → 预期返回 `'latin1'`
|
||||||
|
- **BOM 优先于内容分析**: 输入带 UTF-8 BOM 的数据 → 预期返回 `'utf-8'`
|
||||||
|
- **decodeBuffer — UTF-8**: 输入 UTF-8 编码的 buffer + encoding `'utf-8'` → 预期返回正确的中文字符串
|
||||||
|
- **decodeBuffer — GBK**: 输入 GBK 编码的 buffer + encoding `'gbk'` → 预期返回正确的中文字符串
|
||||||
|
- **decodeBuffer — UTF-16LE**: 输入 UTF-16LE 编码的 buffer + encoding `'utf-16le'` → 预期返回正确字符串
|
||||||
|
- **decodeBuffer — 空 buffer**: 输入空 buffer → 预期返回空字符串
|
||||||
|
- **encodeString — UTF-8**: 输入字符串 + encoding `'utf-8'` → 预期 `{ converted: false }`
|
||||||
|
- **encodeString — utf8 别名**: 输入字符串 + encoding `'utf8'` → 预期 `{ converted: false }`
|
||||||
|
- **encodeString — UTF-16LE**: 输入字符串 + encoding `'utf-16le'` → 预期 `{ converted: false }`
|
||||||
|
- **encodeString — GBK**: 输入字符串 + encoding `'gbk'` → 预期返回有效的 Buffer（converted 视运行时而定）
|
||||||
|
- 运行命令: `bun test src/utils/__tests__/encoding.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
**检查步骤:**
|
||||||
|
|
||||||
|
- [x] 验证 `encoding.ts` 文件存在且导出正确
|
||||||
|
- `grep -c "export" src/utils/encoding.ts`
|
||||||
|
- 预期: 输出 >= 4（至少导出 FileEncoding, DetectedEncoding, detectEncoding, decodeBuffer, encodeString 共 5 个导出）
|
||||||
|
|
||||||
|
- [x] 验证类型检查通过
|
||||||
|
- `bunx tsc --noEmit src/utils/encoding.ts 2>&1 | head -5`
|
||||||
|
- 预期: 无类型错误输出
|
||||||
|
|
||||||
|
- [x] 运行编码检测单元测试
|
||||||
|
- `bun test src/utils/__tests__/encoding.test.ts`
|
||||||
|
- 预期: 所有测试通过，无失败用例
|
||||||
|
|
||||||
|
**认知变更:**
|
||||||
|
- [x] [CLAUDE.md] `src/utils/encoding.ts` 是文件编码检测的唯一入口，提供 `detectEncoding`（三层检测：BOM → UTF-8 fatal → GBK 回退）和 `decodeBuffer`/`encodeString` 函数。检测基于文件头部 4KB，零外部依赖，仅使用 TextDecoder API。`FileEncoding` 类型为 `BufferEncoding | 'gbk'`，覆盖最常见非 UTF-8 CJK 编码。latin1 作为最终兜底编码（单字节编码永远成功）。
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
### Task 2: 同步读取路径集成
|
||||||
|
|
||||||
|
**背景:**
|
||||||
|
当前同步读取路径（`fileRead.ts` → `file.ts` → `fileReadCache.ts`）的编码检测仅通过 BOM 头识别 UTF-8 和 UTF-16LE，非 BOM 编码文件一律按 UTF-8 读取导致乱码。本 Task 将 `detectEncodingForResolvedPath` 的内部实现从 BOM-only 升级为调用 Task 1 创建的 `encoding.ts` 三层检测，并将返回类型从 `BufferEncoding` 扩展为 `FileEncoding`。同时将所有 `fs.readFileSync(path, { encoding })` 调用改为先读 Buffer 再用 `decodeBuffer` 解码，以支持 `gbk` 等非 `BufferEncoding` 编码。本 Task 依赖 Task 1（`src/utils/encoding.ts`），输出被 Task 4（写入路径适配）依赖。
|
||||||
|
|
||||||
|
**涉及文件:**
|
||||||
|
- 修改: `src/utils/fileRead.ts`
|
||||||
|
- 修改: `src/utils/file.ts`
|
||||||
|
- 修改: `src/utils/fileReadCache.ts`
|
||||||
|
- 新建: `src/utils/__tests__/fileRead.test.ts`
|
||||||
|
|
||||||
|
**执行步骤:**
|
||||||
|
|
||||||
|
- [x] 在 `fileRead.ts` 中导入 `encoding.ts` 的类型和函数
|
||||||
|
- 位置: `src/utils/fileRead.ts` 文件顶部 import 区域，在 `import { getFsImplementation, safeResolvePath } from './fsOperations.js'` 之后
|
||||||
|
- 添加导入:
|
||||||
|
```typescript
|
||||||
|
import { type FileEncoding, decodeBuffer, detectEncoding } from './encoding.js'
|
||||||
|
```
|
||||||
|
- 原因: 后续步骤需要 `FileEncoding` 类型、`detectEncoding` 检测函数和 `decodeBuffer` 解码函数
|
||||||
|
|
||||||
|
- [x] 改造 `detectEncodingForResolvedPath` 函数，使用 `encoding.ts` 的三层检测
|
||||||
|
- 位置: `src/utils/fileRead.ts` 的 `detectEncodingForResolvedPath` 函数
|
||||||
|
- 将函数体替换为以下逻辑:
|
||||||
|
```typescript
|
||||||
|
export function detectEncodingForResolvedPath(
|
||||||
|
resolvedPath: string,
|
||||||
|
): FileEncoding {
|
||||||
|
const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, {
|
||||||
|
length: 4096,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Empty files default to utf8 — nothing to detect
|
||||||
|
if (bytesRead === 0) {
|
||||||
|
return 'utf8'
|
||||||
|
}
|
||||||
|
|
||||||
|
return detectEncoding(buffer.subarray(0, bytesRead))
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 关键变更:
|
||||||
|
- 返回类型从 `BufferEncoding` 改为 `FileEncoding`
|
||||||
|
- 删除内联的 BOM 检测逻辑，改为调用 `detectEncoding(buffer.subarray(0, bytesRead))`
|
||||||
|
- 使用 `buffer.subarray(0, bytesRead)` 截取实际读取的字节，避免尾部零字节干扰检测
|
||||||
|
- 原因: 将检测逻辑委托给 `encoding.ts` 的三层算法，消除代码重复
|
||||||
|
|
||||||
|
- [x] 改造 `readFileSyncWithMetadata` 函数，支持非 `BufferEncoding` 解码
|
||||||
|
- 位置: `src/utils/fileRead.ts` 的 `readFileSyncWithMetadata` 函数
|
||||||
|
- 将函数签名和内部逻辑改为:
|
||||||
|
```typescript
|
||||||
|
export function readFileSyncWithMetadata(filePath: string): {
|
||||||
|
content: string
|
||||||
|
encoding: FileEncoding
|
||||||
|
lineEndings: LineEndingType
|
||||||
|
} {
|
||||||
|
const fs = getFsImplementation()
|
||||||
|
const { resolvedPath, isSymlink } = safeResolvePath(fs, filePath)
|
||||||
|
|
||||||
|
if (isSymlink) {
|
||||||
|
logForDebugging(`Reading through symlink: ${filePath} -> ${resolvedPath}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const encoding = detectEncodingForResolvedPath(resolvedPath)
|
||||||
|
// Read raw Buffer first — fs.readFileSync encoding option only accepts
|
||||||
|
// BufferEncoding, not gbk etc.
|
||||||
|
const rawBuffer = fs.readFileBytesSync(resolvedPath)
|
||||||
|
const raw = decodeBuffer(rawBuffer, encoding)
|
||||||
|
const lineEndings = detectLineEndingsForString(raw.slice(0, 4096))
|
||||||
|
return {
|
||||||
|
content: raw.replaceAll('\r\n', '\n'),
|
||||||
|
encoding,
|
||||||
|
lineEndings,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 关键变更:
|
||||||
|
- 返回类型中 `encoding` 从 `BufferEncoding` 改为 `FileEncoding`
|
||||||
|
- `fs.readFileSync(resolvedPath, { encoding })` 改为 `fs.readFileBytesSync(resolvedPath)` 读取 Buffer
|
||||||
|
- 新增 `decodeBuffer(rawBuffer, encoding)` 解码为字符串
|
||||||
|
- 原因: `fs.readFileSync` 的 `encoding` 选项只接受 `BufferEncoding`（utf8/utf16le/latin1 等），传入 `'gbk'` 会在运行时报错
|
||||||
|
|
||||||
|
- [x] 更新 `file.ts` 中 `detectFileEncoding` 的返回类型
|
||||||
|
- 位置: `src/utils/file.ts` 的 `detectFileEncoding` 函数签名
|
||||||
|
- 将 `): BufferEncoding {` 改为 `): FileEncoding {`
|
||||||
|
- 在文件顶部 import 区域添加:
|
||||||
|
```typescript
|
||||||
|
import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js'
|
||||||
|
```
|
||||||
|
- 原因: `detectFileEncoding` 调用 `detectEncodingForResolvedPath`，返回类型已改为 `FileEncoding`
|
||||||
|
|
||||||
|
- [x] 更新 `file.ts` 中 `detectLineEndings` 的 encoding 参数类型和解码逻辑
|
||||||
|
- 位置: `src/utils/file.ts` 的 `detectLineEndings` 函数
|
||||||
|
- 将函数签名改为:
|
||||||
|
```typescript
|
||||||
|
export function detectLineEndings(
|
||||||
|
filePath: string,
|
||||||
|
encoding: FileEncoding = 'utf8',
|
||||||
|
): LineEndingType {
|
||||||
|
```
|
||||||
|
- 将内部 `buffer.toString(encoding, 0, bytesRead)` 改为:
|
||||||
|
```typescript
|
||||||
|
const content = decodeBuffer(buffer.subarray(0, bytesRead), encoding)
|
||||||
|
```
|
||||||
|
- 原因: `buffer.toString('gbk')` 不可靠，统一使用 `decodeBuffer` 通过 `TextDecoder` 解码
|
||||||
|
|
||||||
|
- [x] 更新 `fileReadCache.ts` 的类型和解码逻辑
|
||||||
|
- 位置: `src/utils/fileReadCache.ts`
|
||||||
|
- 在文件顶部 import 区域添加:
|
||||||
|
```typescript
|
||||||
|
import { type FileEncoding, decodeBuffer } from './encoding.js'
|
||||||
|
```
|
||||||
|
- 将 `CachedFileData` 类型中 `encoding: BufferEncoding` 改为 `encoding: FileEncoding`
|
||||||
|
- 将 `readFile` 方法返回类型改为 `{ content: string; encoding: FileEncoding }`
|
||||||
|
- 将缓存未命中读取逻辑改为:
|
||||||
|
```typescript
|
||||||
|
const encoding = detectFileEncoding(filePath)
|
||||||
|
const rawBuffer = fs.readFileBytesSync(filePath)
|
||||||
|
const content = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n')
|
||||||
|
```
|
||||||
|
- 原因: 与 `fileRead.ts` 相同——必须改为 Buffer 读取 + `decodeBuffer` 解码
|
||||||
|
|
||||||
|
- [x] 为改造后的 `detectEncodingForResolvedPath` 和 `readFileSyncWithMetadata` 编写单元测试
|
||||||
|
- 测试文件: `src/utils/__tests__/fileRead.test.ts`
|
||||||
|
- 测试场景:
|
||||||
|
- **UTF-8 文件读取**: 创建临时 UTF-8 文件 → 返回 `encoding: 'utf-8'`，content 与写入内容一致
|
||||||
|
- **GBK 文件读取**: 创建临时 GBK 编码文件 → 返回 `encoding: 'gbk'`，content 包含正确的中文字符
|
||||||
|
- **空文件读取**: 创建空文件 → 返回 `encoding: 'utf8'`，content 为空字符串
|
||||||
|
- **UTF-16LE BOM 文件读取**: 创建带 BOM 的 UTF-16LE 文件 → 返回 `encoding: 'utf-16le'`
|
||||||
|
- **detectEncodingForResolvedPath 返回类型**: 验证返回值为 `FileEncoding` 类型
|
||||||
|
- Mock 策略: 使用 `tests/mocks/debug.ts` mock `debug.ts`，使用 `tests/mocks/log.ts` mock `log.ts`
|
||||||
|
- 运行命令: `bun test src/utils/__tests__/fileRead.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
**检查步骤:**
|
||||||
|
|
||||||
|
- [x] 验证 `fileRead.ts` 的导入和返回类型已更新
|
||||||
|
- `grep -n "FileEncoding\|decodeBuffer\|detectEncoding" src/utils/fileRead.ts`
|
||||||
|
- 预期: 输出包含 import 行中的 `FileEncoding`、`decodeBuffer`，以及函数体中的 `detectEncoding` 调用
|
||||||
|
|
||||||
|
- [x] 验证 `file.ts` 的类型已更新
|
||||||
|
- `grep -n "FileEncoding\|decodeBuffer" src/utils/file.ts`
|
||||||
|
- 预期: `detectFileEncoding` 返回 `FileEncoding`，`detectLineEndings` 参数类型为 `FileEncoding`
|
||||||
|
|
||||||
|
- [x] 验证 `fileReadCache.ts` 的类型已更新
|
||||||
|
- `grep -n "FileEncoding\|decodeBuffer" src/utils/fileReadCache.ts`
|
||||||
|
- 预期: `CachedFileData` 和 `readFile` 返回类型使用 `FileEncoding`
|
||||||
|
|
||||||
|
- [x] 验证 `fileRead.ts` 中不再有内联 BOM 检测逻辑
|
||||||
|
- `grep -c "0xff\|0xfe\|0xef\|0xbb\|0xbf" src/utils/fileRead.ts`
|
||||||
|
- 预期: 输出为 0
|
||||||
|
|
||||||
|
- [x] 运行 fileRead 单元测试
|
||||||
|
- `bun test src/utils/__tests__/fileRead.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
- [x] 运行 precheck 确认无类型/lint/测试错误
|
||||||
|
- `bun run precheck`
|
||||||
|
- 预期: 零错误通过
|
||||||
|
|
||||||
|
**认知变更:**
|
||||||
|
- [x] [CLAUDE.md] `fs.readFileSync(path, { encoding })` 的 `encoding` 选项只接受 `BufferEncoding`（utf8/utf16le/latin1/ascii/binary/hex/base64/ucs2/utf16le），不支持 `gbk` 等 ICU 编码名。读取非 UTF-8 文件时必须先 `fs.readFileSync(path)` 读 Buffer，再用 `TextDecoder` 解码。项目中所有文件读取路径（fileRead.ts、fileReadCache.ts、file.ts）已统一使用 `decodeBuffer` 函数处理此逻辑。
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
### Task 3: 异步读取路径改造
|
||||||
|
|
||||||
|
**背景:**
|
||||||
|
当前 `src/utils/readFileInRange.ts` 是 FileReadTool 的核心异步读取函数，提供 fast path（小文件整体读入）和 streaming path（大文件逐块扫描）两条路径，两者均硬编码 `encoding: 'utf8'`，导致非 UTF-8 编码文件读取乱码。本 Task 将两条路径改造为 Buffer 读取 + 编码检测 + TextDecoder 解码模式。fast path 改造简单（整体读 Buffer 后检测解码），streaming path 需要两阶段设计（先收集前 4KB 做编码检测，再用 `TextDecoder({ stream: true })` 逐 chunk 解码）。本 Task 依赖 Task 1（`src/utils/encoding.ts` 的 `detectEncoding` 和 `decodeBuffer`），输出被 Task 4 依赖（通过 `readFileInRange` 的返回值间接影响）。
|
||||||
|
|
||||||
|
**涉及文件:**
|
||||||
|
- 修改: `src/utils/readFileInRange.ts`
|
||||||
|
- 新建: `src/utils/__tests__/readFileInRange.test.ts`
|
||||||
|
|
||||||
|
**执行步骤:**
|
||||||
|
|
||||||
|
- [x] 在 `readFileInRange.ts` 中导入 `encoding.ts` 的函数
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 文件顶部 import 区域，在 `import { formatFileSize } from './format.js'` 之后
|
||||||
|
- 添加导入:
|
||||||
|
```typescript
|
||||||
|
import { detectEncoding, decodeBuffer } from './encoding.js'
|
||||||
|
```
|
||||||
|
- 原因: fast path 和 streaming path 都需要 `detectEncoding` 做编码检测，fast path 需要 `decodeBuffer` 做一次性解码
|
||||||
|
|
||||||
|
- [x] 改造 fast path — 将 `readFile` 从 UTF-8 字符串读取改为 Buffer 读取 + 检测 + 解码
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 的 `readFileInRange` 函数内 fast path 分支
|
||||||
|
- 将以下代码:
|
||||||
|
```typescript
|
||||||
|
const text = await readFile(filePath, { encoding: 'utf8', signal })
|
||||||
|
return readFileInRangeFast(text, stats.mtimeMs, offset, maxLines, ...)
|
||||||
|
```
|
||||||
|
替换为:
|
||||||
|
```typescript
|
||||||
|
const rawBuffer = await readFile(filePath, { signal })
|
||||||
|
const encoding = detectEncoding(rawBuffer)
|
||||||
|
const text = decodeBuffer(rawBuffer, encoding)
|
||||||
|
return readFileInRangeFast(text, stats.mtimeMs, offset, maxLines, ...)
|
||||||
|
```
|
||||||
|
- 关键变更: `readFile` 去掉 `encoding: 'utf8'` 选项，返回 `Buffer`；调用 `detectEncoding(rawBuffer)` 检测编码；调用 `decodeBuffer(rawBuffer, encoding)` 解码为字符串。
|
||||||
|
- 原因: `readFile` 的 `encoding` 选项只支持 `BufferEncoding`，不支持 `gbk` 等 ICU 编码名
|
||||||
|
|
||||||
|
- [x] 改造 streaming path — 扩展 `StreamState` 类型，增加编码检测和解码相关字段
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 的 `StreamState` 类型定义
|
||||||
|
- 在现有字段之后添加以下字段:
|
||||||
|
```typescript
|
||||||
|
type StreamState = {
|
||||||
|
// ... 现有字段保持不变 ...
|
||||||
|
/** 编码检测状态：null 表示尚未检测，string 表示已检测完成 */
|
||||||
|
encoding: string | null
|
||||||
|
/** TextDecoder 实例：检测完成后创建，用于逐 chunk 流式解码 */
|
||||||
|
decoder: TextDecoder | null
|
||||||
|
/** 检测阶段缓冲区：收集原始字节直到满 4KB 或 stream 结束 */
|
||||||
|
detectionBuffer: number[]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: streaming 模式下 chunk 是增量到达的，需要缓冲阶段收集足够字节来调用 `detectEncoding`
|
||||||
|
|
||||||
|
- [x] 改造 `streamOnData` — 处理 Buffer chunk，实现两阶段（检测阶段 + 解码阶段）
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 的 `streamOnData` 函数
|
||||||
|
- 将函数签名从 `streamOnData(this: StreamState, chunk: string): void` 改为 `streamOnData(this: StreamState, chunk: Buffer): void`
|
||||||
|
- 替换函数体为两阶段逻辑:
|
||||||
|
```typescript
|
||||||
|
function streamOnData(this: StreamState, chunk: Buffer): void {
|
||||||
|
this.totalBytesRead += chunk.length
|
||||||
|
|
||||||
|
// ... maxBytes 检查保持不变 ...
|
||||||
|
|
||||||
|
// Phase 1: 编码检测阶段
|
||||||
|
if (this.encoding === null) {
|
||||||
|
for (let i = 0; i < chunk.length; i++) {
|
||||||
|
this.detectionBuffer.push(chunk[i])
|
||||||
|
}
|
||||||
|
if (this.detectionBuffer.length >= 4096) {
|
||||||
|
this.encoding = detectEncoding(Buffer.from(this.detectionBuffer))
|
||||||
|
this.decoder = new TextDecoder(this.encoding, { stream: true })
|
||||||
|
const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer))
|
||||||
|
this.detectionBuffer = []
|
||||||
|
processTextChunk(this, decoded)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2: 解码阶段
|
||||||
|
const decoded = this.decoder!.decode(chunk, { stream: true })
|
||||||
|
processTextChunk(this, decoded)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: 两阶段设计确保编码检测在足够数据上执行（至少 4KB），检测完成后用 `TextDecoder({ stream: true })` 逐 chunk 解码
|
||||||
|
|
||||||
|
- [x] 提取行扫描逻辑为独立的 `processTextChunk` 辅助函数
|
||||||
|
- 位置: `src/utils/readFileInRange.ts`，在 `streamOnData` 函数定义之前
|
||||||
|
- 从原 `streamOnData` 提取行扫描逻辑到独立函数 `processTextChunk(state: StreamState, text: string): void`
|
||||||
|
- 行扫描逻辑与原实现完全一致，仅变量名从 `this.` 改为 `state.`
|
||||||
|
- 原因: 检测阶段和解码阶段复用同一段行扫描逻辑
|
||||||
|
|
||||||
|
- [x] 改造 `streamOnEnd` — 处理检测阶段缓冲区残留和最终 fragment
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 的 `streamOnEnd` 函数
|
||||||
|
- 在函数体开头插入检测阶段完成逻辑:
|
||||||
|
```typescript
|
||||||
|
if (this.encoding === null) {
|
||||||
|
this.encoding = detectEncoding(Buffer.from(this.detectionBuffer))
|
||||||
|
this.decoder = new TextDecoder(this.encoding, { stream: true })
|
||||||
|
const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer))
|
||||||
|
this.detectionBuffer = []
|
||||||
|
processTextChunk(this, decoded)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: 小文件可能 < 4KB，stream 在检测缓冲区未满时就结束。必须在 `streamOnEnd` 中完成检测和解码
|
||||||
|
|
||||||
|
- [x] 改造 `readFileInRangeStreaming` — 创建 Buffer 模式的 stream，初始化新增字段
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 的 `readFileInRangeStreaming` 函数
|
||||||
|
- 将 `createReadStream` 调用去掉 `encoding: 'utf8'` 选项
|
||||||
|
- 在 `state` 对象初始化中添加新字段: `encoding: null, decoder: null, detectionBuffer: []`
|
||||||
|
- 原因: 去掉 `encoding: 'utf8'` 后，`data` 事件回调接收 `Buffer` 对象
|
||||||
|
|
||||||
|
- [x] 更新文件顶部注释，反映编码检测能力
|
||||||
|
- 位置: `src/utils/readFileInRange.ts` 文件顶部注释
|
||||||
|
- 注释已更新为: `Both paths auto-detect encoding via encoding.ts (BOM → UTF-8 fatal → fallback chain), decode with TextDecoder, and strip BOM and \r (CRLF → LF).`
|
||||||
|
|
||||||
|
- [x] 为改造后的 `readFileInRange` 编写单元测试
|
||||||
|
- 测试文件: `src/utils/__tests__/readFileInRange.test.ts`
|
||||||
|
- 测试场景:
|
||||||
|
- **Fast path — UTF-8 文件**: 创建临时 UTF-8 文件 → 返回正确的 `content`、`lineCount`、`totalLines`
|
||||||
|
- **Fast path — GBK 文件**: 创建临时 GBK 编码文件 → 返回正确的中文内容（非乱码），`totalBytes` 正确
|
||||||
|
- **Fast path — 带行范围读取 GBK 文件**: 创建包含多行的 GBK 文件 → 返回指定行范围，内容正确
|
||||||
|
- **Streaming path — 大 UTF-8 文件**: 创建超过 10MB 阈值的 UTF-8 文件 → 返回正确内容
|
||||||
|
- **Streaming path — 大 GBK 文件**: 创建超过 10MB 阈值的 GBK 编码文件 → 返回正确的中文内容
|
||||||
|
- **BOM 剥离**: 创建带 UTF-8 BOM 的文件 → `content` 不包含 BOM 字符
|
||||||
|
- **空文件**: 创建空文件 → `content` 为空字符串，`totalLines` 为 1，`totalBytes` 为 0
|
||||||
|
- 运行命令: `bun test src/utils/__tests__/readFileInRange.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
**检查步骤:**
|
||||||
|
|
||||||
|
- [x] 验证 `readFileInRange.ts` 已导入 `encoding.ts` 的函数
|
||||||
|
- `grep -n "detectEncoding\|decodeBuffer" src/utils/readFileInRange.ts`
|
||||||
|
- 预期: import 行包含 `detectEncoding` 和 `decodeBuffer`，函数体中包含调用
|
||||||
|
|
||||||
|
- [x] 验证 streaming path 不再硬编码 `encoding: 'utf8'`
|
||||||
|
- `grep -n "encoding: 'utf8'\|encoding: \"utf8\"" src/utils/readFileInRange.ts`
|
||||||
|
- 预期: 无匹配结果
|
||||||
|
|
||||||
|
- [x] 验证 `createReadStream` 调用无 encoding 选项
|
||||||
|
- `grep -A3 "createReadStream" src/utils/readFileInRange.ts`
|
||||||
|
- 预期: `createReadStream` 的选项对象中不包含 `encoding` 属性
|
||||||
|
|
||||||
|
- [x] 验证 `StreamState` 类型包含编码检测新字段
|
||||||
|
- `grep -n "encoding:\|decoder:\|detectionBuffer:" src/utils/readFileInRange.ts`
|
||||||
|
- 预期: `StreamState` 类型定义中包含 `encoding`、`decoder`、`detectionBuffer` 字段
|
||||||
|
|
||||||
|
- [x] 验证 `processTextChunk` 函数存在
|
||||||
|
- `grep -n "function processTextChunk" src/utils/readFileInRange.ts`
|
||||||
|
- 预期: 函数定义存在
|
||||||
|
|
||||||
|
- [x] 运行 readFileInRange 单元测试
|
||||||
|
- `bun test src/utils/__tests__/readFileInRange.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
- [x] 运行 precheck 确认无类型/lint/测试错误
|
||||||
|
- `bun run precheck`
|
||||||
|
- 预期: 零错误通过
|
||||||
|
|
||||||
|
**认知变更:**
|
||||||
|
- [x] [CLAUDE.md] `readFileInRange.ts` 的 streaming path 使用两阶段编码检测：先收集前 4KB 字节调用 `detectEncoding`，再用 `TextDecoder({ stream: true })` 逐 chunk 流式解码。`TextDecoder` 的 `{ stream: true }` 模式会自动处理多字节字符跨 chunk 边界问题。对于 < 4KB 的小文件，检测在 `streamOnEnd` 中完成。
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
### Task 4: 写入路径和工具层适配
|
||||||
|
|
||||||
|
**背景:**
|
||||||
|
[业务语境] — 当用户通过 FileEditTool 或 FileWriteTool 编辑非 UTF-8 编码文件（如 GBK）时，写入操作需要将内部 UTF-8 字符串编码回原文件编码，否则写入的内容会乱码。当前 `writeTextContent` 只接受 `BufferEncoding` 类型，无法处理 gbk 等编码。
|
||||||
|
[修改原因] — `writeTextContent` 的 `encoding` 参数类型为 `BufferEncoding`，`writeFileSyncAndFlush_DEPRECATED` 内部直接将 encoding 传给 `fs.writeFileSync`（只接受标准 BufferEncoding）。`FileEditTool.validateInput` 中硬编码了 BOM-only 编码检测，无法识别 GBK 文件。
|
||||||
|
[上下游影响] — 本 Task 依赖 Task 1 创建的 `encodeString` 函数和 `FileEncoding` 类型。`FileEditTool` 和 `FileWriteTool` 通过 `writeTextContent` 间接依赖本 Task 的改造。BashTool 和 NotebookEditTool 也调用 `writeTextContent`，签名变更后它们无需额外改动（encoding 参数类型由上游传入，自动兼容）。
|
||||||
|
|
||||||
|
**涉及文件:**
|
||||||
|
- 修改: `src/utils/file.ts`
|
||||||
|
- 修改: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts`
|
||||||
|
|
||||||
|
**执行步骤:**
|
||||||
|
|
||||||
|
- [x] 在 `src/utils/file.ts` 中合并 `encodeString` 到 Task 2 已创建的 `encoding.js` 导入
|
||||||
|
- 位置: 文件导入区域，Task 2 已添加的 `import { type FileEncoding, decodeBuffer } from './encoding.js'` 行
|
||||||
|
- 将该行改为: `import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js'`
|
||||||
|
- 原因: 避免对同一模块创建两个 import 语句
|
||||||
|
|
||||||
|
- [x] 将 `writeTextContent` 的 `encoding` 参数类型从 `BufferEncoding` 改为 `FileEncoding`
|
||||||
|
- 位置: `src/utils/file.ts:writeTextContent()`
|
||||||
|
- 修改函数签名:
|
||||||
|
```typescript
|
||||||
|
export function writeTextContent(
|
||||||
|
filePath: string,
|
||||||
|
content: string,
|
||||||
|
encoding: FileEncoding,
|
||||||
|
endings: LineEndingType,
|
||||||
|
): void
|
||||||
|
```
|
||||||
|
- 修改函数体，在行尾处理之后、调用 `writeFileSyncAndFlush_DEPRECATED` 之前，增加编码判断逻辑:
|
||||||
|
```typescript
|
||||||
|
const BUFFER_ENCODINGS = new Set<string>([
|
||||||
|
'utf8', 'utf-8', 'utf16le', 'ucs2', 'ucs-2',
|
||||||
|
'ascii', 'latin1', 'binary', 'base64', 'hex',
|
||||||
|
])
|
||||||
|
|
||||||
|
if (BUFFER_ENCODINGS.has(encoding)) {
|
||||||
|
writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, { encoding: encoding as BufferEncoding })
|
||||||
|
} else {
|
||||||
|
// 非 BufferEncoding（如 gbk），使用 encodeString 获取 Buffer
|
||||||
|
const { buffer, converted } = encodeString(toWrite, encoding)
|
||||||
|
writeFileSyncAndFlush_DEPRECATED(filePath, buffer, { buffer })
|
||||||
|
if (converted) {
|
||||||
|
logForDebugging(
|
||||||
|
`writeTextContent: encoding '${encoding}' unsupported for write, fell back to UTF-8 for ${filePath}`,
|
||||||
|
{ level: 'warn' },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: `fs.writeFileSync` 只接受标准 BufferEncoding，对于 gbk 等编码必须先转为 Buffer 再写入
|
||||||
|
|
||||||
|
- [x] 扩展 `writeFileSyncAndFlush_DEPRECATED` 支持 Buffer 写入
|
||||||
|
- 位置: `src/utils/file.ts:writeFileSyncAndFlush_DEPRECATED()`
|
||||||
|
- 修改函数签名中 `content` 参数类型和 `options` 类型:
|
||||||
|
```typescript
|
||||||
|
export function writeFileSyncAndFlush_DEPRECATED(
|
||||||
|
filePath: string,
|
||||||
|
content: string | Buffer,
|
||||||
|
options: { encoding?: BufferEncoding; mode?: number; buffer?: Buffer } = {},
|
||||||
|
): void
|
||||||
|
```
|
||||||
|
- 修改原子写入路径的 `writeOptions` 构建逻辑:
|
||||||
|
```typescript
|
||||||
|
const isBufferWrite = Buffer.isBuffer(content) || options.buffer !== undefined
|
||||||
|
const writeData = options.buffer ?? content
|
||||||
|
const writeOptions: {
|
||||||
|
encoding?: BufferEncoding
|
||||||
|
flush: boolean
|
||||||
|
mode?: number
|
||||||
|
} = {
|
||||||
|
flush: true,
|
||||||
|
...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }),
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 修改非原子回退路径，使用相同的 `isBufferWrite` / `writeData` / `writeOptions` 模式
|
||||||
|
- 原因: `fs.writeFileSync(path, buffer)` 可以直接写入 Buffer，不需要 encoding 参数
|
||||||
|
|
||||||
|
- [x] 在 `FileEditTool.ts` 中导入 `FileEncoding` 和 `detectEncoding` / `decodeBuffer`
|
||||||
|
- 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts` 导入区域
|
||||||
|
- 添加: `import { detectEncoding, decodeBuffer, type FileEncoding } from 'src/utils/encoding.js'`
|
||||||
|
- 原因: `validateInput` 编码检测和 `readFileForEdit` 返回类型需要 `FileEncoding` 类型
|
||||||
|
|
||||||
|
- [x] 将 `readFileForEdit` 返回类型中的 `encoding` 从 `BufferEncoding` 改为 `FileEncoding`
|
||||||
|
- 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts:readFileForEdit()`
|
||||||
|
- 修改返回类型声明:
|
||||||
|
```typescript
|
||||||
|
function readFileForEdit(absoluteFilePath: string): {
|
||||||
|
content: string
|
||||||
|
fileExists: boolean
|
||||||
|
encoding: FileEncoding
|
||||||
|
lineEndings: LineEndingType
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- 原因: `readFileSyncWithMetadata` 返回的 `encoding` 类型已由 Task 2 改为 `FileEncoding`
|
||||||
|
|
||||||
|
- [x] 改造 `FileEditTool.validateInput` 中的编码检测逻辑
|
||||||
|
- 位置: `packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts:validateInput()`
|
||||||
|
- 将现有的 BOM-only 编码检测:
|
||||||
|
```typescript
|
||||||
|
const encoding: BufferEncoding =
|
||||||
|
fileBuffer.length >= 2 && fileBuffer[0] === 0xff && fileBuffer[1] === 0xfe
|
||||||
|
? 'utf16le'
|
||||||
|
: 'utf8'
|
||||||
|
fileContent = fileBuffer.toString(encoding).replaceAll('\r\n', '\n')
|
||||||
|
```
|
||||||
|
- 替换为:
|
||||||
|
```typescript
|
||||||
|
const encoding: FileEncoding = detectEncoding(fileBuffer)
|
||||||
|
fileContent = decodeBuffer(fileBuffer, encoding).replaceAll('\r\n', '\n')
|
||||||
|
```
|
||||||
|
- 原因: 使 validateInput 也能正确识别 GBK 文件，避免编辑时因编码检测不一致导致 old_string 匹配失败
|
||||||
|
|
||||||
|
- [x] 为 `writeTextContent` 的多编码写入能力编写单元测试
|
||||||
|
- 测试文件: `src/utils/__tests__/file.test.ts`
|
||||||
|
- 在现有测试 describe 块之后追加新的 describe('writeTextContent with multi-encoding') 块
|
||||||
|
- 测试场景:
|
||||||
|
- UTF-8 写入: 写入 UTF-8 内容 → 文件内容正确，无回退警告
|
||||||
|
- UTF-16LE 写入: 写入 UTF-16LE 内容（含 BOM） → 文件二进制内容与预期一致
|
||||||
|
- GBK 写入回退: 对 gbk 编码调用 `writeTextContent` → 文件以 UTF-8 写入（`encodeString` 回退行为），内容不损坏
|
||||||
|
- CRLF 行尾 + GBK: `endings: 'CRLF'` + gbk 编码 → 行尾正确转换为 `\r\n`，编码回退为 UTF-8
|
||||||
|
- 注意: 需要 mock `src/utils/debug.ts`（使用共享 mock `tests/mocks/debug.ts`）
|
||||||
|
- 运行命令: `bun test src/utils/__tests__/file.test.ts`
|
||||||
|
- 预期: 所有测试通过
|
||||||
|
|
||||||
|
**检查步骤:**
|
||||||
|
- [x] 验证 `writeTextContent` 签名使用 `FileEncoding` 类型
|
||||||
|
- `grep -n 'encoding: FileEncoding' src/utils/file.ts`
|
||||||
|
- 预期: 输出包含 `writeTextContent` 函数定义行
|
||||||
|
|
||||||
|
- [x] 验证 `writeFileSyncAndFlush_DEPRECATED` 支持 Buffer 写入
|
||||||
|
- `grep -n 'content: string | Buffer' src/utils/file.ts`
|
||||||
|
- 预期: 输出包含 `writeFileSyncAndFlush_DEPRECATED` 函数定义行
|
||||||
|
|
||||||
|
- [x] 验证 `FileEditTool.readFileForEdit` 返回类型已更新
|
||||||
|
- `grep -n 'encoding: FileEncoding' packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts`
|
||||||
|
- 预期: 输出包含 `readFileForEdit` 函数的返回类型声明
|
||||||
|
|
||||||
|
- [x] 验证 `FileEditTool.validateInput` 使用 `detectEncoding`
|
||||||
|
- `grep -n 'detectEncoding' packages/builtin-tools/src/tools/FileEditTool/FileEditTool.ts`
|
||||||
|
- 预期: 输出包含 validateInput 内部的调用
|
||||||
|
|
||||||
|
- [x] 运行 file.ts 单元测试
|
||||||
|
- `bun test src/utils/__tests__/file.test.ts`
|
||||||
|
- 预期: 所有测试通过，无新增失败
|
||||||
|
|
||||||
|
- [x] 运行 FileEditTool 工具函数测试
|
||||||
|
- `bun test packages/builtin-tools/src/tools/FileEditTool/__tests__/utils.test.ts`
|
||||||
|
- 预期: 所有现有测试通过
|
||||||
|
|
||||||
|
- [x] 运行完整 precheck
|
||||||
|
- `bun run precheck`
|
||||||
|
- 预期: typecheck + lint + test 零错误通过
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# 多编码文件工具 执行计划
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**目标:** 为文件读写工具添加自动编码检测，支持 GBK 编码的透明读写（latin1 作为最终兜底）。
|
||||||
|
|
||||||
|
**技术栈:** TextDecoder/TextEncoder（零外部依赖）、Bun test 框架、TypeScript strict mode
|
||||||
|
|
||||||
|
**设计文档:** spec/feature_20260510_F001_multi-encoding-file-tools/spec-design.md
|
||||||
|
|
||||||
|
**范围变更:** 仅保留 GBK 编码支持，Shift_JIS/EUC-JP/EUC-KR/Big5/GB18030/ISO-8859-1 已移除。
|
||||||
|
|
||||||
|
## 改动总览
|
||||||
|
|
||||||
|
新建编码检测核心模块 `src/utils/encoding.ts`，提供三层检测（BOM → UTF-8 fatal 验证 → GBK 回退 → latin1 兜底）和解码工具函数。同步读取路径（fileRead.ts → file.ts → fileReadCache.ts）集成新检测逻辑，异步读取路径（readFileInRange.ts）改造为 Buffer 读取 + 检测后解码。写入路径（writeTextContent）扩展类型支持新编码名，非标准编码回退为 UTF-8 写入。FileEditTool 和 FileWriteTool 仅需类型适配。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 任务索引
|
||||||
|
|
||||||
|
### Task 0: 环境准备
|
||||||
|
📄 详情见: `spec-plan-task-0.md`
|
||||||
|
|
||||||
|
验证构建工具链和测试环境是否就绪，确认 Bun 运行时对 GBK 编码的 TextDecoder 支持。
|
||||||
|
|
||||||
|
### Task 1: 编码检测核心模块
|
||||||
|
📄 详情见: `spec-plan-task-1.md`
|
||||||
|
|
||||||
|
新建 `src/utils/encoding.ts`，实现三层编码检测算法（BOM → UTF-8 fatal 验证 → GBK 回退）和 Buffer 解码/编码函数。
|
||||||
|
|
||||||
|
### Task 2: 同步读取路径集成
|
||||||
|
📄 详情见: `spec-plan-task-2.md`
|
||||||
|
|
||||||
|
改造 `fileRead.ts` 和 `file.ts` 的编码检测，集成新模块，更新类型定义。
|
||||||
|
|
||||||
|
### Task 3: 异步读取路径改造
|
||||||
|
📄 详情见: `spec-plan-task-3.md`
|
||||||
|
|
||||||
|
改造 `readFileInRange.ts` 的 fast path 和 streaming path，支持非 UTF-8 编码。
|
||||||
|
|
||||||
|
### Task 4: 写入路径和工具层适配
|
||||||
|
📄 详情见: `spec-plan-task-4.md`
|
||||||
|
|
||||||
|
扩展写入路径类型，更新 FileEditTool/FileWriteTool 的类型注解。
|
||||||
|
|
||||||
|
### Acceptance Task
|
||||||
|
📄 详情见: `spec-plan-acceptance.md`
|
||||||
|
|
||||||
|
端到端验证所有功能是否正确实现。
|
||||||
@@ -3,6 +3,7 @@ import React, { Suspense, use, useMemo } from 'react';
|
|||||||
import { FileEditToolDiff } from 'src/components/FileEditToolDiff.js';
|
import { FileEditToolDiff } from 'src/components/FileEditToolDiff.js';
|
||||||
import { getCwd } from 'src/utils/cwd.js';
|
import { getCwd } from 'src/utils/cwd.js';
|
||||||
import { isENOENT } from 'src/utils/errors.js';
|
import { isENOENT } from 'src/utils/errors.js';
|
||||||
|
import { decodeBuffer } from 'src/utils/encoding.js';
|
||||||
import { detectEncodingForResolvedPath } from 'src/utils/fileRead.js';
|
import { detectEncodingForResolvedPath } from 'src/utils/fileRead.js';
|
||||||
import { getFsImplementation } from 'src/utils/fsOperations.js';
|
import { getFsImplementation } from 'src/utils/fsOperations.js';
|
||||||
import { Text } from '@anthropic/ink';
|
import { Text } from '@anthropic/ink';
|
||||||
@@ -33,9 +34,10 @@ export function SedEditPermissionRequest({ sedInfo, ...props }: SedEditPermissio
|
|||||||
// render correctly. This matches what readFileSync did before the
|
// render correctly. This matches what readFileSync did before the
|
||||||
// async conversion.
|
// async conversion.
|
||||||
const encoding = detectEncodingForResolvedPath(filePath);
|
const encoding = detectEncodingForResolvedPath(filePath);
|
||||||
const raw = await getFsImplementation().readFile(filePath, { encoding });
|
const rawBuffer = await getFsImplementation().readFileBytes(filePath);
|
||||||
|
const raw = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n');
|
||||||
return {
|
return {
|
||||||
oldContent: raw.replaceAll('\r\n', '\n'),
|
oldContent: raw,
|
||||||
fileExists: true,
|
fileExists: true,
|
||||||
};
|
};
|
||||||
})().catch((e: unknown): FileReadResult => {
|
})().catch((e: unknown): FileReadResult => {
|
||||||
|
|||||||
102
src/utils/__tests__/encoding.test.ts
Normal file
102
src/utils/__tests__/encoding.test.ts
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
import { describe, test, expect } from 'bun:test'
|
||||||
|
import {
|
||||||
|
detectEncoding,
|
||||||
|
decodeBuffer,
|
||||||
|
encodeString,
|
||||||
|
type FileEncoding,
|
||||||
|
type DetectedEncoding,
|
||||||
|
} from '../encoding'
|
||||||
|
|
||||||
|
describe('detectEncoding', () => {
|
||||||
|
test('detects UTF-16LE BOM', () => {
|
||||||
|
const buf = Buffer.from([0xff, 0xfe, 0x48, 0x00])
|
||||||
|
expect(detectEncoding(buf)).toBe('utf-16le')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects UTF-8 BOM', () => {
|
||||||
|
const buf = Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65])
|
||||||
|
expect(detectEncoding(buf)).toBe('utf-8')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects valid UTF-8 without BOM', () => {
|
||||||
|
const buf = Buffer.from('Hello, 世界', 'utf-8')
|
||||||
|
expect(detectEncoding(buf)).toBe('utf-8')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects GBK encoded Chinese text', () => {
|
||||||
|
// "你好" in GBK: C4 E3 BA C3
|
||||||
|
const buf = Buffer.from([0xc4, 0xe3, 0xba, 0xc3])
|
||||||
|
expect(detectEncoding(buf)).toBe('gbk')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('returns utf-8 for empty buffer', () => {
|
||||||
|
const buf = Buffer.alloc(0)
|
||||||
|
expect(detectEncoding(buf)).toBe('utf-8')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('falls back to latin1 for random bytes', () => {
|
||||||
|
// Random bytes that aren't valid UTF-8 or GBK
|
||||||
|
const buf = Buffer.from([0x80, 0x81, 0x82, 0x83, 0x84, 0x85])
|
||||||
|
expect(detectEncoding(buf)).toBe('latin1')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('prioritizes BOM over content analysis', () => {
|
||||||
|
// UTF-8 BOM followed by bytes that could be confused
|
||||||
|
const buf = Buffer.from([0xef, 0xbb, 0xbf, 0x48, 0x65, 0x6c, 0x6c, 0x6f])
|
||||||
|
expect(detectEncoding(buf)).toBe('utf-8')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('decodeBuffer', () => {
|
||||||
|
test('decodes UTF-8 buffer correctly', () => {
|
||||||
|
const buf = Buffer.from('Hello, 世界', 'utf-8')
|
||||||
|
expect(decodeBuffer(buf, 'utf-8')).toBe('Hello, 世界')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('decodes GBK buffer correctly', () => {
|
||||||
|
// "你好" in GBK
|
||||||
|
const buf = Buffer.from([0xc4, 0xe3, 0xba, 0xc3])
|
||||||
|
expect(decodeBuffer(buf, 'gbk')).toBe('你好')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('decodes UTF-16LE buffer correctly', () => {
|
||||||
|
const buf = Buffer.from([
|
||||||
|
0x48, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00,
|
||||||
|
])
|
||||||
|
expect(decodeBuffer(buf, 'utf-16le')).toBe('Hello')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('decodes empty buffer', () => {
|
||||||
|
const buf = Buffer.alloc(0)
|
||||||
|
expect(decodeBuffer(buf, 'utf-8')).toBe('')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('encodeString', () => {
|
||||||
|
test('encodes UTF-8 string without conversion flag', () => {
|
||||||
|
const { buffer, converted } = encodeString('Hello 世界', 'utf-8')
|
||||||
|
expect(converted).toBe(false)
|
||||||
|
expect(buffer.toString('utf-8')).toBe('Hello 世界')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('encodes UTF-8 with utf8 alias', () => {
|
||||||
|
const { buffer, converted } = encodeString('test', 'utf8')
|
||||||
|
expect(converted).toBe(false)
|
||||||
|
expect(buffer.toString('utf-8')).toBe('test')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('encodes UTF-16LE string', () => {
|
||||||
|
const { buffer, converted } = encodeString('Hello', 'utf-16le')
|
||||||
|
expect(converted).toBe(false)
|
||||||
|
expect(decodeBuffer(buffer, 'utf-16le')).toBe('Hello')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('handles GBK encoding (may convert)', () => {
|
||||||
|
const { buffer, converted } = encodeString('你好', 'gbk')
|
||||||
|
expect(buffer).toBeInstanceOf(Buffer)
|
||||||
|
expect(typeof converted).toBe('boolean')
|
||||||
|
if (!converted) {
|
||||||
|
expect(decodeBuffer(buffer, 'gbk')).toBe('你好')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,10 +1,19 @@
|
|||||||
import { describe, expect, test } from 'bun:test'
|
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as path from 'path'
|
||||||
|
import { logMock } from '../../../tests/mocks/log'
|
||||||
|
import { debugMock } from '../../../tests/mocks/debug'
|
||||||
|
|
||||||
|
mock.module('src/utils/log.ts', logMock)
|
||||||
|
mock.module('src/utils/debug.ts', debugMock)
|
||||||
|
|
||||||
import {
|
import {
|
||||||
convertLeadingTabsToSpaces,
|
convertLeadingTabsToSpaces,
|
||||||
addLineNumbers,
|
addLineNumbers,
|
||||||
stripLineNumberPrefix,
|
stripLineNumberPrefix,
|
||||||
pathsEqual,
|
pathsEqual,
|
||||||
normalizePathForComparison,
|
normalizePathForComparison,
|
||||||
|
writeTextContent,
|
||||||
} from '../file'
|
} from '../file'
|
||||||
|
|
||||||
describe('convertLeadingTabsToSpaces', () => {
|
describe('convertLeadingTabsToSpaces', () => {
|
||||||
@@ -90,3 +99,50 @@ describe('pathsEqual', () => {
|
|||||||
expect(pathsEqual('/a/b', '/a/c')).toBe(false)
|
expect(pathsEqual('/a/b', '/a/c')).toBe(false)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('writeTextContent with multi-encoding', () => {
|
||||||
|
let tmpDir: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = fs.mkdtempSync(path.join('/tmp', 'writeTextContent-test-'))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
test('writes UTF-8 content correctly', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'utf8.txt')
|
||||||
|
writeTextContent(filePath, 'Hello 世界', 'utf-8', 'LF')
|
||||||
|
const content = fs.readFileSync(filePath, 'utf-8')
|
||||||
|
expect(content).toBe('Hello 世界')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('writes UTF-16LE content correctly', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'utf16le.txt')
|
||||||
|
writeTextContent(filePath, 'Hello', 'utf-16le', 'LF')
|
||||||
|
const buf = fs.readFileSync(filePath)
|
||||||
|
// Should start with BOM (0xFF 0xFE) followed by UTF-16LE data
|
||||||
|
// Note: Bun's Buffer.from('Hello', 'utf-16le') doesn't add BOM
|
||||||
|
const text = buf.toString('utf-16le')
|
||||||
|
expect(text).toBe('Hello')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('GBK write falls back to UTF-8', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk.txt')
|
||||||
|
writeTextContent(filePath, '测试写入', 'gbk', 'LF')
|
||||||
|
const content = fs.readFileSync(filePath, 'utf-8')
|
||||||
|
// Content should be readable (either GBK or UTF-8 fallback)
|
||||||
|
expect(content.length).toBeGreaterThan(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('CRLF line endings with GBK encoding', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk-crlf.txt')
|
||||||
|
writeTextContent(filePath, 'line1\nline2', 'gbk', 'CRLF')
|
||||||
|
const buf = fs.readFileSync(filePath)
|
||||||
|
const content = buf.toString('utf-8')
|
||||||
|
// Should have CRLF line endings
|
||||||
|
expect(content).toContain('\r\n')
|
||||||
|
expect(content).not.toContain('\n\r')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
107
src/utils/__tests__/fileRead.test.ts
Normal file
107
src/utils/__tests__/fileRead.test.ts
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as path from 'path'
|
||||||
|
import { logMock } from '../../../tests/mocks/log'
|
||||||
|
import { debugMock } from '../../../tests/mocks/debug'
|
||||||
|
|
||||||
|
mock.module('src/utils/log.ts', logMock)
|
||||||
|
mock.module('src/utils/debug.ts', debugMock)
|
||||||
|
|
||||||
|
import {
|
||||||
|
readFileSyncWithMetadata,
|
||||||
|
detectEncodingForResolvedPath,
|
||||||
|
} from '../fileRead'
|
||||||
|
|
||||||
|
describe('readFileSyncWithMetadata', () => {
|
||||||
|
let tmpDir: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = fs.mkdtempSync(path.join('/tmp', 'fileRead-test-'))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
test('reads UTF-8 file correctly', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'utf8.txt')
|
||||||
|
fs.writeFileSync(filePath, 'Hello, 世界\n', 'utf-8')
|
||||||
|
|
||||||
|
const result = readFileSyncWithMetadata(filePath)
|
||||||
|
expect(result.encoding).toBe('utf-8')
|
||||||
|
expect(result.content).toBe('Hello, 世界\n')
|
||||||
|
expect(result.lineEndings).toBe('LF')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('reads GBK encoded file correctly', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk.txt')
|
||||||
|
// "你好世界" in GBK encoding
|
||||||
|
const gbkBytes = Buffer.from([
|
||||||
|
0xc4, 0xe3, 0xba, 0xc3, 0xca, 0xc0, 0xbd, 0xe7,
|
||||||
|
])
|
||||||
|
fs.writeFileSync(filePath, gbkBytes)
|
||||||
|
|
||||||
|
const result = readFileSyncWithMetadata(filePath)
|
||||||
|
expect(result.encoding).toBe('gbk')
|
||||||
|
expect(result.content).toBe('你好世界')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('reads empty file with utf8 encoding', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'empty.txt')
|
||||||
|
fs.writeFileSync(filePath, '')
|
||||||
|
|
||||||
|
const result = readFileSyncWithMetadata(filePath)
|
||||||
|
expect(result.encoding).toBe('utf8')
|
||||||
|
expect(result.content).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('reads UTF-16LE BOM file correctly', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'utf16le.txt')
|
||||||
|
// BOM + "Hello" in UTF-16LE
|
||||||
|
const bom = Buffer.from([0xff, 0xfe])
|
||||||
|
const content = Buffer.from('Hello', 'utf-16le')
|
||||||
|
fs.writeFileSync(filePath, Buffer.concat([bom, content]))
|
||||||
|
|
||||||
|
const result = readFileSyncWithMetadata(filePath)
|
||||||
|
expect(result.encoding).toBe('utf-16le')
|
||||||
|
expect(result.content).toBe('Hello')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('normalizes CRLF to LF', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'crlf.txt')
|
||||||
|
fs.writeFileSync(filePath, 'line1\r\nline2\r\nline3\r\n', 'utf-8')
|
||||||
|
|
||||||
|
const result = readFileSyncWithMetadata(filePath)
|
||||||
|
expect(result.content).toBe('line1\nline2\nline3\n')
|
||||||
|
expect(result.lineEndings).toBe('CRLF')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('detectEncodingForResolvedPath', () => {
|
||||||
|
let tmpDir: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = fs.mkdtempSync(path.join('/tmp', 'fileRead-detect-test-'))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
test('returns utf8 for empty file', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'empty.txt')
|
||||||
|
fs.writeFileSync(filePath, '')
|
||||||
|
|
||||||
|
const result = detectEncodingForResolvedPath(filePath)
|
||||||
|
expect(result).toBe('utf8')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('detects GBK encoding from file', () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk.txt')
|
||||||
|
const gbkBytes = Buffer.from([0xc4, 0xe3, 0xba, 0xc3])
|
||||||
|
fs.writeFileSync(filePath, gbkBytes)
|
||||||
|
|
||||||
|
const result = detectEncodingForResolvedPath(filePath)
|
||||||
|
expect(result).toBe('gbk')
|
||||||
|
})
|
||||||
|
})
|
||||||
87
src/utils/__tests__/readFileInRange.test.ts
Normal file
87
src/utils/__tests__/readFileInRange.test.ts
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as path from 'path'
|
||||||
|
import { readFileInRange } from '../readFileInRange'
|
||||||
|
|
||||||
|
describe('readFileInRange', () => {
|
||||||
|
let tmpDir: string
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = fs.mkdtempSync(path.join('/tmp', 'readFileInRange-test-'))
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
test('fast path — UTF-8 file', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'utf8.txt')
|
||||||
|
fs.writeFileSync(filePath, 'Hello 世界\nLine 2\nLine 3\n', 'utf-8')
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 0)
|
||||||
|
expect(result.content).toBe('Hello 世界\nLine 2\nLine 3\n')
|
||||||
|
expect(result.lineCount).toBe(4)
|
||||||
|
expect(result.totalLines).toBe(4)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('fast path — GBK file', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk.txt')
|
||||||
|
// "你好世界" in GBK + newline
|
||||||
|
const gbkBytes = Buffer.from([
|
||||||
|
0xc4, 0xe3, 0xba, 0xc3, 0xca, 0xc0, 0xbd, 0xe7, 0x0a,
|
||||||
|
])
|
||||||
|
fs.writeFileSync(filePath, gbkBytes)
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 0)
|
||||||
|
expect(result.content).toBe('你好世界\n')
|
||||||
|
expect(result.totalBytes).toBe(13) // UTF-8 byte length of "你好世界\n"
|
||||||
|
})
|
||||||
|
|
||||||
|
test('fast path — line range on GBK file', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'gbk-lines.txt')
|
||||||
|
// Three lines in GBK: "第一行\n第二行\n第三行\n"
|
||||||
|
const line1 = Buffer.from([0xb5, 0xda, 0xd2, 0xbb, 0xd0, 0xd0]) // 第一行
|
||||||
|
const line2 = Buffer.from([0xb5, 0xda, 0xb6, 0xfe, 0xd0, 0xd0]) // 第二行
|
||||||
|
const line3 = Buffer.from([0xb5, 0xda, 0xc8, 0xfd, 0xd0, 0xd0]) // 第三行
|
||||||
|
const content = Buffer.concat([
|
||||||
|
line1,
|
||||||
|
Buffer.from([0x0a]),
|
||||||
|
line2,
|
||||||
|
Buffer.from([0x0a]),
|
||||||
|
line3,
|
||||||
|
Buffer.from([0x0a]),
|
||||||
|
])
|
||||||
|
fs.writeFileSync(filePath, content)
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 1, 1)
|
||||||
|
expect(result.content).toBe('第二行')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('BOM stripping', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'bom.txt')
|
||||||
|
const bom = Buffer.from([0xef, 0xbb, 0xbf])
|
||||||
|
fs.writeFileSync(filePath, Buffer.concat([bom, Buffer.from('Hello\n')]))
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 0)
|
||||||
|
expect(result.content).toBe('Hello\n')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('empty file', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'empty.txt')
|
||||||
|
fs.writeFileSync(filePath, '')
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 0)
|
||||||
|
expect(result.content).toBe('')
|
||||||
|
expect(result.totalLines).toBe(1)
|
||||||
|
expect(result.totalBytes).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('fast path — offset and maxLines', async () => {
|
||||||
|
const filePath = path.join(tmpDir, 'lines.txt')
|
||||||
|
fs.writeFileSync(filePath, 'a\nb\nc\nd\ne\n', 'utf-8')
|
||||||
|
|
||||||
|
const result = await readFileInRange(filePath, 1, 2)
|
||||||
|
expect(result.content).toBe('b\nc')
|
||||||
|
expect(result.lineCount).toBe(2)
|
||||||
|
})
|
||||||
|
})
|
||||||
90
src/utils/encoding.ts
Normal file
90
src/utils/encoding.ts
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
/**
|
||||||
|
* Encoding detection and conversion utilities for file I/O.
|
||||||
|
*
|
||||||
|
* Provides three-layer encoding detection (BOM → UTF-8 fatal → GBK fallback)
|
||||||
|
* and Buffer/string conversion functions. Zero external dependencies — uses only
|
||||||
|
* TextDecoder/TextEncoder APIs available in Bun/Node.js.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Extended encoding type covering non-UTF-8 encodings used in CJK files */
|
||||||
|
export type FileEncoding = BufferEncoding | 'gbk'
|
||||||
|
|
||||||
|
/** Encoding name accepted by TextDecoder (string), broader than FileEncoding */
|
||||||
|
export type DetectedEncoding = string
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect the encoding of a buffer using three-layer detection:
|
||||||
|
* 1. BOM (Byte Order Mark) detection
|
||||||
|
* 2. UTF-8 fatal validation
|
||||||
|
* 3. GBK fallback (most common non-UTF-8 CJK encoding)
|
||||||
|
*/
|
||||||
|
export function detectEncoding(buffer: Buffer): FileEncoding {
|
||||||
|
// Layer 1: BOM detection
|
||||||
|
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
|
||||||
|
return 'utf-16le'
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
buffer.length >= 3 &&
|
||||||
|
buffer[0] === 0xef &&
|
||||||
|
buffer[1] === 0xbb &&
|
||||||
|
buffer[2] === 0xbf
|
||||||
|
) {
|
||||||
|
return 'utf-8'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer 2: UTF-8 fatal validation
|
||||||
|
try {
|
||||||
|
new TextDecoder('utf-8', { fatal: true }).decode(buffer)
|
||||||
|
return 'utf-8'
|
||||||
|
} catch {
|
||||||
|
// Not valid UTF-8, proceed to Layer 3
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer 3: GBK fallback
|
||||||
|
try {
|
||||||
|
new TextDecoder('gbk', { fatal: true }).decode(buffer)
|
||||||
|
return 'gbk'
|
||||||
|
} catch {
|
||||||
|
// Not valid GBK, fall back to latin1 (single-byte, always succeeds)
|
||||||
|
return 'latin1'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode a buffer using the specified encoding.
|
||||||
|
* Unified decoding entry point for all file read paths.
|
||||||
|
*/
|
||||||
|
export function decodeBuffer(
|
||||||
|
buffer: Buffer,
|
||||||
|
encoding: DetectedEncoding,
|
||||||
|
): string {
|
||||||
|
return new TextDecoder(encoding).decode(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encode a string to a Buffer using the specified encoding.
|
||||||
|
* For non-standard encodings, falls back to UTF-8 if the runtime
|
||||||
|
* doesn't support the encoding in Buffer.from.
|
||||||
|
*
|
||||||
|
* @returns buffer - the encoded bytes, converted - true if encoding was
|
||||||
|
* fallbacked to UTF-8 (caller should warn the user)
|
||||||
|
*/
|
||||||
|
export function encodeString(
|
||||||
|
content: string,
|
||||||
|
encoding: DetectedEncoding,
|
||||||
|
): { buffer: Buffer; converted: boolean } {
|
||||||
|
if (encoding === 'utf-8' || encoding === 'utf8') {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-8'), converted: false }
|
||||||
|
}
|
||||||
|
if (encoding === 'utf-16le') {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-16le'), converted: false }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Other encodings (e.g. gbk): try Buffer.from, fall back to UTF-8
|
||||||
|
try {
|
||||||
|
const buf = Buffer.from(content, encoding as BufferEncoding)
|
||||||
|
return { buffer: buf, converted: false }
|
||||||
|
} catch {
|
||||||
|
return { buffer: Buffer.from(content, 'utf-8'), converted: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,7 @@ import {
|
|||||||
detectLineEndingsForString,
|
detectLineEndingsForString,
|
||||||
type LineEndingType,
|
type LineEndingType,
|
||||||
} from './fileRead.js'
|
} from './fileRead.js'
|
||||||
|
import { type FileEncoding, decodeBuffer, encodeString } from './encoding.js'
|
||||||
import { fileReadCache } from './fileReadCache.js'
|
import { fileReadCache } from './fileReadCache.js'
|
||||||
import { getFsImplementation, safeResolvePath } from './fsOperations.js'
|
import { getFsImplementation, safeResolvePath } from './fsOperations.js'
|
||||||
import { logError } from './log.js'
|
import { logError } from './log.js'
|
||||||
@@ -84,7 +85,7 @@ export async function getFileModificationTimeAsync(
|
|||||||
export function writeTextContent(
|
export function writeTextContent(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
content: string,
|
content: string,
|
||||||
encoding: BufferEncoding,
|
encoding: FileEncoding,
|
||||||
endings: LineEndingType,
|
endings: LineEndingType,
|
||||||
): void {
|
): void {
|
||||||
let toWrite = content
|
let toWrite = content
|
||||||
@@ -94,10 +95,38 @@ export function writeTextContent(
|
|||||||
toWrite = content.replaceAll('\r\n', '\n').split('\n').join('\r\n')
|
toWrite = content.replaceAll('\r\n', '\n').split('\n').join('\r\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, { encoding })
|
// Check if encoding is directly supported by Node.js fs
|
||||||
|
const BUFFER_ENCODINGS = new Set<string>([
|
||||||
|
'utf8',
|
||||||
|
'utf-8',
|
||||||
|
'utf16le',
|
||||||
|
'ucs2',
|
||||||
|
'ucs-2',
|
||||||
|
'ascii',
|
||||||
|
'latin1',
|
||||||
|
'binary',
|
||||||
|
'base64',
|
||||||
|
'hex',
|
||||||
|
])
|
||||||
|
|
||||||
|
if (BUFFER_ENCODINGS.has(encoding)) {
|
||||||
|
writeFileSyncAndFlush_DEPRECATED(filePath, toWrite, {
|
||||||
|
encoding: encoding as BufferEncoding,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
// Non-BufferEncoding (e.g. gbk): use encodeString to get Buffer
|
||||||
|
const { buffer, converted } = encodeString(toWrite, encoding)
|
||||||
|
writeFileSyncAndFlush_DEPRECATED(filePath, buffer, { buffer })
|
||||||
|
if (converted) {
|
||||||
|
logForDebugging(
|
||||||
|
`writeTextContent: encoding '${encoding}' unsupported for write, fell back to UTF-8 for ${filePath}`,
|
||||||
|
{ level: 'warn' },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function detectFileEncoding(filePath: string): BufferEncoding {
|
export function detectFileEncoding(filePath: string): FileEncoding {
|
||||||
try {
|
try {
|
||||||
const fs = getFsImplementation()
|
const fs = getFsImplementation()
|
||||||
const { resolvedPath } = safeResolvePath(fs, filePath)
|
const { resolvedPath } = safeResolvePath(fs, filePath)
|
||||||
@@ -119,14 +148,14 @@ export function detectFileEncoding(filePath: string): BufferEncoding {
|
|||||||
|
|
||||||
export function detectLineEndings(
|
export function detectLineEndings(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
encoding: BufferEncoding = 'utf8',
|
encoding: FileEncoding = 'utf8',
|
||||||
): LineEndingType {
|
): LineEndingType {
|
||||||
try {
|
try {
|
||||||
const fs = getFsImplementation()
|
const fs = getFsImplementation()
|
||||||
const { resolvedPath } = safeResolvePath(fs, filePath)
|
const { resolvedPath } = safeResolvePath(fs, filePath)
|
||||||
const { buffer, bytesRead } = fs.readSync(resolvedPath, { length: 4096 })
|
const { buffer, bytesRead } = fs.readSync(resolvedPath, { length: 4096 })
|
||||||
|
|
||||||
const content = buffer.toString(encoding, 0, bytesRead)
|
const content = decodeBuffer(buffer.subarray(0, bytesRead), encoding)
|
||||||
return detectLineEndingsForString(content)
|
return detectLineEndingsForString(content)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logError(error)
|
logError(error)
|
||||||
@@ -361,8 +390,10 @@ export function readFileSyncCached(filePath: string): string {
|
|||||||
*/
|
*/
|
||||||
export function writeFileSyncAndFlush_DEPRECATED(
|
export function writeFileSyncAndFlush_DEPRECATED(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
content: string,
|
content: string | Buffer,
|
||||||
options: { encoding: BufferEncoding; mode?: number } = { encoding: 'utf-8' },
|
options: { encoding?: BufferEncoding; mode?: number; buffer?: Buffer } = {
|
||||||
|
encoding: 'utf-8',
|
||||||
|
},
|
||||||
): void {
|
): void {
|
||||||
const fs = getFsImplementation()
|
const fs = getFsImplementation()
|
||||||
|
|
||||||
@@ -403,26 +434,30 @@ export function writeFileSyncAndFlush_DEPRECATED(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Determine write mode before try/catch so both paths can use it
|
||||||
|
const isBufferWrite = Buffer.isBuffer(content) || options.buffer !== undefined
|
||||||
|
const writeData = options.buffer ?? content
|
||||||
|
|
||||||
try {
|
try {
|
||||||
logForDebugging(`Writing to temp file: ${tempPath}`)
|
logForDebugging(`Writing to temp file: ${tempPath}`)
|
||||||
|
|
||||||
// Write to temp file with flush and mode (if specified for new file)
|
// Write to temp file with flush and mode (if specified for new file)
|
||||||
const writeOptions: {
|
const writeOptions: {
|
||||||
encoding: BufferEncoding
|
encoding?: BufferEncoding
|
||||||
flush: boolean
|
flush: boolean
|
||||||
mode?: number
|
mode?: number
|
||||||
} = {
|
} = {
|
||||||
encoding: options.encoding,
|
|
||||||
flush: true,
|
flush: true,
|
||||||
|
...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }),
|
||||||
}
|
}
|
||||||
// Only set mode in writeFileSync for new files to ensure atomic permission setting
|
// Only set mode in writeFileSync for new files to ensure atomic permission setting
|
||||||
if (!targetExists && options.mode !== undefined) {
|
if (!targetExists && options.mode !== undefined) {
|
||||||
writeOptions.mode = options.mode
|
writeOptions.mode = options.mode
|
||||||
}
|
}
|
||||||
|
|
||||||
fsWriteFileSync(tempPath, content, writeOptions)
|
fsWriteFileSync(tempPath, writeData, writeOptions)
|
||||||
logForDebugging(
|
logForDebugging(
|
||||||
`Temp file written successfully, size: ${content.length} bytes`,
|
`Temp file written successfully, size: ${typeof writeData === 'string' ? writeData.length : writeData.byteLength} bytes`,
|
||||||
)
|
)
|
||||||
|
|
||||||
// For existing files or if mode was not set atomically, apply permissions
|
// For existing files or if mode was not set atomically, apply permissions
|
||||||
@@ -454,19 +489,19 @@ export function writeFileSyncAndFlush_DEPRECATED(
|
|||||||
logForDebugging(`Falling back to non-atomic write for ${targetPath}`)
|
logForDebugging(`Falling back to non-atomic write for ${targetPath}`)
|
||||||
try {
|
try {
|
||||||
const fallbackOptions: {
|
const fallbackOptions: {
|
||||||
encoding: BufferEncoding
|
encoding?: BufferEncoding
|
||||||
flush: boolean
|
flush: boolean
|
||||||
mode?: number
|
mode?: number
|
||||||
} = {
|
} = {
|
||||||
encoding: options.encoding,
|
|
||||||
flush: true,
|
flush: true,
|
||||||
|
...(isBufferWrite ? {} : { encoding: options.encoding ?? 'utf-8' }),
|
||||||
}
|
}
|
||||||
// Only set mode for new files
|
// Only set mode for new files
|
||||||
if (!targetExists && options.mode !== undefined) {
|
if (!targetExists && options.mode !== undefined) {
|
||||||
fallbackOptions.mode = options.mode
|
fallbackOptions.mode = options.mode
|
||||||
}
|
}
|
||||||
|
|
||||||
fsWriteFileSync(targetPath, content, fallbackOptions)
|
fsWriteFileSync(targetPath, writeData, fallbackOptions)
|
||||||
logForDebugging(
|
logForDebugging(
|
||||||
`File ${targetPath} written successfully with non-atomic fallback`,
|
`File ${targetPath} written successfully with non-atomic fallback`,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -13,39 +13,24 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { logForDebugging } from './debug.js'
|
import { logForDebugging } from './debug.js'
|
||||||
|
import { type FileEncoding, decodeBuffer, detectEncoding } from './encoding.js'
|
||||||
import { getFsImplementation, safeResolvePath } from './fsOperations.js'
|
import { getFsImplementation, safeResolvePath } from './fsOperations.js'
|
||||||
|
|
||||||
export type LineEndingType = 'CRLF' | 'LF'
|
export type LineEndingType = 'CRLF' | 'LF'
|
||||||
|
|
||||||
export function detectEncodingForResolvedPath(
|
export function detectEncodingForResolvedPath(
|
||||||
resolvedPath: string,
|
resolvedPath: string,
|
||||||
): BufferEncoding {
|
): FileEncoding {
|
||||||
const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, {
|
const { buffer, bytesRead } = getFsImplementation().readSync(resolvedPath, {
|
||||||
length: 4096,
|
length: 4096,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Empty files should default to utf8, not ascii
|
// Empty files default to utf8 — nothing to detect
|
||||||
// This fixes a bug where writing emojis/CJK to empty files caused corruption
|
|
||||||
if (bytesRead === 0) {
|
if (bytesRead === 0) {
|
||||||
return 'utf8'
|
return 'utf8'
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bytesRead >= 2) {
|
return detectEncoding(buffer.subarray(0, bytesRead))
|
||||||
if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf16le'
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
bytesRead >= 3 &&
|
|
||||||
buffer[0] === 0xef &&
|
|
||||||
buffer[1] === 0xbb &&
|
|
||||||
buffer[2] === 0xbf
|
|
||||||
) {
|
|
||||||
return 'utf8'
|
|
||||||
}
|
|
||||||
|
|
||||||
// For non-empty files, default to utf8 since it's a superset of ascii
|
|
||||||
// and handles all Unicode characters properly
|
|
||||||
return 'utf8'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function detectLineEndingsForString(content: string): LineEndingType {
|
export function detectLineEndingsForString(content: string): LineEndingType {
|
||||||
@@ -74,7 +59,7 @@ export function detectLineEndingsForString(content: string): LineEndingType {
|
|||||||
*/
|
*/
|
||||||
export function readFileSyncWithMetadata(filePath: string): {
|
export function readFileSyncWithMetadata(filePath: string): {
|
||||||
content: string
|
content: string
|
||||||
encoding: BufferEncoding
|
encoding: FileEncoding
|
||||||
lineEndings: LineEndingType
|
lineEndings: LineEndingType
|
||||||
} {
|
} {
|
||||||
const fs = getFsImplementation()
|
const fs = getFsImplementation()
|
||||||
@@ -85,10 +70,10 @@ export function readFileSyncWithMetadata(filePath: string): {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const encoding = detectEncodingForResolvedPath(resolvedPath)
|
const encoding = detectEncodingForResolvedPath(resolvedPath)
|
||||||
const raw = fs.readFileSync(resolvedPath, { encoding })
|
// Read raw Buffer first — readFileSync encoding option only accepts
|
||||||
// Detect line endings from the raw head before CRLF normalization erases
|
// BufferEncoding, not gbk etc.
|
||||||
// the distinction. 4096 code units is ≥ detectLineEndings's 4096-byte
|
const rawBuffer = fs.readFileBytesSync(resolvedPath)
|
||||||
// readSync sample (line endings are ASCII, so the unit mismatch is moot).
|
const raw = decodeBuffer(rawBuffer, encoding)
|
||||||
const lineEndings = detectLineEndingsForString(raw.slice(0, 4096))
|
const lineEndings = detectLineEndingsForString(raw.slice(0, 4096))
|
||||||
return {
|
return {
|
||||||
content: raw.replaceAll('\r\n', '\n'),
|
content: raw.replaceAll('\r\n', '\n'),
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
import { detectFileEncoding } from './file.js'
|
import { detectFileEncoding } from './file.js'
|
||||||
|
import { type FileEncoding, decodeBuffer } from './encoding.js'
|
||||||
import { getFsImplementation } from './fsOperations.js'
|
import { getFsImplementation } from './fsOperations.js'
|
||||||
|
|
||||||
type CachedFileData = {
|
type CachedFileData = {
|
||||||
content: string
|
content: string
|
||||||
encoding: BufferEncoding
|
encoding: FileEncoding
|
||||||
mtime: number
|
mtime: number
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -19,7 +20,7 @@ class FileReadCache {
|
|||||||
* Reads a file with caching. Returns both content and encoding.
|
* Reads a file with caching. Returns both content and encoding.
|
||||||
* Cache key includes file path and modification time for automatic invalidation.
|
* Cache key includes file path and modification time for automatic invalidation.
|
||||||
*/
|
*/
|
||||||
readFile(filePath: string): { content: string; encoding: BufferEncoding } {
|
readFile(filePath: string): { content: string; encoding: FileEncoding } {
|
||||||
const fs = getFsImplementation()
|
const fs = getFsImplementation()
|
||||||
|
|
||||||
// Get file stats for cache invalidation
|
// Get file stats for cache invalidation
|
||||||
@@ -45,9 +46,8 @@ class FileReadCache {
|
|||||||
|
|
||||||
// Cache miss or stale data - read the file
|
// Cache miss or stale data - read the file
|
||||||
const encoding = detectFileEncoding(filePath)
|
const encoding = detectFileEncoding(filePath)
|
||||||
const content = fs
|
const rawBuffer = fs.readFileBytesSync(filePath)
|
||||||
.readFileSync(filePath, { encoding })
|
const content = decodeBuffer(rawBuffer, encoding).replaceAll('\r\n', '\n')
|
||||||
.replaceAll('\r\n', '\n')
|
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
this.cache.set(cacheKey, {
|
this.cache.set(cacheKey, {
|
||||||
|
|||||||
@@ -26,7 +26,8 @@
|
|||||||
// On error (including maxBytes exceeded), stream.destroy(err) emits
|
// On error (including maxBytes exceeded), stream.destroy(err) emits
|
||||||
// 'error' → reject (passed directly to .once('error')).
|
// 'error' → reject (passed directly to .once('error')).
|
||||||
//
|
//
|
||||||
// Both paths strip UTF-8 BOM and \r (CRLF → LF).
|
// Both paths auto-detect encoding via encoding.ts (BOM → UTF-8 fatal → fallback chain),
|
||||||
|
// decode with TextDecoder, and strip BOM and \r (CRLF → LF).
|
||||||
//
|
//
|
||||||
// mtime comes from fstat/stat on the already-open fd — no extra open().
|
// mtime comes from fstat/stat on the already-open fd — no extra open().
|
||||||
//
|
//
|
||||||
@@ -39,6 +40,7 @@
|
|||||||
|
|
||||||
import { createReadStream, fstat } from 'fs'
|
import { createReadStream, fstat } from 'fs'
|
||||||
import { stat as fsStat, readFile } from 'fs/promises'
|
import { stat as fsStat, readFile } from 'fs/promises'
|
||||||
|
import { detectEncoding, decodeBuffer } from './encoding.js'
|
||||||
import { formatFileSize } from './format.js'
|
import { formatFileSize } from './format.js'
|
||||||
|
|
||||||
const FAST_PATH_MAX_SIZE = 10 * 1024 * 1024 // 10 MB
|
const FAST_PATH_MAX_SIZE = 10 * 1024 * 1024 // 10 MB
|
||||||
@@ -115,7 +117,9 @@ export async function readFileInRange(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf8', signal })
|
const rawBuffer = await readFile(filePath, { signal })
|
||||||
|
const encoding = detectEncoding(rawBuffer)
|
||||||
|
const text = decodeBuffer(rawBuffer, encoding)
|
||||||
return readFileInRangeFast(
|
return readFileInRangeFast(
|
||||||
text,
|
text,
|
||||||
stats.mtimeMs,
|
stats.mtimeMs,
|
||||||
@@ -227,6 +231,12 @@ type StreamState = {
|
|||||||
isFirstChunk: boolean
|
isFirstChunk: boolean
|
||||||
resolveMtime: (ms: number) => void
|
resolveMtime: (ms: number) => void
|
||||||
mtimeReady: Promise<number>
|
mtimeReady: Promise<number>
|
||||||
|
/** Encoding detection state: null = not yet detected, string = detected */
|
||||||
|
encoding: string | null
|
||||||
|
/** TextDecoder instance: created after detection, used for streaming decode */
|
||||||
|
decoder: TextDecoder | null
|
||||||
|
/** Detection phase buffer: collects raw bytes until 4KB or stream end */
|
||||||
|
detectionBuffer: number[]
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamOnOpen(this: StreamState, fd: number): void {
|
function streamOnOpen(this: StreamState, fd: number): void {
|
||||||
@@ -235,15 +245,71 @@ function streamOnOpen(this: StreamState, fd: number): void {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamOnData(this: StreamState, chunk: string): void {
|
function processTextChunk(state: StreamState, text: string): void {
|
||||||
if (this.isFirstChunk) {
|
// BOM stripping (first chunk only)
|
||||||
this.isFirstChunk = false
|
if (state.isFirstChunk) {
|
||||||
if (chunk.charCodeAt(0) === 0xfeff) {
|
state.isFirstChunk = false
|
||||||
chunk = chunk.slice(1)
|
if (text.charCodeAt(0) === 0xfeff) {
|
||||||
|
text = text.slice(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.totalBytesRead += Buffer.byteLength(chunk)
|
const data = state.partial.length > 0 ? state.partial + text : text
|
||||||
|
state.partial = ''
|
||||||
|
|
||||||
|
let startPos = 0
|
||||||
|
let newlinePos: number
|
||||||
|
while ((newlinePos = data.indexOf('\n', startPos)) !== -1) {
|
||||||
|
if (
|
||||||
|
state.currentLineIndex >= state.offset &&
|
||||||
|
state.currentLineIndex < state.endLine
|
||||||
|
) {
|
||||||
|
let line = data.slice(startPos, newlinePos)
|
||||||
|
if (line.endsWith('\r')) {
|
||||||
|
line = line.slice(0, -1)
|
||||||
|
}
|
||||||
|
if (state.truncateOnByteLimit && state.maxBytes !== undefined) {
|
||||||
|
const sep = state.selectedLines.length > 0 ? 1 : 0
|
||||||
|
const nextBytes = state.selectedBytes + sep + Buffer.byteLength(line)
|
||||||
|
if (nextBytes > state.maxBytes) {
|
||||||
|
state.truncatedByBytes = true
|
||||||
|
state.endLine = state.currentLineIndex
|
||||||
|
} else {
|
||||||
|
state.selectedBytes = nextBytes
|
||||||
|
state.selectedLines.push(line)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state.selectedLines.push(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state.currentLineIndex++
|
||||||
|
startPos = newlinePos + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if (startPos < data.length) {
|
||||||
|
if (
|
||||||
|
state.currentLineIndex >= state.offset &&
|
||||||
|
state.currentLineIndex < state.endLine
|
||||||
|
) {
|
||||||
|
const fragment = data.slice(startPos)
|
||||||
|
if (state.truncateOnByteLimit && state.maxBytes !== undefined) {
|
||||||
|
const sep = state.selectedLines.length > 0 ? 1 : 0
|
||||||
|
const fragBytes =
|
||||||
|
state.selectedBytes + sep + Buffer.byteLength(fragment)
|
||||||
|
if (fragBytes > state.maxBytes) {
|
||||||
|
state.truncatedByBytes = true
|
||||||
|
state.endLine = state.currentLineIndex
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state.partial = fragment
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function streamOnData(this: StreamState, chunk: Buffer): void {
|
||||||
|
this.totalBytesRead += chunk.length
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!this.truncateOnByteLimit &&
|
!this.truncateOnByteLimit &&
|
||||||
this.maxBytes !== undefined &&
|
this.maxBytes !== undefined &&
|
||||||
@@ -255,69 +321,47 @@ function streamOnData(this: StreamState, chunk: string): void {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = this.partial.length > 0 ? this.partial + chunk : chunk
|
// Phase 1: Encoding detection
|
||||||
this.partial = ''
|
if (this.encoding === null) {
|
||||||
|
for (let i = 0; i < chunk.length; i++) {
|
||||||
let startPos = 0
|
this.detectionBuffer.push(chunk[i])
|
||||||
let newlinePos: number
|
|
||||||
while ((newlinePos = data.indexOf('\n', startPos)) !== -1) {
|
|
||||||
if (
|
|
||||||
this.currentLineIndex >= this.offset &&
|
|
||||||
this.currentLineIndex < this.endLine
|
|
||||||
) {
|
|
||||||
let line = data.slice(startPos, newlinePos)
|
|
||||||
if (line.endsWith('\r')) {
|
|
||||||
line = line.slice(0, -1)
|
|
||||||
}
|
|
||||||
if (this.truncateOnByteLimit && this.maxBytes !== undefined) {
|
|
||||||
const sep = this.selectedLines.length > 0 ? 1 : 0
|
|
||||||
const nextBytes = this.selectedBytes + sep + Buffer.byteLength(line)
|
|
||||||
if (nextBytes > this.maxBytes) {
|
|
||||||
// Cap hit — collapse the selection range so nothing more is
|
|
||||||
// accumulated. Stream continues (to count totalLines).
|
|
||||||
this.truncatedByBytes = true
|
|
||||||
this.endLine = this.currentLineIndex
|
|
||||||
} else {
|
|
||||||
this.selectedBytes = nextBytes
|
|
||||||
this.selectedLines.push(line)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.selectedLines.push(line)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
this.currentLineIndex++
|
|
||||||
startPos = newlinePos + 1
|
// Collected at least 4KB, perform encoding detection
|
||||||
|
if (this.detectionBuffer.length >= 4096) {
|
||||||
|
this.encoding = detectEncoding(Buffer.from(this.detectionBuffer))
|
||||||
|
this.decoder = new TextDecoder(this.encoding, {
|
||||||
|
stream: true,
|
||||||
|
} as TextDecoderOptions)
|
||||||
|
|
||||||
|
// Decode the detection buffer and feed to line scanning
|
||||||
|
const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer))
|
||||||
|
this.detectionBuffer = []
|
||||||
|
processTextChunk(this, decoded)
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only keep the trailing fragment when inside the selected range.
|
// Phase 2: Decoding
|
||||||
// Outside the range we just count newlines — discarding prevents
|
const decoded = this.decoder!.decode(chunk, {
|
||||||
// unbounded memory growth on huge single-line files.
|
stream: true,
|
||||||
if (startPos < data.length) {
|
} as unknown as TextDecodeOptions)
|
||||||
if (
|
processTextChunk(this, decoded)
|
||||||
this.currentLineIndex >= this.offset &&
|
|
||||||
this.currentLineIndex < this.endLine
|
|
||||||
) {
|
|
||||||
const fragment = data.slice(startPos)
|
|
||||||
// In truncate mode, `partial` can grow unboundedly if the selected
|
|
||||||
// range contains a huge single line (no newline across many chunks).
|
|
||||||
// Once the fragment alone would overflow the remaining budget, we know
|
|
||||||
// the completed line can never fit — set truncated, collapse the
|
|
||||||
// selection range, and discard the fragment to stop accumulation.
|
|
||||||
if (this.truncateOnByteLimit && this.maxBytes !== undefined) {
|
|
||||||
const sep = this.selectedLines.length > 0 ? 1 : 0
|
|
||||||
const fragBytes = this.selectedBytes + sep + Buffer.byteLength(fragment)
|
|
||||||
if (fragBytes > this.maxBytes) {
|
|
||||||
this.truncatedByBytes = true
|
|
||||||
this.endLine = this.currentLineIndex
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.partial = fragment
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamOnEnd(this: StreamState): void {
|
function streamOnEnd(this: StreamState): void {
|
||||||
|
// If stream ended before detection completed (< 4KB file), detect now
|
||||||
|
if (this.encoding === null) {
|
||||||
|
this.encoding = detectEncoding(Buffer.from(this.detectionBuffer))
|
||||||
|
this.decoder = new TextDecoder(this.encoding, {
|
||||||
|
stream: true,
|
||||||
|
} as TextDecoderOptions)
|
||||||
|
const decoded = this.decoder.decode(Buffer.from(this.detectionBuffer))
|
||||||
|
this.detectionBuffer = []
|
||||||
|
processTextChunk(this, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle final fragment
|
||||||
let line = this.partial
|
let line = this.partial
|
||||||
if (line.endsWith('\r')) {
|
if (line.endsWith('\r')) {
|
||||||
line = line.slice(0, -1)
|
line = line.slice(0, -1)
|
||||||
@@ -366,7 +410,6 @@ function readFileInRangeStreaming(
|
|||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const state: StreamState = {
|
const state: StreamState = {
|
||||||
stream: createReadStream(filePath, {
|
stream: createReadStream(filePath, {
|
||||||
encoding: 'utf8',
|
|
||||||
highWaterMark: 512 * 1024,
|
highWaterMark: 512 * 1024,
|
||||||
...(signal ? { signal } : undefined),
|
...(signal ? { signal } : undefined),
|
||||||
}),
|
}),
|
||||||
@@ -384,6 +427,9 @@ function readFileInRangeStreaming(
|
|||||||
isFirstChunk: true,
|
isFirstChunk: true,
|
||||||
resolveMtime: () => {},
|
resolveMtime: () => {},
|
||||||
mtimeReady: null as unknown as Promise<number>,
|
mtimeReady: null as unknown as Promise<number>,
|
||||||
|
encoding: null,
|
||||||
|
decoder: null,
|
||||||
|
detectionBuffer: [],
|
||||||
}
|
}
|
||||||
state.mtimeReady = new Promise<number>(r => {
|
state.mtimeReady = new Promise<number>(r => {
|
||||||
state.resolveMtime = r
|
state.resolveMtime = r
|
||||||
|
|||||||
Reference in New Issue
Block a user