From 1f1becfb423a9fe08da8458bbb5c520a2db8da95 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 20 Oct 2026 00:39:42 +0800 Subject: [PATCH] fix(toml): only normalize smart quotes used as string delimiters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalizeTomlText replaced every curly/fullwidth quote in the text before TOML parsing, including quotes inside already-valid strings. MCP import and validation (tomlUtils, mcpDraft) and the Codex config editor therefore silently rewrote values such as args = ["don’t"] to "don't", and rejected valid configs like args = ["say“hello”"] or args = ['don’t'] because the inner quotes became delimiters. Scan the text instead: smart quotes outside strings are delimiters and become ASCII; strings opened by an ASCII quote keep smart quotes as content; strings opened by a smart quote may close with either form; multi-line strings and comments are handled. To keep the Chinese IME case working (the editor normalizes each keystroke, so the left quote is already ASCII when the right smart quote is typed), an ASCII-opened string left unterminated at end of line is re-scanned with smart closers, and if the strict result still fails to parse, a lenient scan is used only when it parses. Escaped smart double quotes in basic strings become \". Text without smart quotes is returned unchanged. Refs #8559 --- src/utils/textNormalization.ts | 139 +++++++++++++++++++++++++- tests/utils/textNormalization.test.ts | 96 ++++++++++++++++++ 2 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 tests/utils/textNormalization.test.ts diff --git a/src/utils/textNormalization.ts b/src/utils/textNormalization.ts index 8064fae0b..4c5600b8c 100644 --- a/src/utils/textNormalization.ts +++ b/src/utils/textNormalization.ts @@ -1,3 +1,5 @@ +import { parse as parseToml } from "smol-toml"; + /** * 将常见的中文/全角/弯引号统一为 ASCII 引号，以避免 TOML 解析失败。 * - 双引号：” “ „ ‟ ＂ → " @@ -15,8 +17,139 @@ export const normalizeQuotes = (text: string): string => { ); }; +const SMART_DOUBLE_QUOTES = new Set(["“", "”", "„", "‟", "＂"]); +const SMART_SINGLE_QUOTES = new Set(["‘", "’", "＇"]); + +/** 返回字符所属的引号族（ASCII 引号或弯/全角引号），非引号返回 null */ +const quoteFamily = (ch: string): '"' | "'" | null => { + if (ch === '"' || SMART_DOUBLE_QUOTES.has(ch)) return '"'; + if (ch === "'" || SMART_SINGLE_QUOTES.has(ch)) return "'"; + return null; +}; + /** - * 专用于 TOML 文本的归一化；目前等同于 normalizeQuotes，后续可扩展（如空白、行尾等）。 + * 扫描 TOML 文本，把语法位置（字符串定界符）的弯/全角引号换成 ASCII 引号。 + * - 字符串外的弯引号视为定界符，换成 ASCII 并开启字符串； + * - 由弯引号开启的字符串，同族的弯引号或 ASCII 引号都能关闭； + * - 由 ASCII 引号开启的字符串，严格模式下只有 ASCII 引号能关闭（里面的弯引号是内容）， + * 宽松模式（lenient）下同族弯引号也能关闭； + * - 严格模式下，ASCII 开启的字符串到行尾（多行字符串到文末）都没关上时，回头按宽松 + * 规则重扫：编辑器每次输入都会归一化，中文输入法先打出的左引号已变成 ASCII， + * 右引号要靠这一步识别； + * - 基本字符串里的 `\“` 这类转义不是合法 TOML，按 `\"` 处理；注释原样保留。 */ -export const normalizeTomlText = (text: string): string => - normalizeQuotes(text); +const scanTomlQuotes = (text: string, lenient: boolean): string => { + let out = ""; + let i = 0; + const n = text.length; + + while (i < n) { + const ch = text[i]; + + if (ch === "#") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? n : end; + out += text.slice(i, stop); + i = stop; + continue; + } + + const family = quoteFamily(ch); + if (!family) { + out += ch; + i += 1; + continue; + } + + // 开启字符串：判断是否为三引号（多行字符串） + const multiline = + i + 2 < n && + quoteFamily(text[i + 1]) === family && + quoteFamily(text[i + 2]) === family; + const openLen = multiline ? 3 : 1; + const openedBySmart = text.slice(i, i + openLen) !== family.repeat(openLen); + const bodyStart = i + openLen; + out += family.repeat(openLen); + + const scan = ( + acceptSmart: boolean, + ): { body: string; next: number } | null => { + let body = ""; + let j = bodyStart; + const isCloser = (c: string) => + c === family || (acceptSmart && quoteFamily(c) === family); + + while (j < n) { + const c = text[j]; + if ( + family === '"' && + c === "\\" && + j + 1 < n && + (multiline || text[j + 1] !== "\n") + ) { + const escaped = text[j + 1]; + body += "\\" + (quoteFamily(escaped) === '"' ? '"' : escaped); + j += 2; + continue; + } + if (!multiline && c === "\n") return null; + if (!isCloser(c)) { + body += c; + j += 1; + continue; + } + if (!multiline) { + return { body: body + family, next: j + 1 }; + } + // 多行字符串：连续 3 个及以上的定界符才关闭（多出的最多 2 个属于内容） + let run = 0; + while (j + run < n && isCloser(text[j + run])) run += 1; + if (run >= 3) { + return { body: body + family.repeat(run), next: j + run }; + } + body += text.slice(j, j + run); + j += run; + } + return null; + }; + + const acceptSmart = lenient || openedBySmart; + const result = scan(acceptSmart) ?? (acceptSmart ? null : scan(true)); + if (result) { + out += result.body; + i = result.next; + continue; + } + + // 未闭合：原样输出到行尾（多行字符串到文末），交给解析器报错 + const end = multiline ? n : text.indexOf("\n", bodyStart); + const stop = end === -1 ? n : end; + out += text.slice(bodyStart, stop); + i = stop; + } + + return out; +}; + +const parsesAsToml = (text: string): boolean => { + try { + parseToml(text); + return true; + } catch { + return false; + } +}; + +/** + * 专用于 TOML 文本的归一化：只把语法位置的弯/全角引号换成 ASCII 引号，合法字符串 + * 内部的内容原样保留（#8559）。严格结果解析不通过、宽松结果（字符串里的同族弯引号 + * 也当作闭合符）能解析时才用宽松结果，所以合法文档里的字符串内容绝不会被改写。 + */ +export const normalizeTomlText = (text: string): string => { + if (!text || !/[“”„‟＂‘’＇]/.test(text)) return text; + + const strict = scanTomlQuotes(text, false); + if (parsesAsToml(strict)) return strict; + const lenient = scanTomlQuotes(text, true); + return lenient !== strict && parsesAsToml(lenient) ? lenient : strict; +}; diff --git a/tests/utils/textNormalization.test.ts b/tests/utils/textNormalization.test.ts new file mode 100644 index 000000000..8d45e975b --- /dev/null +++ b/tests/utils/textNormalization.test.ts @@ -0,0 +1,96 @@ +import { parse as parseToml } from "smol-toml"; +import { describe, expect, it } from "vitest"; +import { normalizeTomlText } from "@/utils/textNormalization"; +import { tomlToMcpServer, validateToml } from "@/utils/tomlUtils"; + +describe("normalizeTomlText", () => { + it("normalizes smart quotes used as string delimiters", () => { + expect(normalizeTomlText("command = “npx”\nargs = [‘-y’]")).toBe( + "command = \"npx\"\nargs = ['-y']", + ); + expect(normalizeTomlText("[mcp_servers.＂demo＂]")).toBe( + '[mcp_servers."demo"]', + ); + }); + + it("keeps smart quotes inside strings opened by ASCII quotes", () => { + const input = [ + 'a = "don’t"', + 'b = "say“hello”"', + "c = 'don’t'", + 'd = """multi “line” ’text’"""', + "e = '''it’s'''", + 'f = "escaped \\" then “x”"', + ].join("\n"); + expect(normalizeTomlText(input)).toBe(input); + }); + + it("keeps quotes of the other family inside smart-quoted strings", () => { + expect(normalizeTomlText("a = “don’t”")).toBe('a = "don’t"'); + expect(parseToml(normalizeTomlText("a = “don’t”")).a).toBe("don’t"); + }); + + it("closes an ASCII-opened string with a smart quote when nothing else closes it", () => { + // 编辑器逐字归一化后，输入法打出的左引号已是 ASCII，右引号仍是弯引号 + expect(normalizeTomlText('model = "gpt”\nx = 1')).toBe( + 'model = "gpt"\nx = 1', + ); + expect(normalizeTomlText("name = 'abc’")).toBe("name = 'abc'"); + }); + + it("repairs smart closers that are followed by more ASCII quotes on the line", () => { + // 合法文档不会走到这里：只有严格结果解析失败时才把字符串里的弯引号当闭合符 + expect(normalizeTomlText('args = ["npx”, "-y"]')).toBe( + 'args = ["npx", "-y"]', + ); + expect(normalizeTomlText('model = "gpt” # "example"')).toBe( + 'model = "gpt" # "example"', + ); + }); + + it("turns escaped smart double quotes into ASCII escapes", () => { + const normalized = normalizeTomlText("args = [“say \\“hello\\””]"); + expect(normalized).toBe('args = ["say \\"hello\\""]'); + expect(parseToml(normalized).args).toEqual(['say "hello"']); + }); + + it("leaves comments untouched", () => { + const input = "# “comment” it’s\nkey = “v” # don’t"; + expect(normalizeTomlText(input)).toBe( + '# “comment” it’s\nkey = "v" # don’t', + ); + }); + + it("does not touch unterminated strings beyond their opening quote", () => { + expect(normalizeTomlText("x = “abc")).toBe('x = "abc'); + }); +}); + +describe("tomlUtils with smart quotes in valid strings (#8559)", () => { + it("preserves a right single quotation mark in args", () => { + const input = 'command = "echo"\nargs = ["don’t"]'; + expect(validateToml(input)).toBe(""); + expect(tomlToMcpServer(input).args).toEqual(["don’t"]); + }); + + it("accepts curly double quotes inside a basic string", () => { + const input = 'command = "echo"\nargs = ["say“hello”"]'; + expect(validateToml(input)).toBe(""); + expect(tomlToMcpServer(input).args).toEqual(["say“hello”"]); + }); + + it("accepts curly single quotes inside a literal string", () => { + const input = "command = 'echo'\nargs = ['don’t']"; + expect(validateToml(input)).toBe(""); + expect(tomlToMcpServer(input).args).toEqual(["don’t"]); + }); + + it("still accepts configs written entirely with smart quotes", () => { + const input = "command = “npx”\nargs = [“-y”, “pkg”]"; + expect(validateToml(input)).toBe(""); + expect(tomlToMcpServer(input)).toMatchObject({ + command: "npx", + args: ["-y", "pkg"], + }); + }); +});