From 801e7fc66ac341a762a4a791e0b4c6cd600d1a86 Mon Sep 17 00:00:00 2001 From: digua Date: Sat, 4 Jul 2026 14:02:48 +0800 Subject: [PATCH] fix(cli): harden agent query safeguards Tighten agent-facing query validation, privacy guards, SQL result handling, config writes, and LIKE search semantics for the CLI query workflow. --- apps/cli/src/query/commands-messages.test.ts | 71 ++++++++ apps/cli/src/query/commands-messages.ts | 117 +++++++++---- .../cli/src/query/commands-topics-sql.test.ts | 61 ++++++- apps/cli/src/query/commands-topics-sql.ts | 105 +++++++++++- apps/cli/src/query/manifest.ts | 6 +- apps/cli/src/query/messages-output.ts | 3 +- apps/cli/src/query/parse.test.ts | 49 +++++- apps/cli/src/query/parse.ts | 154 +++++++++++++++++- apps/cli/src/query/register.test.ts | 41 +++++ apps/cli/src/query/register.ts | 29 +++- .../ai/tools/worker-data-provider.test.ts | 18 ++ .../main/ai/tools/worker-data-provider.ts | 4 +- .../main/ai/tools/worker-sql-result.ts | 11 ++ docs/cn/usage/cli-query.md | 10 +- docs/en/usage/cli-query.md | 10 +- packages/config/src/loader.ts | 7 + packages/config/src/set-config-field.test.ts | 35 ++++ packages/config/src/set-config-field.ts | 10 +- .../message-search-extensions.test.ts | 50 ++++++ packages/core/src/query/message-queries.ts | 8 +- packages/core/src/query/message-sql.ts | 4 +- .../src/ai/preprocessor/format.test.ts | 27 ++- .../src/ai/preprocessor/format.ts | 15 +- .../src/ai/preprocessor/pipeline.ts | 5 + .../node-runtime/src/ai/preprocessor/types.ts | 2 + .../src/definitions/analysis-tools.test.ts | 64 ++++++-- .../src/definitions/deep-search-messages.ts | 5 +- .../src/definitions/get-message-context.ts | 6 +- .../tools/src/definitions/search-context.ts | 30 ++++ .../tools/src/definitions/search-messages.ts | 5 +- .../src/providers/core-data-provider.test.ts | 12 ++ .../tools/src/providers/core-data-provider.ts | 3 + 32 files changed, 880 insertions(+), 97 deletions(-) create mode 100644 apps/cli/src/query/commands-messages.test.ts create mode 100644 apps/cli/src/query/register.test.ts create mode 100644 apps/desktop/main/ai/tools/worker-data-provider.test.ts create mode 100644 apps/desktop/main/ai/tools/worker-sql-result.ts create mode 100644 packages/tools/src/definitions/search-context.ts diff --git a/apps/cli/src/query/commands-messages.test.ts b/apps/cli/src/query/commands-messages.test.ts new file mode 100644 index 00000000..c382f74f --- /dev/null +++ b/apps/cli/src/query/commands-messages.test.ts @@ -0,0 +1,71 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' + +import { + assertContextAnchorsPresent, + capExpandedSearchMessages, + parseContextIds, + parseSearchKeywords, + searchTruncationStrategy, +} from './commands-messages' +import type { MessageLike } from './messages-output' + +describe('capExpandedSearchMessages', () => { + it('keeps hit messages when the cap is smaller than pre-context', () => { + const messages: MessageLike[] = [ + { id: 10, senderName: 'Alice', content: 'before 1', timestamp: 1710000000 }, + { id: 11, senderName: 'Alice', content: 'before 2', timestamp: 1710000001 }, + { id: 12, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }, + ] + + const capped = capExpandedSearchMessages(messages, new Set([12]), 1) + + assert.deepEqual( + capped.map((message) => message.id), + [12] + ) + }) +}) + +describe('parseContextIds', () => { + it('rejects blank id tokens instead of converting them to 0', () => { + assert.throws(() => parseContextIds('1021,'), /Invalid --id value/) + assert.throws(() => parseContextIds('1021,,1058'), /Invalid --id value/) + }) + + it('requires positive numeric message ids', () => { + assert.deepEqual(parseContextIds('1021,1058'), [1021, 1058]) + assert.throws(() => parseContextIds('0'), /Invalid --id value/) + }) +}) + +describe('parseSearchKeywords', () => { + it('trims keywords and rejects blank-only searches', () => { + assert.deepEqual(parseSearchKeywords([' alpha ', 'beta']), ['alpha', 'beta']) + assert.throws(() => parseSearchKeywords([' ']), /Invalid search keywords/) + }) +}) + +describe('searchTruncationStrategy', () => { + it('keeps newest chronological messages when search sort is descending', () => { + assert.equal(searchTruncationStrategy('desc'), 'keep_last') + assert.equal(searchTruncationStrategy('asc'), 'keep_first') + }) +}) + +describe('assertContextAnchorsPresent', () => { + it('rejects context results that do not include every requested anchor id', () => { + assert.throws( + () => + assertContextAnchorsPresent( + [999], + [ + { id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000000 }, + { id: 11, senderName: 'Bob', content: 'after', timestamp: 1710000001 }, + ], + '999' + ), + /No messages found/ + ) + }) +}) diff --git a/apps/cli/src/query/commands-messages.ts b/apps/cli/src/query/commands-messages.ts index 36d8672f..ff0a560e 100644 --- a/apps/cli/src/query/commands-messages.ts +++ b/apps/cli/src/query/commands-messages.ts @@ -16,7 +16,7 @@ import { import { runQuery } from './runner' import { createQueryContext, assertRawAllowed } from './context' import { resolveMember } from './resolve' -import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor } from './parse' +import { parseLimit, queryFingerprint, encodeCursor, decodeCursor, resolveTimeOptionsForCursor } from './parse' import { QueryError } from './envelope' import { buildMessagesResult, assertRawFormatCompatible, type MessageLike } from './messages-output' @@ -37,6 +37,73 @@ interface CommonMessageOptions { verbose?: boolean } +export function capExpandedSearchMessages( + messages: MessageLike[], + hitIds: ReadonlySet, + maxMessages: number +): MessageLike[] { + if (maxMessages <= 0) return [] + if (messages.length <= maxMessages) return messages + + const capped = messages.slice(0, maxMessages) + const includedIds = new Set(capped.map((message) => message.id)) + const missingHits = messages.filter((message) => hitIds.has(message.id) && !includedIds.has(message.id)) + + for (const hit of missingHits) { + let replaceIndex = -1 + for (let i = capped.length - 1; i >= 0; i--) { + if (!hitIds.has(capped[i].id)) { + replaceIndex = i + break + } + } + if (replaceIndex === -1) break + capped[replaceIndex] = hit + } + + return capped.sort((a, b) => messages.indexOf(a) - messages.indexOf(b)) +} + +export function parseContextIds(value: string): number[] { + const tokens = value.split(',').map((token) => token.trim()) + const ids = tokens.map((token) => (token.length > 0 ? Number(token) : Number.NaN)) + if (ids.length === 0 || ids.some((id) => !Number.isInteger(id) || id <= 0)) { + throw new QueryError({ + code: 'INVALID_ARGUMENT', + message: `Invalid --id value: ${value}`, + hint: 'Pass positive numeric message ids, e.g. --id 1021 or --id 1021,1058', + }) + } + return ids +} + +export function parseSearchKeywords(values: string[]): string[] { + const keywords = values.map((value) => value.trim()).filter(Boolean) + if (keywords.length === 0) { + throw new QueryError({ + code: 'INVALID_ARGUMENT', + message: 'Invalid search keywords', + hint: 'Pass at least one non-empty search keyword', + }) + } + return keywords +} + +export function searchTruncationStrategy(sort: string | undefined): 'keep_first' | 'keep_last' { + return sort === 'desc' ? 'keep_last' : 'keep_first' +} + +export function assertContextAnchorsPresent(ids: number[], messages: MessageLike[], rawValue: string): void { + const returnedIds = new Set(messages.map((message) => message.id)) + if (ids.some((id) => !returnedIds.has(id))) { + throw new QueryError({ + code: 'MESSAGE_NOT_FOUND', + message: `No messages found for id(s) ${rawValue}`, + hint: 'Use a single numeric id from [#id]/[#id*] markers; merged [#a-b] ranges are display-only', + }) + } +} + function addSharedOptions(cmd: Command): Command { return cmd .option('--session ', 'Session id or unique name (auto-selected when only one exists)') @@ -70,9 +137,9 @@ export function registerMessageCommands(program: Command): void { const ctx = createQueryContext(options) try { assertRawAllowed(ctx, options) - const time = parseTimeOptions(options) + const time = resolveTimeOptionsForCursor(options) const member = options.member ? resolveMember(ctx.db, options.member) : undefined - const limit = parseLimit(options.limit, 50, 500, '--limit') + const limit = parseLimit(options.limit, 50, 500, '--limit', 1) const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords const fingerprint = queryFingerprint({ @@ -104,7 +171,7 @@ export function registerMessageCommands(program: Command): void { totalHits: total, returnedHits: result.messages.length, hasMore, - ...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint) } : {}), + ...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint, { time }) } : {}), } return buildMessagesResult(format, ctx, options, result.messages, meta, { @@ -142,6 +209,7 @@ export function registerMessageCommands(program: Command): void { ) => { await runQuery('messages.search', options, async (format) => { assertRawFormatCompatible(format, options) + const searchKeywords = parseSearchKeywords(keywords) if (options.match !== 'any' && options.match !== 'all') { throw new QueryError({ code: 'INVALID_ARGUMENT', @@ -159,9 +227,9 @@ export function registerMessageCommands(program: Command): void { const ctx = createQueryContext(options) try { assertRawAllowed(ctx, options) - const time = parseTimeOptions(options) + const time = resolveTimeOptionsForCursor(options) const member = options.member ? resolveMember(ctx.db, options.member) : undefined - const limit = parseLimit(options.limit, 20, 500, '--limit') + const limit = parseLimit(options.limit, 20, 500, '--limit', 1) const context = parseLimit(options.context, 0, 50, '--context') const maxMessages = parseLimit(options.maxMessages, 200, 2000, '--max-messages') const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords @@ -169,7 +237,7 @@ export function registerMessageCommands(program: Command): void { const fingerprint = queryFingerprint({ command: 'messages.search', session: ctx.session.id, - keywords, + keywords: searchKeywords, match: options.match, sort: options.sort, startTs: time.startTs ?? null, @@ -179,7 +247,7 @@ export function registerMessageCommands(program: Command): void { }) const offset = options.cursor ? decodeCursor(options.cursor, fingerprint) : 0 - const result = searchMessagesByKeywords(ctx.db, keywords, { + const result = searchMessagesByKeywords(ctx.db, searchKeywords, { startTs: time.startTs, endTs: time.endTs, senderId: member?.id, @@ -215,7 +283,7 @@ export function registerMessageCommands(program: Command): void { const tail = hits.filter((m) => !keptHits.has(m.id)) pipelineMessages = [...expanded, ...tail] if (pipelineMessages.length > maxMessages) { - pipelineMessages = pipelineMessages.slice(0, maxMessages) + pipelineMessages = capExpandedSearchMessages(pipelineMessages, hitIds, maxMessages) warnings.push(`response capped at ${maxMessages} messages`) } } @@ -225,18 +293,18 @@ export function registerMessageCommands(program: Command): void { const meta: Record = { session: ctx.session, timeRange: time.meta, - query: { keywords, match: options.match, engine: 'like', sort: options.sort, context }, + query: { keywords: searchKeywords, match: options.match, engine: 'like', sort: options.sort, context }, ...(member ? { member } : {}), totalHits: total, returnedHits: hits.length, hasMore, - ...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint) } : {}), + ...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint, { time }) } : {}), ...(warnings.length > 0 ? { warnings } : {}), } return buildMessagesResult(format, ctx, options, pipelineMessages, meta, { hitIds, - strategy: 'keep_first', + strategy: searchTruncationStrategy(options.sort), defaultMaxChars: 120, }) } finally { @@ -249,33 +317,20 @@ export function registerMessageCommands(program: Command): void { // ---------- messages context ---------- const contextCmd = messagesCmd .command('context') - .description('Show messages around specific message ids (evidence follow-up)') + .description('Show messages around specific numeric message ids') .requiredOption('--id ', 'Message id(s), comma-separated') .option('--window ', 'Messages before/after each id (default 10, max 100)') addSharedOptions(contextCmd) contextCmd.action(async (options: CommonMessageOptions & { id: string; window?: string }) => { await runQuery('messages.context', options, async (format) => { assertRawFormatCompatible(format, options) - const ids = options.id.split(',').map((s) => Number(s.trim())) - if (ids.length === 0 || ids.some((n) => !Number.isInteger(n) || n < 0)) { - throw new QueryError({ - code: 'INVALID_ARGUMENT', - message: `Invalid --id value: ${options.id}`, - hint: 'Pass numeric message ids, e.g. --id 1021 or --id 1021,1058', - }) - } + const ids = parseContextIds(options.id) const ctx = createQueryContext(options) try { assertRawAllowed(ctx, options) const window = parseLimit(options.window, 10, 100, '--window') const messages = getMessageContext(ctx.db, ids, window) - if (messages.length === 0) { - throw new QueryError({ - code: 'MESSAGE_NOT_FOUND', - message: `No messages found for id(s) ${options.id}`, - hint: 'Message ids come from [#id] markers in search results', - }) - } + assertContextAnchorsPresent(ids, messages, options.id) const meta: Record = { session: ctx.session, anchorIds: ids, @@ -312,10 +367,10 @@ export function registerMessageCommands(program: Command): void { const ctx = createQueryContext(options) try { assertRawAllowed(ctx, options) - const time = parseTimeOptions(options) + const time = resolveTimeOptionsForCursor(options) const memberA = resolveMember(ctx.db, options.member[0]) const memberB = resolveMember(ctx.db, options.member[1]) - const limit = parseLimit(options.limit, 50, 500, '--limit') + const limit = parseLimit(options.limit, 50, 500, '--limit', 1) const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords const fingerprint = queryFingerprint({ @@ -346,7 +401,7 @@ export function registerMessageCommands(program: Command): void { totalHits: result.total, returnedHits: returned, hasMore, - ...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint) } : {}), + ...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint, { time }) } : {}), } return buildMessagesResult(format, ctx, options, result.messages, meta, { diff --git a/apps/cli/src/query/commands-topics-sql.test.ts b/apps/cli/src/query/commands-topics-sql.test.ts index 6405e86e..dc81c5cf 100644 --- a/apps/cli/src/query/commands-topics-sql.test.ts +++ b/apps/cli/src/query/commands-topics-sql.test.ts @@ -6,7 +6,14 @@ import { describe, it } from 'node:test' import assert from 'node:assert/strict' -import { topicsAgentText, type TopicListItem } from './commands-topics-sql' +import { + assertSqlPrivacyAllowed, + assertSqlResultPrivacyAllowed, + parseSqlRowLimit, + topicsAgentText, + type TopicListItem, +} from './commands-topics-sql' +import { QueryError } from './envelope' const items: TopicListItem[] = [ { @@ -41,3 +48,55 @@ describe('topicsAgentText', () => { assert.equal(topicsAgentText([]), 'No summaries.') }) }) + +describe('parseSqlRowLimit', () => { + it('rejects zero because SQL maxRows=0 means unlimited', () => { + assert.throws( + () => parseSqlRowLimit('0'), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + }) +}) + +describe('assertSqlPrivacyAllowed', () => { + it('requires raw mode before SQL can read message content', () => { + assert.throws( + () => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertSqlPrivacyAllowed('SELECT * FROM message LIMIT 10', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertSqlPrivacyAllowed('SELECT * FROM main.message LIMIT 10', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertSqlPrivacyAllowed('SELECT msg.* FROM message msg LIMIT 10', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertSqlPrivacyAllowed('SELECT "message".* FROM message LIMIT 10', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertSqlPrivacyAllowed('SELECT "m".* FROM message AS "m" LIMIT 10', false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT COUNT(*) AS total FROM message', false)) + assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT * FROM sqlite_master', false)) + assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', true)) + }) +}) + +describe('assertSqlResultPrivacyAllowed', () => { + it('requires raw mode when SQL result columns expose message content', () => { + assert.throws( + () => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], false), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], true)) + assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id'], false)) + }) +}) diff --git a/apps/cli/src/query/commands-topics-sql.ts b/apps/cli/src/query/commands-topics-sql.ts index 51152f63..e720d65a 100644 --- a/apps/cli/src/query/commands-topics-sql.ts +++ b/apps/cli/src/query/commands-topics-sql.ts @@ -2,9 +2,9 @@ * topics / sql / schema query commands. * * Derived-text privacy (design §6.2): topic summaries are LLM-generated from raw - * content, and SQL results can carry message bodies — both are desensitized by - * default, with blacklist-matching rows dropped and counted into meta.warnings. - * --raw (user-gated) and cli.allow_sql are the only escape hatches. + * content and are desensitized by default. SQL string cells are desensitized, + * but reading the message content column requires --raw so SQL expressions + * cannot encode raw bodies before the sanitizer sees them. */ import type { Command } from 'commander' @@ -20,6 +20,99 @@ function enabledRules(ctx: QueryContext): DesensitizeRule[] { return ctx.preprocessConfig.desensitize ? ctx.preprocessConfig.desensitizeRules.filter((r) => r.enabled) : [] } +export function parseSqlRowLimit(value: string | undefined): number { + return parseLimit(value, 100, 1000, '--limit', 1) +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +const SQL_IDENTIFIER = String.raw`(?:"[^"]+"|'[^']+'|` + '`[^`]+`' + String.raw`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_]*)` +const SQL_TABLE_REF_END = String.raw`(?=\s|$|[,;])` +const SQL_KEYWORDS = new Set([ + 'where', + 'join', + 'left', + 'right', + 'inner', + 'outer', + 'cross', + 'order', + 'group', + 'limit', + 'on', + 'using', +]) + +function sqlNamePattern(name: string): string { + const escaped = escapeRegExp(name) + return String.raw`(?:"${escaped}"|'${escaped}'|` + '`' + escaped + '`' + String.raw`|\[${escaped}\]|${escaped})` +} + +function unquoteSqlIdentifier(value: string): string { + const trimmed = value.trim() + if (trimmed.startsWith('[') && trimmed.endsWith(']')) return trimmed.slice(1, -1) + const quote = trimmed[0] + if ((quote === '"' || quote === "'" || quote === '`') && trimmed.endsWith(quote)) { + return trimmed.slice(1, -1) + } + return trimmed +} + +function messageTablePattern(): string { + const schemaPrefix = String.raw`(?:${SQL_IDENTIFIER}\s*\.\s*)?` + return `${schemaPrefix}${sqlNamePattern('message')}` +} + +function selectsMessageStar(query: string): boolean { + const selectMatch = query.match(/\bselect\b([\s\S]+?)\bfrom\b/i) + if (!selectMatch) return false + + const projection = selectMatch[1] + const messageTable = messageTablePattern() + if (/(^|[,\s])\*(?=$|[,\s])/i.test(projection)) { + return new RegExp(String.raw`\b(?:from|join)\s+${messageTable}${SQL_TABLE_REF_END}`, 'i').test(query) + } + + const aliasMatches = query.matchAll( + new RegExp(String.raw`\b(?:from|join)\s+${messageTable}(?:\s+(?:as\s+)?(${SQL_IDENTIFIER}))?`, 'gi') + ) + const aliases = new Set(['message']) + for (const match of aliasMatches) { + const alias = match[1] ? unquoteSqlIdentifier(match[1]).toLowerCase() : undefined + if (alias && !SQL_KEYWORDS.has(alias)) aliases.add(alias) + } + + return [...aliases].some((alias) => + new RegExp(`${sqlNamePattern(alias)}\\s*\\.\\s*\\*(?=$|[\\s,])`, 'i').test(projection) + ) +} + +export function assertSqlPrivacyAllowed(query: string, raw: boolean): void { + if (raw) return + if (!/(^|[^A-Za-z0-9_])["'`[]?content["'`\]]?(?![A-Za-z0-9_])/i.test(query) && !selectsMessageStar(query)) { + return + } + + throw new QueryError({ + code: 'INVALID_ARGUMENT', + message: 'SQL queries that read message content require --raw', + hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging', + }) +} + +export function assertSqlResultPrivacyAllowed(columns: readonly string[], raw: boolean): void { + if (raw) return + if (!columns.some((column) => unquoteSqlIdentifier(column).toLowerCase() === 'content')) return + + throw new QueryError({ + code: 'INVALID_ARGUMENT', + message: 'SQL results that include message content require --raw', + hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging', + }) +} + export interface TopicListItem { id: number since: string @@ -209,7 +302,7 @@ export function registerTopicCommands(program: Command): void { export function registerSqlCommands(program: Command): void { program .command('sql') - .description('Read-only SQL fallback (string cells are desensitized by default)') + .description('Read-only SQL fallback (string cells are desensitized; message content requires --raw)') .argument('', 'SELECT / WITH statement') .option('--session ', 'Session id or unique name') .option('--limit ', 'Max rows (default 100, max 1000)') @@ -227,7 +320,8 @@ export function registerSqlCommands(program: Command): void { }) } assertRawAllowed(ctx, options) - const limit = parseLimit(options.limit, 100, 1000, '--limit') + assertSqlPrivacyAllowed(query, !!options.raw) + const limit = parseSqlRowLimit(options.limit) let result try { @@ -239,6 +333,7 @@ export function registerSqlCommands(program: Command): void { hint: 'Run `chatlab schema` to inspect available tables and columns', }) } + assertSqlResultPrivacyAllowed(result.columns, !!options.raw) let rows = result.rows let removed = 0 diff --git a/apps/cli/src/query/manifest.ts b/apps/cli/src/query/manifest.ts index d036b6db..006acc64 100644 --- a/apps/cli/src/query/manifest.ts +++ b/apps/cli/src/query/manifest.ts @@ -66,11 +66,11 @@ const EXAMPLES: ManifestExample[] = [ command: 'chatlab messages search 旅游 --since 2026-01-01 --limit 20 --format agent', }, { - task: 'Who mentioned it first (evidence ids)?', + task: 'Who mentioned it first (message ids)?', command: 'chatlab messages search 报销 --sort asc --limit 5 --format agent', }, { - task: 'Follow up a [#id] citation with context', + task: 'Follow up a single [#id] marker with context', command: 'chatlab messages context --id 1021 --window 10 --format agent', }, { @@ -92,7 +92,7 @@ const NOTES: string[] = [ 'stdout carries exactly one JSON envelope in agent/json mode; logs go to stderr.', 'Success envelope: { ok, command, data, meta }; failure: { ok: false, command, error }.', 'Pagination: follow meta.nextCursor with --cursor; cursors are bound to the exact query.', - '[#id] markers in agent text are message ids usable with `messages context --id`.', + 'Single-message [#id] and [#id*] markers are usable with `messages context --id`; merged [#a-b] ranges are display-only.', 'Privacy preprocessing (desensitize/blacklist) is always on by default.', ] diff --git a/apps/cli/src/query/messages-output.ts b/apps/cli/src/query/messages-output.ts index 1f43481e..5e35e72e 100644 --- a/apps/cli/src/query/messages-output.ts +++ b/apps/cli/src/query/messages-output.ts @@ -2,7 +2,8 @@ * Shared output shaping for message-carrying query commands. * * agent format: AI-facing text from the shared preprocessing pipeline with - * [#id] citations (design §6.2). json format: structured items with privacy + * [#id] / [#id*] message markers and display-only [#a-b] merged ranges. + * json format: structured items with privacy * steps applied (cleaning/blacklist/desensitize) but no merging/denoising. */ diff --git a/apps/cli/src/query/parse.test.ts b/apps/cli/src/query/parse.test.ts index a30a87d5..30636b9f 100644 --- a/apps/cli/src/query/parse.test.ts +++ b/apps/cli/src/query/parse.test.ts @@ -6,7 +6,15 @@ import { describe, it } from 'node:test' import assert from 'node:assert/strict' -import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor, epochToIso } from './parse' +import { + parseTimeOptions, + parseLimit, + queryFingerprint, + encodeCursor, + decodeCursor, + epochToIso, + resolveTimeOptionsForCursor, +} from './parse' import { QueryError } from './envelope' // Fixed reference: 2026-07-03 15:30:00 local time @@ -60,6 +68,22 @@ describe('parseTimeOptions', () => { } }) + it('rejects invalid calendar dates instead of normalizing them', () => { + for (const bad of [ + { since: '2026-02-31' }, + { until: '2026-13-01' }, + { since: '2026-02-31 08:30' }, + { since: '2026-06-01 25:00' }, + { until: '2026-02-31T10:00:00+08:00' }, + ]) { + assert.throws( + () => parseTimeOptions(bad, NOW), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT', + JSON.stringify(bad) + ) + } + }) + it('rejects inverted ranges', () => { assert.throws( () => parseTimeOptions({ since: '2026-07-01', until: '2026-06-01' }, NOW), @@ -90,6 +114,14 @@ describe('parseLimit', () => { ) } }) + + it('can reject zero for cursor-paginated page sizes', () => { + assert.equal(parseLimit('0', 0, 50, '--context'), 0) + assert.throws( + () => parseLimit('0', 50, 500, '--limit', 1), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + }) }) describe('cursor', () => { @@ -116,4 +148,19 @@ describe('cursor', () => { (err: unknown) => err instanceof QueryError && err.code === 'CURSOR_INVALID' ) }) + + it('reuses frozen time bounds embedded in cursor tokens', () => { + const firstRange = resolveTimeOptionsForCursor({ last: '7d' }, NOW) + const fp = queryFingerprint({ + command: 'messages.list', + startTs: firstRange.startTs ?? null, + endTs: firstRange.endTs ?? null, + }) + const token = encodeCursor(50, fp, { time: firstRange }) + + const later = new Date(NOW.getTime() + 60_000) + const secondRange = resolveTimeOptionsForCursor({ last: '7d', cursor: token }, later) + assert.deepEqual(secondRange, firstRange) + assert.equal(decodeCursor(token, fp), 50) + }) }) diff --git a/apps/cli/src/query/parse.ts b/apps/cli/src/query/parse.ts index 901ca2e4..00fdaa8e 100644 --- a/apps/cli/src/query/parse.ts +++ b/apps/cli/src/query/parse.ts @@ -23,6 +23,10 @@ export interface ResolvedTimeRange { meta: { since: string | null; until: string | null } } +export interface CursorSnapshot { + time?: ResolvedTimeRange +} + /** Format a Date as ISO 8601 with the local UTC offset (sortable, self-describing). */ export function toIsoWithOffset(date: Date): string { const pad = (n: number, w = 2) => String(Math.abs(n)).padStart(w, '0') @@ -43,6 +47,7 @@ export function epochToIso(ts: number): string { const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/ const DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?$/ +const ISO_DATE_PREFIX = /^(\d{4})-(\d{2})-(\d{2})T/ const LAST_DURATION = /^(\d+)([hdw])$/ function startOfDay(base: Date, dayOffset = 0): Date { @@ -57,6 +62,14 @@ function invalidTime(flag: string, value: string): QueryError { }) } +function isSameLocalDate(date: Date, y: number, m: number, d: number): boolean { + return date.getFullYear() === y && date.getMonth() === m - 1 && date.getDate() === d +} + +function isSameLocalDateTime(date: Date, y: number, m: number, d: number, h: number, min: number, s: number): boolean { + return isSameLocalDate(date, y, m, d) && date.getHours() === h && date.getMinutes() === min && date.getSeconds() === s +} + /** * Parse a single time value. Date-only values resolve to the start of the day; * when `boundary` is 'end' they resolve to the end of the day (inclusive whole-day). @@ -71,23 +84,43 @@ function parseTimeValue(value: string, boundary: 'start' | 'end', flag: string, const dateOnly = DATE_ONLY.exec(value) if (dateOnly) { const [, y, m, d] = dateOnly - const base = new Date(Number(y), Number(m) - 1, Number(d)) - if (Number.isNaN(base.getTime())) throw invalidTime(flag, value) + const year = Number(y) + const month = Number(m) + const day = Number(d) + const base = new Date(year, month - 1, day) + if (Number.isNaN(base.getTime()) || !isSameLocalDate(base, year, month, day)) throw invalidTime(flag, value) if (boundary === 'start') return base - return new Date(new Date(Number(y), Number(m) - 1, Number(d) + 1).getTime() - 1000) + return new Date(new Date(year, month - 1, day + 1).getTime() - 1000) } const dateTime = DATE_TIME.exec(value) if (dateTime) { const [, y, m, d, h, min, s] = dateTime - const parsed = new Date(Number(y), Number(m) - 1, Number(d), Number(h), Number(min), Number(s ?? 0)) - if (Number.isNaN(parsed.getTime())) throw invalidTime(flag, value) + const year = Number(y) + const month = Number(m) + const day = Number(d) + const hour = Number(h) + const minute = Number(min) + const second = Number(s ?? 0) + const parsed = new Date(year, month - 1, day, hour, minute, second) + if (Number.isNaN(parsed.getTime()) || !isSameLocalDateTime(parsed, year, month, day, hour, minute, second)) { + throw invalidTime(flag, value) + } return parsed } // full ISO 8601 (with timezone offset or Z) const parsed = new Date(value) if (!Number.isNaN(parsed.getTime()) && /\d{4}-\d{2}-\d{2}T/.test(value)) { + const isoDate = ISO_DATE_PREFIX.exec(value) + if (isoDate) { + const [, y, m, d] = isoDate + const year = Number(y) + const month = Number(m) + const day = Number(d) + const localDate = new Date(year, month - 1, day) + if (!isSameLocalDate(localDate, year, month, day)) throw invalidTime(flag, value) + } return parsed } @@ -161,15 +194,19 @@ export function parseLimit( value: string | number | undefined, defaultValue: number, cap: number, - flag: string + flag: string, + min = 0 ): number { if (value === undefined) return Math.min(defaultValue, cap) const n = typeof value === 'number' ? value : Number(value) - if (!Number.isInteger(n) || n < 0) { + if (!Number.isInteger(n) || n < min) { throw new QueryError({ code: 'INVALID_ARGUMENT', message: `Invalid ${flag} value: ${value}`, - hint: `${flag} must be a non-negative integer (max ${cap})`, + hint: + min > 0 + ? `${flag} must be an integer from ${min} to ${cap}` + : `${flag} must be a non-negative integer (max ${cap})`, }) } return Math.min(n, cap) @@ -183,12 +220,111 @@ export function queryFingerprint(conditions: Record): string { return createHash('sha256').update(canonical).digest('hex').slice(0, 12) } -export function encodeCursor(offset: number, fingerprint: string): string { +export function freezeTimeRangeForCursor(time: ResolvedTimeRange): ResolvedTimeRange { + if (time.endTs !== undefined || time.meta.until === null) return time + const endMs = new Date(time.meta.until).getTime() + if (Number.isNaN(endMs)) return time + return { ...time, endTs: Math.floor(endMs / 1000) } +} + +export function resolveTimeOptionsForCursor( + options: TimeCliOptions & { cursor?: string }, + now = new Date() +): ResolvedTimeRange { + const snapshot = options.cursor ? getCursorSnapshot(options.cursor) : undefined + return snapshot?.time ?? freezeTimeRangeForCursor(parseTimeOptions(options, now)) +} + +export function encodeCursor(offset: number, fingerprint: string, snapshot?: CursorSnapshot): string { + if (snapshot?.time) { + return Buffer.from(JSON.stringify({ v: 2, fp: fingerprint, offset, time: snapshot.time }), 'utf8').toString( + 'base64url' + ) + } return Buffer.from(`${fingerprint}:${offset}`, 'utf8').toString('base64url') } +function asObject(value: unknown): Record | null { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null +} + +function decodeCursorEnvelope( + token: string +): { fingerprint: string; offset: number; snapshot?: CursorSnapshot } | null { + let decoded = '' + try { + decoded = Buffer.from(token, 'base64url').toString('utf8') + } catch { + return null + } + + let payload: unknown + try { + payload = JSON.parse(decoded) + } catch { + return null + } + + const object = asObject(payload) + if ( + !object || + object.v !== 2 || + typeof object.fp !== 'string' || + typeof object.offset !== 'number' || + !Number.isInteger(object.offset) || + object.offset < 0 + ) { + return null + } + + const snapshot = decodeCursorSnapshot(object) + return { fingerprint: object.fp, offset: object.offset, ...(snapshot ? { snapshot } : {}) } +} + +function decodeCursorSnapshot(payload: Record): CursorSnapshot | undefined { + const timeObject = asObject(payload.time) + if (!timeObject) return undefined + const metaObject = asObject(timeObject.meta) + if (!metaObject) return undefined + + const startTs = timeObject.startTs + const endTs = timeObject.endTs + const since = metaObject.since + const until = metaObject.until + if (startTs !== undefined && typeof startTs !== 'number') return undefined + if (endTs !== undefined && typeof endTs !== 'number') return undefined + if (since !== null && typeof since !== 'string') return undefined + if (until !== null && typeof until !== 'string') return undefined + + return { + time: { + ...(startTs !== undefined ? { startTs } : {}), + ...(endTs !== undefined ? { endTs } : {}), + meta: { since, until }, + }, + } +} + +export function getCursorSnapshot(token: string): CursorSnapshot | undefined { + return decodeCursorEnvelope(token)?.snapshot +} + /** Decode a cursor and validate it against the current query fingerprint. */ export function decodeCursor(token: string, fingerprint: string): number { + const envelope = decodeCursorEnvelope(token) + if (envelope) { + if (envelope.fingerprint !== fingerprint) { + throw new QueryError({ + code: 'CURSOR_INVALID', + message: 'Cursor does not match the current query conditions', + hint: 'Cursors are only valid for the exact query that produced them; re-run the query without --cursor', + }) + } + return envelope.offset + } + let decoded = '' try { decoded = Buffer.from(token, 'base64url').toString('utf8') diff --git a/apps/cli/src/query/register.test.ts b/apps/cli/src/query/register.test.ts new file mode 100644 index 00000000..309ba358 --- /dev/null +++ b/apps/cli/src/query/register.test.ts @@ -0,0 +1,41 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import type { ChatLabConfig } from '@openchatlab/config' + +import { assertLegacySqlAllowed } from './register' +import { QueryError } from './envelope' + +function configWithSql(allowSql: boolean): ChatLabConfig { + return { + llm: { provider: '', model: '', base_url: '' }, + data: { user_data_dir: '', electron_migration_done: false }, + api: { port: 3110, host: '127.0.0.1', token: '', require_auth: false }, + locale: { lang: '' }, + ui: { default_session_tab: 'overview', session_gap_threshold: 1800, summary_strategy: 'standard' }, + cli: { allow_raw: false, allow_sql: allowSql }, + } +} + +describe('legacy query alias SQL gate', () => { + it('rejects SQL when cli.allow_sql is disabled', () => { + assert.throws( + () => assertLegacySqlAllowed(configWithSql(false)), + (err: unknown) => err instanceof QueryError && err.code === 'SQL_DISABLED' + ) + }) + + it('allows SQL when cli.allow_sql is enabled', () => { + assert.doesNotThrow(() => assertLegacySqlAllowed(configWithSql(true))) + }) + + it('rejects message content SQL because the legacy alias has no raw mode', () => { + assert.throws( + () => assertLegacySqlAllowed(configWithSql(true), 'SELECT content FROM message'), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + assert.throws( + () => assertLegacySqlAllowed(configWithSql(true), 'SELECT * FROM message'), + (err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT' + ) + }) +}) diff --git a/apps/cli/src/query/register.ts b/apps/cli/src/query/register.ts index ee66b867..95e47279 100644 --- a/apps/cli/src/query/register.ts +++ b/apps/cli/src/query/register.ts @@ -4,12 +4,19 @@ */ import type { Command } from 'commander' +import type { ChatLabConfig } from '@openchatlab/config' import { searchMessagesLike, executeReadonlySql } from '@openchatlab/core' import { initRuntime } from '../runtime' import { registerSessionCommands, registerMemberCommands } from './commands-sessions-members' import { registerMessageCommands } from './commands-messages' import { registerStatsCommands } from './commands-stats' -import { registerTopicCommands, registerSqlCommands } from './commands-topics-sql' +import { + registerTopicCommands, + registerSqlCommands, + assertSqlPrivacyAllowed, + assertSqlResultPrivacyAllowed, +} from './commands-topics-sql' +import { QueryError } from './envelope' export function registerQueryCommands(program: Command): void { registerSessionCommands(program) @@ -21,6 +28,17 @@ export function registerQueryCommands(program: Command): void { registerLegacyAliases(program) } +export function assertLegacySqlAllowed(config: Pick, sql?: string): void { + if (!config.cli.allow_sql) { + throw new QueryError({ + code: 'SQL_DISABLED', + message: 'The sql command is disabled by configuration', + hint: 'The user can re-enable it with `chatlab config set cli.allow_sql true`', + }) + } + if (sql) assertSqlPrivacyAllowed(sql, false) +} + // ==================== Hidden legacy aliases (output kept byte-compatible) ==================== function registerLegacyAliases(program: Command): void { @@ -67,7 +85,7 @@ function registerLegacyAliases(program: Command): void { console.error( '[Deprecated] `chatlab query` — use `chatlab sql "