mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
fix(cli): harden agent query safeguards
Tighten agent-facing query validation, privacy guards, SQL result handling, config writes, and LIKE search semantics for the CLI query workflow.
This commit is contained in:
1 parent
b33cbd065d
commit
801e7fc66a
32 files changed
+880
-97
No files matched your search
@@ -0,0 +1,71 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import {
|
||||
assertContextAnchorsPresent,
|
||||
capExpandedSearchMessages,
|
||||
parseContextIds,
|
||||
parseSearchKeywords,
|
||||
searchTruncationStrategy,
|
||||
} from './commands-messages'
|
||||
import type { MessageLike } from './messages-output'
|
||||
|
||||
describe('capExpandedSearchMessages', () => {
|
||||
it('keeps hit messages when the cap is smaller than pre-context', () => {
|
||||
const messages: MessageLike[] = [
|
||||
{ id: 10, senderName: 'Alice', content: 'before 1', timestamp: 1710000000 },
|
||||
{ id: 11, senderName: 'Alice', content: 'before 2', timestamp: 1710000001 },
|
||||
{ id: 12, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 },
|
||||
]
|
||||
|
||||
const capped = capExpandedSearchMessages(messages, new Set([12]), 1)
|
||||
|
||||
assert.deepEqual(
|
||||
capped.map((message) => message.id),
|
||||
[12]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseContextIds', () => {
|
||||
it('rejects blank id tokens instead of converting them to 0', () => {
|
||||
assert.throws(() => parseContextIds('1021,'), /Invalid --id value/)
|
||||
assert.throws(() => parseContextIds('1021,,1058'), /Invalid --id value/)
|
||||
})
|
||||
|
||||
it('requires positive numeric message ids', () => {
|
||||
assert.deepEqual(parseContextIds('1021,1058'), [1021, 1058])
|
||||
assert.throws(() => parseContextIds('0'), /Invalid --id value/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseSearchKeywords', () => {
|
||||
it('trims keywords and rejects blank-only searches', () => {
|
||||
assert.deepEqual(parseSearchKeywords([' alpha ', 'beta']), ['alpha', 'beta'])
|
||||
assert.throws(() => parseSearchKeywords([' ']), /Invalid search keywords/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('searchTruncationStrategy', () => {
|
||||
it('keeps newest chronological messages when search sort is descending', () => {
|
||||
assert.equal(searchTruncationStrategy('desc'), 'keep_last')
|
||||
assert.equal(searchTruncationStrategy('asc'), 'keep_first')
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertContextAnchorsPresent', () => {
|
||||
it('rejects context results that do not include every requested anchor id', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
assertContextAnchorsPresent(
|
||||
[999],
|
||||
[
|
||||
{ id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000000 },
|
||||
{ id: 11, senderName: 'Bob', content: 'after', timestamp: 1710000001 },
|
||||
],
|
||||
'999'
|
||||
),
|
||||
/No messages found/
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
import { runQuery } from './runner'
|
||||
import { createQueryContext, assertRawAllowed } from './context'
|
||||
import { resolveMember } from './resolve'
|
||||
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor } from './parse'
|
||||
import { parseLimit, queryFingerprint, encodeCursor, decodeCursor, resolveTimeOptionsForCursor } from './parse'
|
||||
import { QueryError } from './envelope'
|
||||
import { buildMessagesResult, assertRawFormatCompatible, type MessageLike } from './messages-output'
|
||||
|
||||
@@ -37,6 +37,73 @@ interface CommonMessageOptions {
|
||||
verbose?: boolean
|
||||
}
|
||||
|
||||
export function capExpandedSearchMessages(
|
||||
messages: MessageLike[],
|
||||
hitIds: ReadonlySet<number>,
|
||||
maxMessages: number
|
||||
): MessageLike[] {
|
||||
if (maxMessages <= 0) return []
|
||||
if (messages.length <= maxMessages) return messages
|
||||
|
||||
const capped = messages.slice(0, maxMessages)
|
||||
const includedIds = new Set(capped.map((message) => message.id))
|
||||
const missingHits = messages.filter((message) => hitIds.has(message.id) && !includedIds.has(message.id))
|
||||
|
||||
for (const hit of missingHits) {
|
||||
let replaceIndex = -1
|
||||
for (let i = capped.length - 1; i >= 0; i--) {
|
||||
if (!hitIds.has(capped[i].id)) {
|
||||
replaceIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (replaceIndex === -1) break
|
||||
capped[replaceIndex] = hit
|
||||
}
|
||||
|
||||
return capped.sort((a, b) => messages.indexOf(a) - messages.indexOf(b))
|
||||
}
|
||||
|
||||
export function parseContextIds(value: string): number[] {
|
||||
const tokens = value.split(',').map((token) => token.trim())
|
||||
const ids = tokens.map((token) => (token.length > 0 ? Number(token) : Number.NaN))
|
||||
if (ids.length === 0 || ids.some((id) => !Number.isInteger(id) || id <= 0)) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid --id value: ${value}`,
|
||||
hint: 'Pass positive numeric message ids, e.g. --id 1021 or --id 1021,1058',
|
||||
})
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
export function parseSearchKeywords(values: string[]): string[] {
|
||||
const keywords = values.map((value) => value.trim()).filter(Boolean)
|
||||
if (keywords.length === 0) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'Invalid search keywords',
|
||||
hint: 'Pass at least one non-empty search keyword',
|
||||
})
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
export function searchTruncationStrategy(sort: string | undefined): 'keep_first' | 'keep_last' {
|
||||
return sort === 'desc' ? 'keep_last' : 'keep_first'
|
||||
}
|
||||
|
||||
export function assertContextAnchorsPresent(ids: number[], messages: MessageLike[], rawValue: string): void {
|
||||
const returnedIds = new Set(messages.map((message) => message.id))
|
||||
if (ids.some((id) => !returnedIds.has(id))) {
|
||||
throw new QueryError({
|
||||
code: 'MESSAGE_NOT_FOUND',
|
||||
message: `No messages found for id(s) ${rawValue}`,
|
||||
hint: 'Use a single numeric id from [#id]/[#id*] markers; merged [#a-b] ranges are display-only',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function addSharedOptions(cmd: Command): Command {
|
||||
return cmd
|
||||
.option('--session <ref>', 'Session id or unique name (auto-selected when only one exists)')
|
||||
@@ -70,9 +137,9 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
|
||||
const fingerprint = queryFingerprint({
|
||||
@@ -104,7 +171,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
totalHits: total,
|
||||
returnedHits: result.messages.length,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint, { time }) } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, result.messages, meta, {
|
||||
@@ -142,6 +209,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
) => {
|
||||
await runQuery('messages.search', options, async (format) => {
|
||||
assertRawFormatCompatible(format, options)
|
||||
const searchKeywords = parseSearchKeywords(keywords)
|
||||
if (options.match !== 'any' && options.match !== 'all') {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
@@ -159,9 +227,9 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
|
||||
const limit = parseLimit(options.limit, 20, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 20, 500, '--limit', 1)
|
||||
const context = parseLimit(options.context, 0, 50, '--context')
|
||||
const maxMessages = parseLimit(options.maxMessages, 200, 2000, '--max-messages')
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
@@ -169,7 +237,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
const fingerprint = queryFingerprint({
|
||||
command: 'messages.search',
|
||||
session: ctx.session.id,
|
||||
keywords,
|
||||
keywords: searchKeywords,
|
||||
match: options.match,
|
||||
sort: options.sort,
|
||||
startTs: time.startTs ?? null,
|
||||
@@ -179,7 +247,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
})
|
||||
const offset = options.cursor ? decodeCursor(options.cursor, fingerprint) : 0
|
||||
|
||||
const result = searchMessagesByKeywords(ctx.db, keywords, {
|
||||
const result = searchMessagesByKeywords(ctx.db, searchKeywords, {
|
||||
startTs: time.startTs,
|
||||
endTs: time.endTs,
|
||||
senderId: member?.id,
|
||||
@@ -215,7 +283,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
const tail = hits.filter((m) => !keptHits.has(m.id))
|
||||
pipelineMessages = [...expanded, ...tail]
|
||||
if (pipelineMessages.length > maxMessages) {
|
||||
pipelineMessages = pipelineMessages.slice(0, maxMessages)
|
||||
pipelineMessages = capExpandedSearchMessages(pipelineMessages, hitIds, maxMessages)
|
||||
warnings.push(`response capped at ${maxMessages} messages`)
|
||||
}
|
||||
}
|
||||
@@ -225,18 +293,18 @@ export function registerMessageCommands(program: Command): void {
|
||||
const meta: Record<string, unknown> = {
|
||||
session: ctx.session,
|
||||
timeRange: time.meta,
|
||||
query: { keywords, match: options.match, engine: 'like', sort: options.sort, context },
|
||||
query: { keywords: searchKeywords, match: options.match, engine: 'like', sort: options.sort, context },
|
||||
...(member ? { member } : {}),
|
||||
totalHits: total,
|
||||
returnedHits: hits.length,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint, { time }) } : {}),
|
||||
...(warnings.length > 0 ? { warnings } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, pipelineMessages, meta, {
|
||||
hitIds,
|
||||
strategy: 'keep_first',
|
||||
strategy: searchTruncationStrategy(options.sort),
|
||||
defaultMaxChars: 120,
|
||||
})
|
||||
} finally {
|
||||
@@ -249,33 +317,20 @@ export function registerMessageCommands(program: Command): void {
|
||||
// ---------- messages context ----------
|
||||
const contextCmd = messagesCmd
|
||||
.command('context')
|
||||
.description('Show messages around specific message ids (evidence follow-up)')
|
||||
.description('Show messages around specific numeric message ids')
|
||||
.requiredOption('--id <ids>', 'Message id(s), comma-separated')
|
||||
.option('--window <n>', 'Messages before/after each id (default 10, max 100)')
|
||||
addSharedOptions(contextCmd)
|
||||
contextCmd.action(async (options: CommonMessageOptions & { id: string; window?: string }) => {
|
||||
await runQuery('messages.context', options, async (format) => {
|
||||
assertRawFormatCompatible(format, options)
|
||||
const ids = options.id.split(',').map((s) => Number(s.trim()))
|
||||
if (ids.length === 0 || ids.some((n) => !Number.isInteger(n) || n < 0)) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid --id value: ${options.id}`,
|
||||
hint: 'Pass numeric message ids, e.g. --id 1021 or --id 1021,1058',
|
||||
})
|
||||
}
|
||||
const ids = parseContextIds(options.id)
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const window = parseLimit(options.window, 10, 100, '--window')
|
||||
const messages = getMessageContext(ctx.db, ids, window)
|
||||
if (messages.length === 0) {
|
||||
throw new QueryError({
|
||||
code: 'MESSAGE_NOT_FOUND',
|
||||
message: `No messages found for id(s) ${options.id}`,
|
||||
hint: 'Message ids come from [#id] markers in search results',
|
||||
})
|
||||
}
|
||||
assertContextAnchorsPresent(ids, messages, options.id)
|
||||
const meta: Record<string, unknown> = {
|
||||
session: ctx.session,
|
||||
anchorIds: ids,
|
||||
@@ -312,10 +367,10 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const memberA = resolveMember(ctx.db, options.member[0])
|
||||
const memberB = resolveMember(ctx.db, options.member[1])
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
|
||||
const fingerprint = queryFingerprint({
|
||||
@@ -346,7 +401,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
totalHits: result.total,
|
||||
returnedHits: returned,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint, { time }) } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, result.messages, meta, {
|
||||
|
||||
@@ -6,7 +6,14 @@
|
||||
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { topicsAgentText, type TopicListItem } from './commands-topics-sql'
|
||||
import {
|
||||
assertSqlPrivacyAllowed,
|
||||
assertSqlResultPrivacyAllowed,
|
||||
parseSqlRowLimit,
|
||||
topicsAgentText,
|
||||
type TopicListItem,
|
||||
} from './commands-topics-sql'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
const items: TopicListItem[] = [
|
||||
{
|
||||
@@ -41,3 +48,55 @@ describe('topicsAgentText', () => {
|
||||
assert.equal(topicsAgentText([]), 'No summaries.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseSqlRowLimit', () => {
|
||||
it('rejects zero because SQL maxRows=0 means unlimited', () => {
|
||||
assert.throws(
|
||||
() => parseSqlRowLimit('0'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSqlPrivacyAllowed', () => {
|
||||
it('requires raw mode before SQL can read message content', () => {
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT * FROM message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT * FROM main.message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT msg.* FROM message msg LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT "message".* FROM message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT "m".* FROM message AS "m" LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT COUNT(*) AS total FROM message', false))
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT * FROM sqlite_master', false))
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', true))
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSqlResultPrivacyAllowed', () => {
|
||||
it('requires raw mode when SQL result columns expose message content', () => {
|
||||
assert.throws(
|
||||
() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], true))
|
||||
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id'], false))
|
||||
})
|
||||
})
|
||||
@@ -2,9 +2,9 @@
|
||||
* topics / sql / schema query commands.
|
||||
*
|
||||
* Derived-text privacy (design §6.2): topic summaries are LLM-generated from raw
|
||||
* content, and SQL results can carry message bodies — both are desensitized by
|
||||
* default, with blacklist-matching rows dropped and counted into meta.warnings.
|
||||
* --raw (user-gated) and cli.allow_sql are the only escape hatches.
|
||||
* content and are desensitized by default. SQL string cells are desensitized,
|
||||
* but reading the message content column requires --raw so SQL expressions
|
||||
* cannot encode raw bodies before the sanitizer sees them.
|
||||
*/
|
||||
|
||||
import type { Command } from 'commander'
|
||||
@@ -20,6 +20,99 @@ function enabledRules(ctx: QueryContext): DesensitizeRule[] {
|
||||
return ctx.preprocessConfig.desensitize ? ctx.preprocessConfig.desensitizeRules.filter((r) => r.enabled) : []
|
||||
}
|
||||
|
||||
export function parseSqlRowLimit(value: string | undefined): number {
|
||||
return parseLimit(value, 100, 1000, '--limit', 1)
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
||||
const SQL_IDENTIFIER = String.raw`(?:"[^"]+"|'[^']+'|` + '`[^`]+`' + String.raw`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_]*)`
|
||||
const SQL_TABLE_REF_END = String.raw`(?=\s|$|[,;])`
|
||||
const SQL_KEYWORDS = new Set([
|
||||
'where',
|
||||
'join',
|
||||
'left',
|
||||
'right',
|
||||
'inner',
|
||||
'outer',
|
||||
'cross',
|
||||
'order',
|
||||
'group',
|
||||
'limit',
|
||||
'on',
|
||||
'using',
|
||||
])
|
||||
|
||||
function sqlNamePattern(name: string): string {
|
||||
const escaped = escapeRegExp(name)
|
||||
return String.raw`(?:"${escaped}"|'${escaped}'|` + '`' + escaped + '`' + String.raw`|\[${escaped}\]|${escaped})`
|
||||
}
|
||||
|
||||
function unquoteSqlIdentifier(value: string): string {
|
||||
const trimmed = value.trim()
|
||||
if (trimmed.startsWith('[') && trimmed.endsWith(']')) return trimmed.slice(1, -1)
|
||||
const quote = trimmed[0]
|
||||
if ((quote === '"' || quote === "'" || quote === '`') && trimmed.endsWith(quote)) {
|
||||
return trimmed.slice(1, -1)
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
function messageTablePattern(): string {
|
||||
const schemaPrefix = String.raw`(?:${SQL_IDENTIFIER}\s*\.\s*)?`
|
||||
return `${schemaPrefix}${sqlNamePattern('message')}`
|
||||
}
|
||||
|
||||
function selectsMessageStar(query: string): boolean {
|
||||
const selectMatch = query.match(/\bselect\b([\s\S]+?)\bfrom\b/i)
|
||||
if (!selectMatch) return false
|
||||
|
||||
const projection = selectMatch[1]
|
||||
const messageTable = messageTablePattern()
|
||||
if (/(^|[,\s])\*(?=$|[,\s])/i.test(projection)) {
|
||||
return new RegExp(String.raw`\b(?:from|join)\s+${messageTable}${SQL_TABLE_REF_END}`, 'i').test(query)
|
||||
}
|
||||
|
||||
const aliasMatches = query.matchAll(
|
||||
new RegExp(String.raw`\b(?:from|join)\s+${messageTable}(?:\s+(?:as\s+)?(${SQL_IDENTIFIER}))?`, 'gi')
|
||||
)
|
||||
const aliases = new Set(['message'])
|
||||
for (const match of aliasMatches) {
|
||||
const alias = match[1] ? unquoteSqlIdentifier(match[1]).toLowerCase() : undefined
|
||||
if (alias && !SQL_KEYWORDS.has(alias)) aliases.add(alias)
|
||||
}
|
||||
|
||||
return [...aliases].some((alias) =>
|
||||
new RegExp(`${sqlNamePattern(alias)}\\s*\\.\\s*\\*(?=$|[\\s,])`, 'i').test(projection)
|
||||
)
|
||||
}
|
||||
|
||||
export function assertSqlPrivacyAllowed(query: string, raw: boolean): void {
|
||||
if (raw) return
|
||||
if (!/(^|[^A-Za-z0-9_])["'`[]?content["'`\]]?(?![A-Za-z0-9_])/i.test(query) && !selectsMessageStar(query)) {
|
||||
return
|
||||
}
|
||||
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'SQL queries that read message content require --raw',
|
||||
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
|
||||
})
|
||||
}
|
||||
|
||||
export function assertSqlResultPrivacyAllowed(columns: readonly string[], raw: boolean): void {
|
||||
if (raw) return
|
||||
if (!columns.some((column) => unquoteSqlIdentifier(column).toLowerCase() === 'content')) return
|
||||
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'SQL results that include message content require --raw',
|
||||
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
|
||||
})
|
||||
}
|
||||
|
||||
export interface TopicListItem {
|
||||
id: number
|
||||
since: string
|
||||
@@ -209,7 +302,7 @@ export function registerTopicCommands(program: Command): void {
|
||||
export function registerSqlCommands(program: Command): void {
|
||||
program
|
||||
.command('sql')
|
||||
.description('Read-only SQL fallback (string cells are desensitized by default)')
|
||||
.description('Read-only SQL fallback (string cells are desensitized; message content requires --raw)')
|
||||
.argument('<query>', 'SELECT / WITH statement')
|
||||
.option('--session <ref>', 'Session id or unique name')
|
||||
.option('--limit <n>', 'Max rows (default 100, max 1000)')
|
||||
@@ -227,7 +320,8 @@ export function registerSqlCommands(program: Command): void {
|
||||
})
|
||||
}
|
||||
assertRawAllowed(ctx, options)
|
||||
const limit = parseLimit(options.limit, 100, 1000, '--limit')
|
||||
assertSqlPrivacyAllowed(query, !!options.raw)
|
||||
const limit = parseSqlRowLimit(options.limit)
|
||||
|
||||
let result
|
||||
try {
|
||||
@@ -239,6 +333,7 @@ export function registerSqlCommands(program: Command): void {
|
||||
hint: 'Run `chatlab schema` to inspect available tables and columns',
|
||||
})
|
||||
}
|
||||
assertSqlResultPrivacyAllowed(result.columns, !!options.raw)
|
||||
|
||||
let rows = result.rows
|
||||
let removed = 0
|
||||
|
||||
@@ -66,11 +66,11 @@ const EXAMPLES: ManifestExample[] = [
|
||||
command: 'chatlab messages search 旅游 --since 2026-01-01 --limit 20 --format agent',
|
||||
},
|
||||
{
|
||||
task: 'Who mentioned it first (evidence ids)?',
|
||||
task: 'Who mentioned it first (message ids)?',
|
||||
command: 'chatlab messages search 报销 --sort asc --limit 5 --format agent',
|
||||
},
|
||||
{
|
||||
task: 'Follow up a [#id] citation with context',
|
||||
task: 'Follow up a single [#id] marker with context',
|
||||
command: 'chatlab messages context --id 1021 --window 10 --format agent',
|
||||
},
|
||||
{
|
||||
@@ -92,7 +92,7 @@ const NOTES: string[] = [
|
||||
'stdout carries exactly one JSON envelope in agent/json mode; logs go to stderr.',
|
||||
'Success envelope: { ok, command, data, meta }; failure: { ok: false, command, error }.',
|
||||
'Pagination: follow meta.nextCursor with --cursor; cursors are bound to the exact query.',
|
||||
'[#id] markers in agent text are message ids usable with `messages context --id`.',
|
||||
'Single-message [#id] and [#id*] markers are usable with `messages context --id`; merged [#a-b] ranges are display-only.',
|
||||
'Privacy preprocessing (desensitize/blacklist) is always on by default.',
|
||||
]
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
* Shared output shaping for message-carrying query commands.
|
||||
*
|
||||
* agent format: AI-facing text from the shared preprocessing pipeline with
|
||||
* [#id] citations (design §6.2). json format: structured items with privacy
|
||||
* [#id] / [#id*] message markers and display-only [#a-b] merged ranges.
|
||||
* json format: structured items with privacy
|
||||
* steps applied (cleaning/blacklist/desensitize) but no merging/denoising.
|
||||
*/
|
||||
|
||||
|
||||
@@ -6,7 +6,15 @@
|
||||
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor, epochToIso } from './parse'
|
||||
import {
|
||||
parseTimeOptions,
|
||||
parseLimit,
|
||||
queryFingerprint,
|
||||
encodeCursor,
|
||||
decodeCursor,
|
||||
epochToIso,
|
||||
resolveTimeOptionsForCursor,
|
||||
} from './parse'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
// Fixed reference: 2026-07-03 15:30:00 local time
|
||||
@@ -60,6 +68,22 @@ describe('parseTimeOptions', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects invalid calendar dates instead of normalizing them', () => {
|
||||
for (const bad of [
|
||||
{ since: '2026-02-31' },
|
||||
{ until: '2026-13-01' },
|
||||
{ since: '2026-02-31 08:30' },
|
||||
{ since: '2026-06-01 25:00' },
|
||||
{ until: '2026-02-31T10:00:00+08:00' },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => parseTimeOptions(bad, NOW),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT',
|
||||
JSON.stringify(bad)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects inverted ranges', () => {
|
||||
assert.throws(
|
||||
() => parseTimeOptions({ since: '2026-07-01', until: '2026-06-01' }, NOW),
|
||||
@@ -90,6 +114,14 @@ describe('parseLimit', () => {
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('can reject zero for cursor-paginated page sizes', () => {
|
||||
assert.equal(parseLimit('0', 0, 50, '--context'), 0)
|
||||
assert.throws(
|
||||
() => parseLimit('0', 50, 500, '--limit', 1),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('cursor', () => {
|
||||
@@ -116,4 +148,19 @@ describe('cursor', () => {
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'CURSOR_INVALID'
|
||||
)
|
||||
})
|
||||
|
||||
it('reuses frozen time bounds embedded in cursor tokens', () => {
|
||||
const firstRange = resolveTimeOptionsForCursor({ last: '7d' }, NOW)
|
||||
const fp = queryFingerprint({
|
||||
command: 'messages.list',
|
||||
startTs: firstRange.startTs ?? null,
|
||||
endTs: firstRange.endTs ?? null,
|
||||
})
|
||||
const token = encodeCursor(50, fp, { time: firstRange })
|
||||
|
||||
const later = new Date(NOW.getTime() + 60_000)
|
||||
const secondRange = resolveTimeOptionsForCursor({ last: '7d', cursor: token }, later)
|
||||
assert.deepEqual(secondRange, firstRange)
|
||||
assert.equal(decodeCursor(token, fp), 50)
|
||||
})
|
||||
})
|
||||
+145
-9
@@ -23,6 +23,10 @@ export interface ResolvedTimeRange {
|
||||
meta: { since: string | null; until: string | null }
|
||||
}
|
||||
|
||||
export interface CursorSnapshot {
|
||||
time?: ResolvedTimeRange
|
||||
}
|
||||
|
||||
/** Format a Date as ISO 8601 with the local UTC offset (sortable, self-describing). */
|
||||
export function toIsoWithOffset(date: Date): string {
|
||||
const pad = (n: number, w = 2) => String(Math.abs(n)).padStart(w, '0')
|
||||
@@ -43,6 +47,7 @@ export function epochToIso(ts: number): string {
|
||||
|
||||
const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/
|
||||
const DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?$/
|
||||
const ISO_DATE_PREFIX = /^(\d{4})-(\d{2})-(\d{2})T/
|
||||
const LAST_DURATION = /^(\d+)([hdw])$/
|
||||
|
||||
function startOfDay(base: Date, dayOffset = 0): Date {
|
||||
@@ -57,6 +62,14 @@ function invalidTime(flag: string, value: string): QueryError {
|
||||
})
|
||||
}
|
||||
|
||||
function isSameLocalDate(date: Date, y: number, m: number, d: number): boolean {
|
||||
return date.getFullYear() === y && date.getMonth() === m - 1 && date.getDate() === d
|
||||
}
|
||||
|
||||
function isSameLocalDateTime(date: Date, y: number, m: number, d: number, h: number, min: number, s: number): boolean {
|
||||
return isSameLocalDate(date, y, m, d) && date.getHours() === h && date.getMinutes() === min && date.getSeconds() === s
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single time value. Date-only values resolve to the start of the day;
|
||||
* when `boundary` is 'end' they resolve to the end of the day (inclusive whole-day).
|
||||
@@ -71,23 +84,43 @@ function parseTimeValue(value: string, boundary: 'start' | 'end', flag: string,
|
||||
const dateOnly = DATE_ONLY.exec(value)
|
||||
if (dateOnly) {
|
||||
const [, y, m, d] = dateOnly
|
||||
const base = new Date(Number(y), Number(m) - 1, Number(d))
|
||||
if (Number.isNaN(base.getTime())) throw invalidTime(flag, value)
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const base = new Date(year, month - 1, day)
|
||||
if (Number.isNaN(base.getTime()) || !isSameLocalDate(base, year, month, day)) throw invalidTime(flag, value)
|
||||
if (boundary === 'start') return base
|
||||
return new Date(new Date(Number(y), Number(m) - 1, Number(d) + 1).getTime() - 1000)
|
||||
return new Date(new Date(year, month - 1, day + 1).getTime() - 1000)
|
||||
}
|
||||
|
||||
const dateTime = DATE_TIME.exec(value)
|
||||
if (dateTime) {
|
||||
const [, y, m, d, h, min, s] = dateTime
|
||||
const parsed = new Date(Number(y), Number(m) - 1, Number(d), Number(h), Number(min), Number(s ?? 0))
|
||||
if (Number.isNaN(parsed.getTime())) throw invalidTime(flag, value)
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const hour = Number(h)
|
||||
const minute = Number(min)
|
||||
const second = Number(s ?? 0)
|
||||
const parsed = new Date(year, month - 1, day, hour, minute, second)
|
||||
if (Number.isNaN(parsed.getTime()) || !isSameLocalDateTime(parsed, year, month, day, hour, minute, second)) {
|
||||
throw invalidTime(flag, value)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// full ISO 8601 (with timezone offset or Z)
|
||||
const parsed = new Date(value)
|
||||
if (!Number.isNaN(parsed.getTime()) && /\d{4}-\d{2}-\d{2}T/.test(value)) {
|
||||
const isoDate = ISO_DATE_PREFIX.exec(value)
|
||||
if (isoDate) {
|
||||
const [, y, m, d] = isoDate
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const localDate = new Date(year, month - 1, day)
|
||||
if (!isSameLocalDate(localDate, year, month, day)) throw invalidTime(flag, value)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
@@ -161,15 +194,19 @@ export function parseLimit(
|
||||
value: string | number | undefined,
|
||||
defaultValue: number,
|
||||
cap: number,
|
||||
flag: string
|
||||
flag: string,
|
||||
min = 0
|
||||
): number {
|
||||
if (value === undefined) return Math.min(defaultValue, cap)
|
||||
const n = typeof value === 'number' ? value : Number(value)
|
||||
if (!Number.isInteger(n) || n < 0) {
|
||||
if (!Number.isInteger(n) || n < min) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid ${flag} value: ${value}`,
|
||||
hint: `${flag} must be a non-negative integer (max ${cap})`,
|
||||
hint:
|
||||
min > 0
|
||||
? `${flag} must be an integer from ${min} to ${cap}`
|
||||
: `${flag} must be a non-negative integer (max ${cap})`,
|
||||
})
|
||||
}
|
||||
return Math.min(n, cap)
|
||||
@@ -183,12 +220,111 @@ export function queryFingerprint(conditions: Record<string, unknown>): string {
|
||||
return createHash('sha256').update(canonical).digest('hex').slice(0, 12)
|
||||
}
|
||||
|
||||
export function encodeCursor(offset: number, fingerprint: string): string {
|
||||
export function freezeTimeRangeForCursor(time: ResolvedTimeRange): ResolvedTimeRange {
|
||||
if (time.endTs !== undefined || time.meta.until === null) return time
|
||||
const endMs = new Date(time.meta.until).getTime()
|
||||
if (Number.isNaN(endMs)) return time
|
||||
return { ...time, endTs: Math.floor(endMs / 1000) }
|
||||
}
|
||||
|
||||
export function resolveTimeOptionsForCursor(
|
||||
options: TimeCliOptions & { cursor?: string },
|
||||
now = new Date()
|
||||
): ResolvedTimeRange {
|
||||
const snapshot = options.cursor ? getCursorSnapshot(options.cursor) : undefined
|
||||
return snapshot?.time ?? freezeTimeRangeForCursor(parseTimeOptions(options, now))
|
||||
}
|
||||
|
||||
export function encodeCursor(offset: number, fingerprint: string, snapshot?: CursorSnapshot): string {
|
||||
if (snapshot?.time) {
|
||||
return Buffer.from(JSON.stringify({ v: 2, fp: fingerprint, offset, time: snapshot.time }), 'utf8').toString(
|
||||
'base64url'
|
||||
)
|
||||
}
|
||||
return Buffer.from(`${fingerprint}:${offset}`, 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
function asObject(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null
|
||||
}
|
||||
|
||||
function decodeCursorEnvelope(
|
||||
token: string
|
||||
): { fingerprint: string; offset: number; snapshot?: CursorSnapshot } | null {
|
||||
let decoded = ''
|
||||
try {
|
||||
decoded = Buffer.from(token, 'base64url').toString('utf8')
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
let payload: unknown
|
||||
try {
|
||||
payload = JSON.parse(decoded)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
const object = asObject(payload)
|
||||
if (
|
||||
!object ||
|
||||
object.v !== 2 ||
|
||||
typeof object.fp !== 'string' ||
|
||||
typeof object.offset !== 'number' ||
|
||||
!Number.isInteger(object.offset) ||
|
||||
object.offset < 0
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
const snapshot = decodeCursorSnapshot(object)
|
||||
return { fingerprint: object.fp, offset: object.offset, ...(snapshot ? { snapshot } : {}) }
|
||||
}
|
||||
|
||||
function decodeCursorSnapshot(payload: Record<string, unknown>): CursorSnapshot | undefined {
|
||||
const timeObject = asObject(payload.time)
|
||||
if (!timeObject) return undefined
|
||||
const metaObject = asObject(timeObject.meta)
|
||||
if (!metaObject) return undefined
|
||||
|
||||
const startTs = timeObject.startTs
|
||||
const endTs = timeObject.endTs
|
||||
const since = metaObject.since
|
||||
const until = metaObject.until
|
||||
if (startTs !== undefined && typeof startTs !== 'number') return undefined
|
||||
if (endTs !== undefined && typeof endTs !== 'number') return undefined
|
||||
if (since !== null && typeof since !== 'string') return undefined
|
||||
if (until !== null && typeof until !== 'string') return undefined
|
||||
|
||||
return {
|
||||
time: {
|
||||
...(startTs !== undefined ? { startTs } : {}),
|
||||
...(endTs !== undefined ? { endTs } : {}),
|
||||
meta: { since, until },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function getCursorSnapshot(token: string): CursorSnapshot | undefined {
|
||||
return decodeCursorEnvelope(token)?.snapshot
|
||||
}
|
||||
|
||||
/** Decode a cursor and validate it against the current query fingerprint. */
|
||||
export function decodeCursor(token: string, fingerprint: string): number {
|
||||
const envelope = decodeCursorEnvelope(token)
|
||||
if (envelope) {
|
||||
if (envelope.fingerprint !== fingerprint) {
|
||||
throw new QueryError({
|
||||
code: 'CURSOR_INVALID',
|
||||
message: 'Cursor does not match the current query conditions',
|
||||
hint: 'Cursors are only valid for the exact query that produced them; re-run the query without --cursor',
|
||||
})
|
||||
}
|
||||
return envelope.offset
|
||||
}
|
||||
|
||||
let decoded = ''
|
||||
try {
|
||||
decoded = Buffer.from(token, 'base64url').toString('utf8')
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import type { ChatLabConfig } from '@openchatlab/config'
|
||||
|
||||
import { assertLegacySqlAllowed } from './register'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
function configWithSql(allowSql: boolean): ChatLabConfig {
|
||||
return {
|
||||
llm: { provider: '', model: '', base_url: '' },
|
||||
data: { user_data_dir: '', electron_migration_done: false },
|
||||
api: { port: 3110, host: '127.0.0.1', token: '', require_auth: false },
|
||||
locale: { lang: '' },
|
||||
ui: { default_session_tab: 'overview', session_gap_threshold: 1800, summary_strategy: 'standard' },
|
||||
cli: { allow_raw: false, allow_sql: allowSql },
|
||||
}
|
||||
}
|
||||
|
||||
describe('legacy query alias SQL gate', () => {
|
||||
it('rejects SQL when cli.allow_sql is disabled', () => {
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(false)),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'SQL_DISABLED'
|
||||
)
|
||||
})
|
||||
|
||||
it('allows SQL when cli.allow_sql is enabled', () => {
|
||||
assert.doesNotThrow(() => assertLegacySqlAllowed(configWithSql(true)))
|
||||
})
|
||||
|
||||
it('rejects message content SQL because the legacy alias has no raw mode', () => {
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT content FROM message'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT * FROM message'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -4,12 +4,19 @@
|
||||
*/
|
||||
|
||||
import type { Command } from 'commander'
|
||||
import type { ChatLabConfig } from '@openchatlab/config'
|
||||
import { searchMessagesLike, executeReadonlySql } from '@openchatlab/core'
|
||||
import { initRuntime } from '../runtime'
|
||||
import { registerSessionCommands, registerMemberCommands } from './commands-sessions-members'
|
||||
import { registerMessageCommands } from './commands-messages'
|
||||
import { registerStatsCommands } from './commands-stats'
|
||||
import { registerTopicCommands, registerSqlCommands } from './commands-topics-sql'
|
||||
import {
|
||||
registerTopicCommands,
|
||||
registerSqlCommands,
|
||||
assertSqlPrivacyAllowed,
|
||||
assertSqlResultPrivacyAllowed,
|
||||
} from './commands-topics-sql'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
export function registerQueryCommands(program: Command): void {
|
||||
registerSessionCommands(program)
|
||||
@@ -21,6 +28,17 @@ export function registerQueryCommands(program: Command): void {
|
||||
registerLegacyAliases(program)
|
||||
}
|
||||
|
||||
export function assertLegacySqlAllowed(config: Pick<ChatLabConfig, 'cli'>, sql?: string): void {
|
||||
if (!config.cli.allow_sql) {
|
||||
throw new QueryError({
|
||||
code: 'SQL_DISABLED',
|
||||
message: 'The sql command is disabled by configuration',
|
||||
hint: 'The user can re-enable it with `chatlab config set cli.allow_sql true`',
|
||||
})
|
||||
}
|
||||
if (sql) assertSqlPrivacyAllowed(sql, false)
|
||||
}
|
||||
|
||||
// ==================== Hidden legacy aliases (output kept byte-compatible) ====================
|
||||
|
||||
function registerLegacyAliases(program: Command): void {
|
||||
@@ -67,7 +85,7 @@ function registerLegacyAliases(program: Command): void {
|
||||
console.error(
|
||||
'[Deprecated] `chatlab query` — use `chatlab sql "<SELECT ...>" --session <id>`; alias kept for one major version'
|
||||
)
|
||||
const { dbManager } = initRuntime()
|
||||
const { config, dbManager } = initRuntime()
|
||||
const db = dbManager.open(sessionId)
|
||||
if (!db) {
|
||||
console.error(`Session ${sessionId} not found`)
|
||||
@@ -75,7 +93,9 @@ function registerLegacyAliases(program: Command): void {
|
||||
}
|
||||
|
||||
try {
|
||||
assertLegacySqlAllowed(config, options.sql)
|
||||
const result = executeReadonlySql(db, options.sql)
|
||||
assertSqlResultPrivacyAllowed(result.columns, false)
|
||||
if (options.format === 'json') {
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
} else {
|
||||
@@ -87,6 +107,11 @@ function registerLegacyAliases(program: Command): void {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof QueryError) {
|
||||
console.error(`${err.code}: ${err.message}`)
|
||||
if (err.hint) console.error(`Hint: ${err.hint}`)
|
||||
process.exit(2)
|
||||
}
|
||||
console.error(`SQL error: ${err instanceof Error ? err.message : err}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user