fix(cli): harden agent query safeguards

Tighten agent-facing query validation, privacy guards, SQL result handling, config writes, and LIKE search semantics for the CLI query workflow.
This commit is contained in:
digua committed 2026-07-04 15:12:50 +08:00
1 parent b33cbd065d
commit 801e7fc66a
32 files changed
+880 -97

No files matched your search

@@ -0,0 +1,71 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import {
assertContextAnchorsPresent,
capExpandedSearchMessages,
parseContextIds,
parseSearchKeywords,
searchTruncationStrategy,
} from './commands-messages'
import type { MessageLike } from './messages-output'
describe('capExpandedSearchMessages', () => {
it('keeps hit messages when the cap is smaller than pre-context', () => {
const messages: MessageLike[] = [
{ id: 10, senderName: 'Alice', content: 'before 1', timestamp: 1710000000 },
{ id: 11, senderName: 'Alice', content: 'before 2', timestamp: 1710000001 },
{ id: 12, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 },
]
const capped = capExpandedSearchMessages(messages, new Set([12]), 1)
assert.deepEqual(
capped.map((message) => message.id),
[12]
)
})
})
describe('parseContextIds', () => {
it('rejects blank id tokens instead of converting them to 0', () => {
assert.throws(() => parseContextIds('1021,'), /Invalid --id value/)
assert.throws(() => parseContextIds('1021,,1058'), /Invalid --id value/)
})
it('requires positive numeric message ids', () => {
assert.deepEqual(parseContextIds('1021,1058'), [1021, 1058])
assert.throws(() => parseContextIds('0'), /Invalid --id value/)
})
})
describe('parseSearchKeywords', () => {
it('trims keywords and rejects blank-only searches', () => {
assert.deepEqual(parseSearchKeywords([' alpha ', 'beta']), ['alpha', 'beta'])
assert.throws(() => parseSearchKeywords([' ']), /Invalid search keywords/)
})
})
describe('searchTruncationStrategy', () => {
it('keeps newest chronological messages when search sort is descending', () => {
assert.equal(searchTruncationStrategy('desc'), 'keep_last')
assert.equal(searchTruncationStrategy('asc'), 'keep_first')
})
})
describe('assertContextAnchorsPresent', () => {
it('rejects context results that do not include every requested anchor id', () => {
assert.throws(
() =>
assertContextAnchorsPresent(
[999],
[
{ id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000000 },
{ id: 11, senderName: 'Bob', content: 'after', timestamp: 1710000001 },
],
'999'
),
/No messages found/
)
})
})
+86 -31
View File
@@ -16,7 +16,7 @@ import {
import { runQuery } from './runner'
import { createQueryContext, assertRawAllowed } from './context'
import { resolveMember } from './resolve'
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor } from './parse'
import { parseLimit, queryFingerprint, encodeCursor, decodeCursor, resolveTimeOptionsForCursor } from './parse'
import { QueryError } from './envelope'
import { buildMessagesResult, assertRawFormatCompatible, type MessageLike } from './messages-output'
@@ -37,6 +37,73 @@ interface CommonMessageOptions {
verbose?: boolean
}
export function capExpandedSearchMessages(
messages: MessageLike[],
hitIds: ReadonlySet<number>,
maxMessages: number
): MessageLike[] {
if (maxMessages <= 0) return []
if (messages.length <= maxMessages) return messages
const capped = messages.slice(0, maxMessages)
const includedIds = new Set(capped.map((message) => message.id))
const missingHits = messages.filter((message) => hitIds.has(message.id) && !includedIds.has(message.id))
for (const hit of missingHits) {
let replaceIndex = -1
for (let i = capped.length - 1; i >= 0; i--) {
if (!hitIds.has(capped[i].id)) {
replaceIndex = i
break
}
}
if (replaceIndex === -1) break
capped[replaceIndex] = hit
}
return capped.sort((a, b) => messages.indexOf(a) - messages.indexOf(b))
}
export function parseContextIds(value: string): number[] {
const tokens = value.split(',').map((token) => token.trim())
const ids = tokens.map((token) => (token.length > 0 ? Number(token) : Number.NaN))
if (ids.length === 0 || ids.some((id) => !Number.isInteger(id) || id <= 0)) {
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: `Invalid --id value: ${value}`,
hint: 'Pass positive numeric message ids, e.g. --id 1021 or --id 1021,1058',
})
}
return ids
}
export function parseSearchKeywords(values: string[]): string[] {
const keywords = values.map((value) => value.trim()).filter(Boolean)
if (keywords.length === 0) {
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: 'Invalid search keywords',
hint: 'Pass at least one non-empty search keyword',
})
}
return keywords
}
export function searchTruncationStrategy(sort: string | undefined): 'keep_first' | 'keep_last' {
return sort === 'desc' ? 'keep_last' : 'keep_first'
}
export function assertContextAnchorsPresent(ids: number[], messages: MessageLike[], rawValue: string): void {
const returnedIds = new Set(messages.map((message) => message.id))
if (ids.some((id) => !returnedIds.has(id))) {
throw new QueryError({
code: 'MESSAGE_NOT_FOUND',
message: `No messages found for id(s) ${rawValue}`,
hint: 'Use a single numeric id from [#id]/[#id*] markers; merged [#a-b] ranges are display-only',
})
}
}
function addSharedOptions(cmd: Command): Command {
return cmd
.option('--session <ref>', 'Session id or unique name (auto-selected when only one exists)')
@@ -70,9 +137,9 @@ export function registerMessageCommands(program: Command): void {
const ctx = createQueryContext(options)
try {
assertRawAllowed(ctx, options)
const time = parseTimeOptions(options)
const time = resolveTimeOptionsForCursor(options)
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
const limit = parseLimit(options.limit, 50, 500, '--limit')
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
const fingerprint = queryFingerprint({
@@ -104,7 +171,7 @@ export function registerMessageCommands(program: Command): void {
totalHits: total,
returnedHits: result.messages.length,
hasMore,
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint) } : {}),
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint, { time }) } : {}),
}
return buildMessagesResult(format, ctx, options, result.messages, meta, {
@@ -142,6 +209,7 @@ export function registerMessageCommands(program: Command): void {
) => {
await runQuery('messages.search', options, async (format) => {
assertRawFormatCompatible(format, options)
const searchKeywords = parseSearchKeywords(keywords)
if (options.match !== 'any' && options.match !== 'all') {
throw new QueryError({
code: 'INVALID_ARGUMENT',
@@ -159,9 +227,9 @@ export function registerMessageCommands(program: Command): void {
const ctx = createQueryContext(options)
try {
assertRawAllowed(ctx, options)
const time = parseTimeOptions(options)
const time = resolveTimeOptionsForCursor(options)
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
const limit = parseLimit(options.limit, 20, 500, '--limit')
const limit = parseLimit(options.limit, 20, 500, '--limit', 1)
const context = parseLimit(options.context, 0, 50, '--context')
const maxMessages = parseLimit(options.maxMessages, 200, 2000, '--max-messages')
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
@@ -169,7 +237,7 @@ export function registerMessageCommands(program: Command): void {
const fingerprint = queryFingerprint({
command: 'messages.search',
session: ctx.session.id,
keywords,
keywords: searchKeywords,
match: options.match,
sort: options.sort,
startTs: time.startTs ?? null,
@@ -179,7 +247,7 @@ export function registerMessageCommands(program: Command): void {
})
const offset = options.cursor ? decodeCursor(options.cursor, fingerprint) : 0
const result = searchMessagesByKeywords(ctx.db, keywords, {
const result = searchMessagesByKeywords(ctx.db, searchKeywords, {
startTs: time.startTs,
endTs: time.endTs,
senderId: member?.id,
@@ -215,7 +283,7 @@ export function registerMessageCommands(program: Command): void {
const tail = hits.filter((m) => !keptHits.has(m.id))
pipelineMessages = [...expanded, ...tail]
if (pipelineMessages.length > maxMessages) {
pipelineMessages = pipelineMessages.slice(0, maxMessages)
pipelineMessages = capExpandedSearchMessages(pipelineMessages, hitIds, maxMessages)
warnings.push(`response capped at ${maxMessages} messages`)
}
}
@@ -225,18 +293,18 @@ export function registerMessageCommands(program: Command): void {
const meta: Record<string, unknown> = {
session: ctx.session,
timeRange: time.meta,
query: { keywords, match: options.match, engine: 'like', sort: options.sort, context },
query: { keywords: searchKeywords, match: options.match, engine: 'like', sort: options.sort, context },
...(member ? { member } : {}),
totalHits: total,
returnedHits: hits.length,
hasMore,
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint) } : {}),
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint, { time }) } : {}),
...(warnings.length > 0 ? { warnings } : {}),
}
return buildMessagesResult(format, ctx, options, pipelineMessages, meta, {
hitIds,
strategy: 'keep_first',
strategy: searchTruncationStrategy(options.sort),
defaultMaxChars: 120,
})
} finally {
@@ -249,33 +317,20 @@ export function registerMessageCommands(program: Command): void {
// ---------- messages context ----------
const contextCmd = messagesCmd
.command('context')
.description('Show messages around specific message ids (evidence follow-up)')
.description('Show messages around specific numeric message ids')
.requiredOption('--id <ids>', 'Message id(s), comma-separated')
.option('--window <n>', 'Messages before/after each id (default 10, max 100)')
addSharedOptions(contextCmd)
contextCmd.action(async (options: CommonMessageOptions & { id: string; window?: string }) => {
await runQuery('messages.context', options, async (format) => {
assertRawFormatCompatible(format, options)
const ids = options.id.split(',').map((s) => Number(s.trim()))
if (ids.length === 0 || ids.some((n) => !Number.isInteger(n) || n < 0)) {
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: `Invalid --id value: ${options.id}`,
hint: 'Pass numeric message ids, e.g. --id 1021 or --id 1021,1058',
})
}
const ids = parseContextIds(options.id)
const ctx = createQueryContext(options)
try {
assertRawAllowed(ctx, options)
const window = parseLimit(options.window, 10, 100, '--window')
const messages = getMessageContext(ctx.db, ids, window)
if (messages.length === 0) {
throw new QueryError({
code: 'MESSAGE_NOT_FOUND',
message: `No messages found for id(s) ${options.id}`,
hint: 'Message ids come from [#id] markers in search results',
})
}
assertContextAnchorsPresent(ids, messages, options.id)
const meta: Record<string, unknown> = {
session: ctx.session,
anchorIds: ids,
@@ -312,10 +367,10 @@ export function registerMessageCommands(program: Command): void {
const ctx = createQueryContext(options)
try {
assertRawAllowed(ctx, options)
const time = parseTimeOptions(options)
const time = resolveTimeOptionsForCursor(options)
const memberA = resolveMember(ctx.db, options.member[0])
const memberB = resolveMember(ctx.db, options.member[1])
const limit = parseLimit(options.limit, 50, 500, '--limit')
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
const fingerprint = queryFingerprint({
@@ -346,7 +401,7 @@ export function registerMessageCommands(program: Command): void {
totalHits: result.total,
returnedHits: returned,
hasMore,
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint) } : {}),
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint, { time }) } : {}),
}
return buildMessagesResult(format, ctx, options, result.messages, meta, {
+60 -1
View File
@@ -6,7 +6,14 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { topicsAgentText, type TopicListItem } from './commands-topics-sql'
import {
assertSqlPrivacyAllowed,
assertSqlResultPrivacyAllowed,
parseSqlRowLimit,
topicsAgentText,
type TopicListItem,
} from './commands-topics-sql'
import { QueryError } from './envelope'
const items: TopicListItem[] = [
{
@@ -41,3 +48,55 @@ describe('topicsAgentText', () => {
assert.equal(topicsAgentText([]), 'No summaries.')
})
})
describe('parseSqlRowLimit', () => {
it('rejects zero because SQL maxRows=0 means unlimited', () => {
assert.throws(
() => parseSqlRowLimit('0'),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
})
})
describe('assertSqlPrivacyAllowed', () => {
it('requires raw mode before SQL can read message content', () => {
assert.throws(
() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertSqlPrivacyAllowed('SELECT * FROM message LIMIT 10', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertSqlPrivacyAllowed('SELECT * FROM main.message LIMIT 10', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertSqlPrivacyAllowed('SELECT msg.* FROM message msg LIMIT 10', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertSqlPrivacyAllowed('SELECT "message".* FROM message LIMIT 10', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertSqlPrivacyAllowed('SELECT "m".* FROM message AS "m" LIMIT 10', false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT COUNT(*) AS total FROM message', false))
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT * FROM sqlite_master', false))
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', true))
})
})
describe('assertSqlResultPrivacyAllowed', () => {
it('requires raw mode when SQL result columns expose message content', () => {
assert.throws(
() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], false),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], true))
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id'], false))
})
})
+100 -5
View File
@@ -2,9 +2,9 @@
* topics / sql / schema query commands.
*
* Derived-text privacy (design §6.2): topic summaries are LLM-generated from raw
* content, and SQL results can carry message bodies — both are desensitized by
* default, with blacklist-matching rows dropped and counted into meta.warnings.
* --raw (user-gated) and cli.allow_sql are the only escape hatches.
* content and are desensitized by default. SQL string cells are desensitized,
* but reading the message content column requires --raw so SQL expressions
* cannot encode raw bodies before the sanitizer sees them.
*/
import type { Command } from 'commander'
@@ -20,6 +20,99 @@ function enabledRules(ctx: QueryContext): DesensitizeRule[] {
return ctx.preprocessConfig.desensitize ? ctx.preprocessConfig.desensitizeRules.filter((r) => r.enabled) : []
}
export function parseSqlRowLimit(value: string | undefined): number {
return parseLimit(value, 100, 1000, '--limit', 1)
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
const SQL_IDENTIFIER = String.raw`(?:"[^"]+"|'[^']+'|` + '`[^`]+`' + String.raw`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_]*)`
const SQL_TABLE_REF_END = String.raw`(?=\s|$|[,;])`
const SQL_KEYWORDS = new Set([
'where',
'join',
'left',
'right',
'inner',
'outer',
'cross',
'order',
'group',
'limit',
'on',
'using',
])
function sqlNamePattern(name: string): string {
const escaped = escapeRegExp(name)
return String.raw`(?:"${escaped}"|'${escaped}'|` + '`' + escaped + '`' + String.raw`|\[${escaped}\]|${escaped})`
}
function unquoteSqlIdentifier(value: string): string {
const trimmed = value.trim()
if (trimmed.startsWith('[') && trimmed.endsWith(']')) return trimmed.slice(1, -1)
const quote = trimmed[0]
if ((quote === '"' || quote === "'" || quote === '`') && trimmed.endsWith(quote)) {
return trimmed.slice(1, -1)
}
return trimmed
}
function messageTablePattern(): string {
const schemaPrefix = String.raw`(?:${SQL_IDENTIFIER}\s*\.\s*)?`
return `${schemaPrefix}${sqlNamePattern('message')}`
}
function selectsMessageStar(query: string): boolean {
const selectMatch = query.match(/\bselect\b([\s\S]+?)\bfrom\b/i)
if (!selectMatch) return false
const projection = selectMatch[1]
const messageTable = messageTablePattern()
if (/(^|[,\s])\*(?=$|[,\s])/i.test(projection)) {
return new RegExp(String.raw`\b(?:from|join)\s+${messageTable}${SQL_TABLE_REF_END}`, 'i').test(query)
}
const aliasMatches = query.matchAll(
new RegExp(String.raw`\b(?:from|join)\s+${messageTable}(?:\s+(?:as\s+)?(${SQL_IDENTIFIER}))?`, 'gi')
)
const aliases = new Set(['message'])
for (const match of aliasMatches) {
const alias = match[1] ? unquoteSqlIdentifier(match[1]).toLowerCase() : undefined
if (alias && !SQL_KEYWORDS.has(alias)) aliases.add(alias)
}
return [...aliases].some((alias) =>
new RegExp(`${sqlNamePattern(alias)}\\s*\\.\\s*\\*(?=$|[\\s,])`, 'i').test(projection)
)
}
export function assertSqlPrivacyAllowed(query: string, raw: boolean): void {
if (raw) return
if (!/(^|[^A-Za-z0-9_])["'`[]?content["'`\]]?(?![A-Za-z0-9_])/i.test(query) && !selectsMessageStar(query)) {
return
}
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: 'SQL queries that read message content require --raw',
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
})
}
export function assertSqlResultPrivacyAllowed(columns: readonly string[], raw: boolean): void {
if (raw) return
if (!columns.some((column) => unquoteSqlIdentifier(column).toLowerCase() === 'content')) return
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: 'SQL results that include message content require --raw',
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
})
}
export interface TopicListItem {
id: number
since: string
@@ -209,7 +302,7 @@ export function registerTopicCommands(program: Command): void {
export function registerSqlCommands(program: Command): void {
program
.command('sql')
.description('Read-only SQL fallback (string cells are desensitized by default)')
.description('Read-only SQL fallback (string cells are desensitized; message content requires --raw)')
.argument('<query>', 'SELECT / WITH statement')
.option('--session <ref>', 'Session id or unique name')
.option('--limit <n>', 'Max rows (default 100, max 1000)')
@@ -227,7 +320,8 @@ export function registerSqlCommands(program: Command): void {
})
}
assertRawAllowed(ctx, options)
const limit = parseLimit(options.limit, 100, 1000, '--limit')
assertSqlPrivacyAllowed(query, !!options.raw)
const limit = parseSqlRowLimit(options.limit)
let result
try {
@@ -239,6 +333,7 @@ export function registerSqlCommands(program: Command): void {
hint: 'Run `chatlab schema` to inspect available tables and columns',
})
}
assertSqlResultPrivacyAllowed(result.columns, !!options.raw)
let rows = result.rows
let removed = 0
+3 -3
View File
@@ -66,11 +66,11 @@ const EXAMPLES: ManifestExample[] = [
command: 'chatlab messages search 旅游 --since 2026-01-01 --limit 20 --format agent',
},
{
task: 'Who mentioned it first (evidence ids)?',
task: 'Who mentioned it first (message ids)?',
command: 'chatlab messages search 报销 --sort asc --limit 5 --format agent',
},
{
task: 'Follow up a [#id] citation with context',
task: 'Follow up a single [#id] marker with context',
command: 'chatlab messages context --id 1021 --window 10 --format agent',
},
{
@@ -92,7 +92,7 @@ const NOTES: string[] = [
'stdout carries exactly one JSON envelope in agent/json mode; logs go to stderr.',
'Success envelope: { ok, command, data, meta }; failure: { ok: false, command, error }.',
'Pagination: follow meta.nextCursor with --cursor; cursors are bound to the exact query.',
'[#id] markers in agent text are message ids usable with `messages context --id`.',
'Single-message [#id] and [#id*] markers are usable with `messages context --id`; merged [#a-b] ranges are display-only.',
'Privacy preprocessing (desensitize/blacklist) is always on by default.',
]
+2 -1
View File
@@ -2,7 +2,8 @@
* Shared output shaping for message-carrying query commands.
*
* agent format: AI-facing text from the shared preprocessing pipeline with
* [#id] citations (design §6.2). json format: structured items with privacy
* [#id] / [#id*] message markers and display-only [#a-b] merged ranges.
* json format: structured items with privacy
* steps applied (cleaning/blacklist/desensitize) but no merging/denoising.
*/
+48 -1
View File
@@ -6,7 +6,15 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor, epochToIso } from './parse'
import {
parseTimeOptions,
parseLimit,
queryFingerprint,
encodeCursor,
decodeCursor,
epochToIso,
resolveTimeOptionsForCursor,
} from './parse'
import { QueryError } from './envelope'
// Fixed reference: 2026-07-03 15:30:00 local time
@@ -60,6 +68,22 @@ describe('parseTimeOptions', () => {
}
})
it('rejects invalid calendar dates instead of normalizing them', () => {
for (const bad of [
{ since: '2026-02-31' },
{ until: '2026-13-01' },
{ since: '2026-02-31 08:30' },
{ since: '2026-06-01 25:00' },
{ until: '2026-02-31T10:00:00+08:00' },
]) {
assert.throws(
() => parseTimeOptions(bad, NOW),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT',
JSON.stringify(bad)
)
}
})
it('rejects inverted ranges', () => {
assert.throws(
() => parseTimeOptions({ since: '2026-07-01', until: '2026-06-01' }, NOW),
@@ -90,6 +114,14 @@ describe('parseLimit', () => {
)
}
})
it('can reject zero for cursor-paginated page sizes', () => {
assert.equal(parseLimit('0', 0, 50, '--context'), 0)
assert.throws(
() => parseLimit('0', 50, 500, '--limit', 1),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
})
})
describe('cursor', () => {
@@ -116,4 +148,19 @@ describe('cursor', () => {
(err: unknown) => err instanceof QueryError && err.code === 'CURSOR_INVALID'
)
})
it('reuses frozen time bounds embedded in cursor tokens', () => {
const firstRange = resolveTimeOptionsForCursor({ last: '7d' }, NOW)
const fp = queryFingerprint({
command: 'messages.list',
startTs: firstRange.startTs ?? null,
endTs: firstRange.endTs ?? null,
})
const token = encodeCursor(50, fp, { time: firstRange })
const later = new Date(NOW.getTime() + 60_000)
const secondRange = resolveTimeOptionsForCursor({ last: '7d', cursor: token }, later)
assert.deepEqual(secondRange, firstRange)
assert.equal(decodeCursor(token, fp), 50)
})
})
+145 -9
View File
@@ -23,6 +23,10 @@ export interface ResolvedTimeRange {
meta: { since: string | null; until: string | null }
}
export interface CursorSnapshot {
time?: ResolvedTimeRange
}
/** Format a Date as ISO 8601 with the local UTC offset (sortable, self-describing). */
export function toIsoWithOffset(date: Date): string {
const pad = (n: number, w = 2) => String(Math.abs(n)).padStart(w, '0')
@@ -43,6 +47,7 @@ export function epochToIso(ts: number): string {
const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/
const DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?$/
const ISO_DATE_PREFIX = /^(\d{4})-(\d{2})-(\d{2})T/
const LAST_DURATION = /^(\d+)([hdw])$/
function startOfDay(base: Date, dayOffset = 0): Date {
@@ -57,6 +62,14 @@ function invalidTime(flag: string, value: string): QueryError {
})
}
function isSameLocalDate(date: Date, y: number, m: number, d: number): boolean {
return date.getFullYear() === y && date.getMonth() === m - 1 && date.getDate() === d
}
function isSameLocalDateTime(date: Date, y: number, m: number, d: number, h: number, min: number, s: number): boolean {
return isSameLocalDate(date, y, m, d) && date.getHours() === h && date.getMinutes() === min && date.getSeconds() === s
}
/**
* Parse a single time value. Date-only values resolve to the start of the day;
* when `boundary` is 'end' they resolve to the end of the day (inclusive whole-day).
@@ -71,23 +84,43 @@ function parseTimeValue(value: string, boundary: 'start' | 'end', flag: string,
const dateOnly = DATE_ONLY.exec(value)
if (dateOnly) {
const [, y, m, d] = dateOnly
const base = new Date(Number(y), Number(m) - 1, Number(d))
if (Number.isNaN(base.getTime())) throw invalidTime(flag, value)
const year = Number(y)
const month = Number(m)
const day = Number(d)
const base = new Date(year, month - 1, day)
if (Number.isNaN(base.getTime()) || !isSameLocalDate(base, year, month, day)) throw invalidTime(flag, value)
if (boundary === 'start') return base
return new Date(new Date(Number(y), Number(m) - 1, Number(d) + 1).getTime() - 1000)
return new Date(new Date(year, month - 1, day + 1).getTime() - 1000)
}
const dateTime = DATE_TIME.exec(value)
if (dateTime) {
const [, y, m, d, h, min, s] = dateTime
const parsed = new Date(Number(y), Number(m) - 1, Number(d), Number(h), Number(min), Number(s ?? 0))
if (Number.isNaN(parsed.getTime())) throw invalidTime(flag, value)
const year = Number(y)
const month = Number(m)
const day = Number(d)
const hour = Number(h)
const minute = Number(min)
const second = Number(s ?? 0)
const parsed = new Date(year, month - 1, day, hour, minute, second)
if (Number.isNaN(parsed.getTime()) || !isSameLocalDateTime(parsed, year, month, day, hour, minute, second)) {
throw invalidTime(flag, value)
}
return parsed
}
// full ISO 8601 (with timezone offset or Z)
const parsed = new Date(value)
if (!Number.isNaN(parsed.getTime()) && /\d{4}-\d{2}-\d{2}T/.test(value)) {
const isoDate = ISO_DATE_PREFIX.exec(value)
if (isoDate) {
const [, y, m, d] = isoDate
const year = Number(y)
const month = Number(m)
const day = Number(d)
const localDate = new Date(year, month - 1, day)
if (!isSameLocalDate(localDate, year, month, day)) throw invalidTime(flag, value)
}
return parsed
}
@@ -161,15 +194,19 @@ export function parseLimit(
value: string | number | undefined,
defaultValue: number,
cap: number,
flag: string
flag: string,
min = 0
): number {
if (value === undefined) return Math.min(defaultValue, cap)
const n = typeof value === 'number' ? value : Number(value)
if (!Number.isInteger(n) || n < 0) {
if (!Number.isInteger(n) || n < min) {
throw new QueryError({
code: 'INVALID_ARGUMENT',
message: `Invalid ${flag} value: ${value}`,
hint: `${flag} must be a non-negative integer (max ${cap})`,
hint:
min > 0
? `${flag} must be an integer from ${min} to ${cap}`
: `${flag} must be a non-negative integer (max ${cap})`,
})
}
return Math.min(n, cap)
@@ -183,12 +220,111 @@ export function queryFingerprint(conditions: Record<string, unknown>): string {
return createHash('sha256').update(canonical).digest('hex').slice(0, 12)
}
export function encodeCursor(offset: number, fingerprint: string): string {
export function freezeTimeRangeForCursor(time: ResolvedTimeRange): ResolvedTimeRange {
if (time.endTs !== undefined || time.meta.until === null) return time
const endMs = new Date(time.meta.until).getTime()
if (Number.isNaN(endMs)) return time
return { ...time, endTs: Math.floor(endMs / 1000) }
}
export function resolveTimeOptionsForCursor(
options: TimeCliOptions & { cursor?: string },
now = new Date()
): ResolvedTimeRange {
const snapshot = options.cursor ? getCursorSnapshot(options.cursor) : undefined
return snapshot?.time ?? freezeTimeRangeForCursor(parseTimeOptions(options, now))
}
export function encodeCursor(offset: number, fingerprint: string, snapshot?: CursorSnapshot): string {
if (snapshot?.time) {
return Buffer.from(JSON.stringify({ v: 2, fp: fingerprint, offset, time: snapshot.time }), 'utf8').toString(
'base64url'
)
}
return Buffer.from(`${fingerprint}:${offset}`, 'utf8').toString('base64url')
}
function asObject(value: unknown): Record<string, unknown> | null {
return value !== null && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, unknown>)
: null
}
function decodeCursorEnvelope(
token: string
): { fingerprint: string; offset: number; snapshot?: CursorSnapshot } | null {
let decoded = ''
try {
decoded = Buffer.from(token, 'base64url').toString('utf8')
} catch {
return null
}
let payload: unknown
try {
payload = JSON.parse(decoded)
} catch {
return null
}
const object = asObject(payload)
if (
!object ||
object.v !== 2 ||
typeof object.fp !== 'string' ||
typeof object.offset !== 'number' ||
!Number.isInteger(object.offset) ||
object.offset < 0
) {
return null
}
const snapshot = decodeCursorSnapshot(object)
return { fingerprint: object.fp, offset: object.offset, ...(snapshot ? { snapshot } : {}) }
}
function decodeCursorSnapshot(payload: Record<string, unknown>): CursorSnapshot | undefined {
const timeObject = asObject(payload.time)
if (!timeObject) return undefined
const metaObject = asObject(timeObject.meta)
if (!metaObject) return undefined
const startTs = timeObject.startTs
const endTs = timeObject.endTs
const since = metaObject.since
const until = metaObject.until
if (startTs !== undefined && typeof startTs !== 'number') return undefined
if (endTs !== undefined && typeof endTs !== 'number') return undefined
if (since !== null && typeof since !== 'string') return undefined
if (until !== null && typeof until !== 'string') return undefined
return {
time: {
...(startTs !== undefined ? { startTs } : {}),
...(endTs !== undefined ? { endTs } : {}),
meta: { since, until },
},
}
}
export function getCursorSnapshot(token: string): CursorSnapshot | undefined {
return decodeCursorEnvelope(token)?.snapshot
}
/** Decode a cursor and validate it against the current query fingerprint. */
export function decodeCursor(token: string, fingerprint: string): number {
const envelope = decodeCursorEnvelope(token)
if (envelope) {
if (envelope.fingerprint !== fingerprint) {
throw new QueryError({
code: 'CURSOR_INVALID',
message: 'Cursor does not match the current query conditions',
hint: 'Cursors are only valid for the exact query that produced them; re-run the query without --cursor',
})
}
return envelope.offset
}
let decoded = ''
try {
decoded = Buffer.from(token, 'base64url').toString('utf8')
+41
View File
@@ -0,0 +1,41 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import type { ChatLabConfig } from '@openchatlab/config'
import { assertLegacySqlAllowed } from './register'
import { QueryError } from './envelope'
function configWithSql(allowSql: boolean): ChatLabConfig {
return {
llm: { provider: '', model: '', base_url: '' },
data: { user_data_dir: '', electron_migration_done: false },
api: { port: 3110, host: '127.0.0.1', token: '', require_auth: false },
locale: { lang: '' },
ui: { default_session_tab: 'overview', session_gap_threshold: 1800, summary_strategy: 'standard' },
cli: { allow_raw: false, allow_sql: allowSql },
}
}
describe('legacy query alias SQL gate', () => {
it('rejects SQL when cli.allow_sql is disabled', () => {
assert.throws(
() => assertLegacySqlAllowed(configWithSql(false)),
(err: unknown) => err instanceof QueryError && err.code === 'SQL_DISABLED'
)
})
it('allows SQL when cli.allow_sql is enabled', () => {
assert.doesNotThrow(() => assertLegacySqlAllowed(configWithSql(true)))
})
it('rejects message content SQL because the legacy alias has no raw mode', () => {
assert.throws(
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT content FROM message'),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
assert.throws(
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT * FROM message'),
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
)
})
})
+27 -2
View File
@@ -4,12 +4,19 @@
*/
import type { Command } from 'commander'
import type { ChatLabConfig } from '@openchatlab/config'
import { searchMessagesLike, executeReadonlySql } from '@openchatlab/core'
import { initRuntime } from '../runtime'
import { registerSessionCommands, registerMemberCommands } from './commands-sessions-members'
import { registerMessageCommands } from './commands-messages'
import { registerStatsCommands } from './commands-stats'
import { registerTopicCommands, registerSqlCommands } from './commands-topics-sql'
import {
registerTopicCommands,
registerSqlCommands,
assertSqlPrivacyAllowed,
assertSqlResultPrivacyAllowed,
} from './commands-topics-sql'
import { QueryError } from './envelope'
export function registerQueryCommands(program: Command): void {
registerSessionCommands(program)
@@ -21,6 +28,17 @@ export function registerQueryCommands(program: Command): void {
registerLegacyAliases(program)
}
export function assertLegacySqlAllowed(config: Pick<ChatLabConfig, 'cli'>, sql?: string): void {
if (!config.cli.allow_sql) {
throw new QueryError({
code: 'SQL_DISABLED',
message: 'The sql command is disabled by configuration',
hint: 'The user can re-enable it with `chatlab config set cli.allow_sql true`',
})
}
if (sql) assertSqlPrivacyAllowed(sql, false)
}
// ==================== Hidden legacy aliases (output kept byte-compatible) ====================
function registerLegacyAliases(program: Command): void {
@@ -67,7 +85,7 @@ function registerLegacyAliases(program: Command): void {
console.error(
'[Deprecated] `chatlab query` — use `chatlab sql "<SELECT ...>" --session <id>`; alias kept for one major version'
)
const { dbManager } = initRuntime()
const { config, dbManager } = initRuntime()
const db = dbManager.open(sessionId)
if (!db) {
console.error(`Session ${sessionId} not found`)
@@ -75,7 +93,9 @@ function registerLegacyAliases(program: Command): void {
}
try {
assertLegacySqlAllowed(config, options.sql)
const result = executeReadonlySql(db, options.sql)
assertSqlResultPrivacyAllowed(result.columns, false)
if (options.format === 'json') {
console.log(JSON.stringify(result, null, 2))
} else {
@@ -87,6 +107,11 @@ function registerLegacyAliases(program: Command): void {
}
}
} catch (err) {
if (err instanceof QueryError) {
console.error(`${err.code}: ${err.message}`)
if (err.hint) console.error(`Hint: ${err.hint}`)
process.exit(2)
}
console.error(`SQL error: ${err instanceof Error ? err.message : err}`)
process.exit(1)
}
@@ -0,0 +1,18 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { adaptWorkerSqlResult } from './worker-sql-result'
describe('adaptWorkerSqlResult', () => {
it('maps the worker limited flag to the shared truncated contract', () => {
const result = adaptWorkerSqlResult({
columns: ['id'],
rows: [[1]],
rowCount: 1,
duration: 1,
limited: true,
})
assert.equal(result.truncated, true)
})
})
@@ -20,6 +20,7 @@ import type {
SegmentSummaryItem,
RawMessage,
} from '@openchatlab/tools'
import { adaptWorkerSqlResult } from './worker-sql-result'
function mapSearchMessages(messages: workerManager.SearchMessageResult[]): RawMessage[] {
return messages.map((m) => ({
@@ -186,7 +187,8 @@ export class WorkerDataProvider implements ToolDataProvider {
}
async executeSql(sql: string, options?: { maxRows?: number }): Promise<unknown> {
return this.run(() => workerManager.executeRawSQL(this.sessionId, sql, options?.maxRows))
const result = await this.run(() => workerManager.executeRawSQL(this.sessionId, sql, options?.maxRows))
return adaptWorkerSqlResult(result)
}
async executeParameterizedSql<T = Record<string, unknown>>(
@@ -0,0 +1,11 @@
interface WorkerSqlResultLike {
limited?: boolean
truncated?: boolean
}
export function adaptWorkerSqlResult<T extends WorkerSqlResultLike>(result: T): T & { truncated?: boolean } {
return {
...result,
truncated: result.truncated ?? result.limited,
}
}
+5 -5
View File
@@ -12,7 +12,7 @@ ChatLab 可以作为 Codex、Claude Code、HermesAgent 等外部 Agent 的本地
- 你已经在使用 Codex、Claude Code、HermesAgent 等外部 Agent,并希望复用它们的订阅、token、模型能力和工作流。
- 你希望外部 Agent 能按需分析 ChatLab 里的聊天记录,但不想把原始数据库或完整聊天导出直接交给 Agent。
- 你希望先由 ChatLab 完成脱敏、黑名单过滤、清洗、合并和截断,再把适合分析的文本与证据 id 提供给 Agent。
- 你希望先由 ChatLab 完成脱敏、黑名单过滤、清洗、合并和截断,再把适合分析的文本与可追溯的单条消息 id 提供给 Agent。
`chatlab` CLI 提供一组面向 AI Agent 优化、同时人类可读的查询命令。所有查询命令只读,默认应用你在 ChatLab 中配置的隐私预处理(脱敏、黑名单)。
@@ -47,7 +47,7 @@ chatlab manifest # 机读命令清单(给 Agent 一次读
通过 `--format` 显式指定:
- **`agent`**:推荐给 AI Agent 的格式。返回 JSON 信封,正文是经过完整预处理管道(清洗、黑名单、去噪、合并连续发言、脱敏、token 截断)的紧凑文本,每条消息带 `[#id]` 引用标记,搜索命中标 `[#id*]`。
- **`agent`**:推荐给 AI Agent 的格式。返回 JSON 信封,正文是经过完整预处理管道(清洗、黑名单、去噪、合并连续发言、脱敏、token 截断)的紧凑文本;单条消息标记 `[#id]` / `[#id*]` 可用于 `messages context --id`,合并块范围 `[#a-b]` 仅用于展示。
- **`json`**:程序化解析格式。返回结构化消息数组,仍应用隐私步骤(清洗、黑名单、脱敏),但不合并、不去噪。适合 `--no-content`、`--fields` 之类的结构化勘察。
- **`text`**:人类可读输出(终端默认)。
@@ -73,7 +73,7 @@ agent/json 模式下 stdout 只包含一个 JSON 响应信封,日志一律走
## 隐私边界
- 所有查询命令默认应用你在 ChatLab 设置中的脱敏规则与黑名单,包括 `stats keywords` 的词表、`topics list` 的摘要和 `sql` 结果中的字符串字段。
- 所有查询命令默认应用你在 ChatLab 设置中的脱敏规则与黑名单,包括 `stats keywords` 的词表、`topics list` 的摘要和 `sql` 结果中的字符串字段;`sql` 读取消息 `content` 列需要显式 `--raw`。
- `--raw`(绕过预处理)默认禁用,仅当你显式执行 `chatlab config set cli.allow_raw true` 或设置 `CHATLAB_CLI_ALLOW_RAW=1` 后可用。
- 不需要 SQL 兜底能力时,可用 `chatlab config set cli.allow_sql false` 关闭 `sql` 命令。
@@ -93,10 +93,10 @@ chatlab-analyze 帮我分析我和小红的聊天记录
这个技能会引导 Agent 先执行 `chatlab manifest` 获取机读命令清单,再用安全的 `--format agent/json` 查询聊天记录。
典型配方——"谁最早提到某个问题?给出证据":
典型配方——"谁最早提到某个问题?查看现场":
```bash
chatlab messages search "服务器迁移" --sort asc --limit 5 --context 3 --format agent
# 从返回文本的 [#1021*] 标记拿到消息 id,再深挖现场:
# 从返回文本的单条消息标记 [#1021*] 拿到消息 id,再深挖现场:
chatlab messages context --id 1021 --window 10 --format agent
```
+5 -5
View File
@@ -12,7 +12,7 @@ This mode does not require opening the ChatLab web UI, and it does not require s
- You already use an external agent such as Codex, Claude Code, or HermesAgent, and want to reuse its subscription, tokens, model capabilities, and workflow.
- You want the agent to analyze ChatLab records on demand, but you do not want to hand it the raw database or a full chat export.
- You want ChatLab to handle desensitization, blacklist filtering, cleaning, merging, and truncation before sending analysis-ready text and evidence ids to the agent.
- You want ChatLab to handle desensitization, blacklist filtering, cleaning, merging, and truncation before sending analysis-ready text and traceable single-message ids to the agent.
The `chatlab` CLI ships a set of query commands optimized for AI agents while staying human-readable. All query commands are read-only and apply your ChatLab privacy preprocessing (desensitization, blacklist) by default.
@@ -47,7 +47,7 @@ With a single session `--session` can be omitted; with multiple sessions, comman
Pass `--format` explicitly:
- **`agent`**: recommended for AI agents. A JSON envelope whose body is compact text produced by the full preprocessing pipeline (cleaning, blacklist, denoising, merging consecutive messages, desensitization, token-aware truncation). Every message carries a `[#id]` citation marker; search hits are marked `[#id*]`.
- **`agent`**: recommended for AI agents. A JSON envelope whose body is compact text produced by the full preprocessing pipeline (cleaning, blacklist, denoising, merging consecutive messages, desensitization, token-aware truncation). Single-message markers `[#id]` / `[#id*]` can be used with `messages context --id`; merged ranges such as `[#a-b]` are display-only.
- **`json`**: for programmatic parsing. Structured message items with privacy steps applied (cleaning, blacklist, desensitization) but no merging or denoising. The right choice for structural scouting with `--no-content` / `--fields`.
- **`text`**: human-readable output (TTY default).
@@ -73,7 +73,7 @@ Failures return `{ "ok": false, "error": { "code", "message", "hint", "candidate
## Privacy boundaries
- All query commands apply your ChatLab desensitize rules and blacklist by default — including the `stats keywords` vocabulary, `topics list` summaries, and string cells in `sql` results.
- All query commands apply your ChatLab desensitize rules and blacklist by default — including the `stats keywords` vocabulary, `topics list` summaries, and string cells in `sql` results; reading the message `content` column with `sql` requires explicit `--raw`.
- `--raw` (bypassing preprocessing) is disabled by default; it only works after you explicitly run `chatlab config set cli.allow_raw true` or set `CHATLAB_CLI_ALLOW_RAW=1`.
- If you don't need the SQL fallback, disable it with `chatlab config set cli.allow_sql false`.
@@ -93,10 +93,10 @@ chatlab-analyze help me analyze my chat history with Alice
The skill tells the agent to run `chatlab manifest` first, then query chat records safely with explicit `--format agent/json` commands.
A typical recipe — "who mentioned this first? give evidence":
A typical recipe — "who mentioned this first? inspect the surrounding context":
```bash
chatlab messages search "server migration" --sort asc --limit 5 --context 3 --format agent
# Take the message id from the [#1021*] marker in the returned text, then dig in:
# Take the message id from a single-message [#1021*] marker in the returned text, then dig in:
chatlab messages context --id 1021 --window 10 --format agent
```
+7
View File
@@ -126,6 +126,13 @@ export function writeConfigField(section: string, key: string, value: string | n
} catch {
// 解析失败时从空开始,旧文件会被覆盖
}
} else if (fs.existsSync(CONFIG_JSON)) {
try {
const content = fs.readFileSync(CONFIG_JSON, 'utf-8')
existing = JSON.parse(content) as Record<string, Record<string, unknown>>
} catch {
// Match loadConfigFile: unreadable legacy JSON does not seed TOML writes.
}
}
if (!existing[section] || typeof existing[section] !== 'object') {
@@ -13,10 +13,12 @@ const [{ setConfigField, ConfigSetError }, { loadConfig }] = await Promise.all([
])
const configToml = join(tempHome, '.chatlab', 'config.toml')
const configJson = join(tempHome, '.chatlab', 'config.json')
beforeEach(() => {
rmSync(join(tempHome, '.chatlab'), { recursive: true, force: true })
delete process.env.CHATLAB_CLI_ALLOW_RAW
delete process.env.CHATLAB_API_PORT
})
after(() => {
@@ -57,6 +59,39 @@ describe('setConfigField', () => {
assert.equal(loadConfig().api.port, 8080)
})
it('validates written file values without environment overrides', () => {
process.env.CHATLAB_API_PORT = '3110'
assert.throws(
() => setConfigField('api.port', '70000'),
(err: unknown) => {
assert.ok(err instanceof ConfigSetError)
assert.equal(err.reason, 'invalid_config')
return true
}
)
assert.equal(existsSync(configToml), false)
})
it('preserves legacy JSON config when creating TOML', () => {
mkdirSync(join(tempHome, '.chatlab'), { recursive: true })
writeFileSync(
configJson,
JSON.stringify({
data: { user_data_dir: '/tmp/chatlab-legacy-data' },
api: { host: '0.0.0.0' },
}),
'utf-8'
)
setConfigField('cli.allow_raw', 'true')
const config = loadConfig()
assert.equal(config.cli.allow_raw, true)
assert.equal(config.data.user_data_dir, '/tmp/chatlab-legacy-data')
assert.equal(config.api.host, '0.0.0.0')
})
it('rejects unknown section or key without touching the file', () => {
assert.throws(
() => setConfigField('nope.key', 'x'),
+8 -2
View File
@@ -10,7 +10,7 @@ import * as fs from 'fs'
import * as path from 'path'
import { parse as parseToml } from 'smol-toml'
import { configSchema } from './schema'
import { loadConfig, writeConfigField, getConfigDir } from './loader'
import { writeConfigField, getConfigDir } from './loader'
export type ConfigSetErrorReason = 'unknown_key' | 'invalid_value' | 'invalid_config' | 'unreadable_config'
@@ -70,7 +70,7 @@ export function setConfigField(fieldPath: string, rawValue: string): ConfigSetRe
writeConfigField(section, key, value)
try {
loadConfig()
validateConfigFile(tomlPath)
} catch (err) {
if (original === null) {
fs.unlinkSync(tomlPath)
@@ -87,6 +87,12 @@ export function setConfigField(fieldPath: string, rawValue: string): ConfigSetRe
return { section, key, value }
}
function validateConfigFile(tomlPath: string): void {
const written = fs.readFileSync(tomlPath, 'utf-8')
const parsed = parseToml(written) as Record<string, unknown>
configSchema.parse(parsed)
}
function parseValueByType(
section: string,
key: string,
@@ -118,6 +118,26 @@ describe('search extensions (in-memory SQLite)', () => {
assert.equal(wildcard.total, 2)
})
it('search keywords escape SQL wildcards instead of expanding them', () => {
raw
.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
.run(6, 1, 6000, 0, '包含_下划线和 100% 符号')
const underscore = searchMessagesByKeywords(db, ['_'], { sort: 'asc' })
assert.equal(underscore.total, 1)
assert.deepEqual(
underscore.messages.map((m) => m.id),
[6]
)
const percent = searchMessagesByKeywords(db, ['%'], { sort: 'asc' })
assert.equal(percent.total, 2)
assert.deepEqual(
percent.messages.map((m) => m.id),
[4, 6]
)
})
it('excludeKeywords keeps NULL-content rows (parity with pipeline blacklist)', () => {
const { clause, params } = buildMsgConditions({ excludeKeywords: ['机密'] })
const row = db
@@ -139,6 +159,24 @@ describe('search extensions (in-memory SQLite)', () => {
[3, 2]
)
})
it('uses id as a tie-breaker for same-second search paging', () => {
const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
insert.run(6, 1, 7000, 0, '同秒消息')
insert.run(7, 2, 7000, 0, '同秒消息')
const asc = searchMessagesByKeywords(db, ['同秒'], { sort: 'asc' })
assert.deepEqual(
asc.messages.map((m) => m.id),
[6, 7]
)
const desc = searchMessagesByKeywords(db, ['同秒'])
assert.deepEqual(
desc.messages.map((m) => m.id),
[7, 6]
)
})
})
describe('getConversationBetween pagination and blacklist (in-memory SQLite)', () => {
@@ -181,6 +219,18 @@ describe('getConversationBetween pagination and blacklist (in-memory SQLite)', (
)
})
it('uses id as a tie-breaker for same-second between paging', () => {
const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
insert.run(7, 1, 6000, 0, '同秒 A')
insert.run(8, 2, 6000, 0, '同秒 B')
const page = getConversationBetween(db, 1, 2, undefined, 2)
assert.deepEqual(
page.messages.map((m) => m.id),
[7, 8]
)
})
it('excludeKeywords removes rows from both page and total (visible counting)', () => {
const result = getConversationBetween(db, 1, 2, undefined, 10, { excludeKeywords: ['机密'] })
assert.equal(result.total, 4)
+5 -3
View File
@@ -39,6 +39,7 @@ export interface QueryMessagesOptions {
endTs?: number
senderId?: number
limit?: number
maxLimit?: number
offset?: number
}
@@ -55,7 +56,8 @@ export interface QueryMessagesResult {
* 支持关键词、时间范围、发送者过滤
*/
export function queryMessages(db: DatabaseAdapter, options?: QueryMessagesOptions): QueryMessagesResult {
const limit = Math.min(1000, Math.max(1, options?.limit ?? 100))
const maxLimit = Math.max(1, options?.maxLimit ?? 1000)
const limit = Math.min(maxLimit, Math.max(1, options?.limit ?? 100))
const offset = options?.offset ?? 0
const page = Math.floor(offset / limit) + 1
@@ -240,7 +242,7 @@ export function searchMessagesByKeywords(
.get(...params) as { total: number }
const rows = db
.prepare(`${FULL_MSG_SELECT} WHERE 1=1 ${clause} ORDER BY msg.ts ${order} LIMIT ? OFFSET ?`)
.prepare(`${FULL_MSG_SELECT} WHERE 1=1 ${clause} ORDER BY msg.ts ${order}, msg.id ${order} LIMIT ? OFFSET ?`)
.all(...params, limit + 1, offset) as unknown as FullMessageRow[]
const hasMore = rows.length > limit
@@ -752,7 +754,7 @@ export function getConversationBetween(
WHERE msg.sender_id IN (?, ?) ${timeCondition}
AND msg.content IS NOT NULL AND msg.content != ''
${excludeCondition}
ORDER BY msg.ts DESC LIMIT ? OFFSET ?
ORDER BY msg.ts DESC, msg.id DESC LIMIT ? OFFSET ?
`
const rows = db
.prepare(sql)
+2 -2
View File
@@ -163,9 +163,9 @@ export function buildMsgConditions(options?: {
}
if (options?.keywords && options.keywords.length > 0) {
const joiner = options.matchMode === 'all' ? ' AND ' : ' OR '
const kwConds = options.keywords.map(() => 'msg.content LIKE ?')
const kwConds = options.keywords.map(() => "msg.content LIKE ? ESCAPE '\\'")
conds.push(`(${kwConds.join(joiner)})`)
params.push(...options.keywords.map((k) => `%${k}%`))
params.push(...options.keywords.map((k) => `%${escapeLikePattern(k)}%`))
}
if (options?.excludeKeywords && options.excludeKeywords.length > 0) {
const exclude = buildExcludeKeywordsConditions(options.excludeKeywords)
@@ -13,7 +13,7 @@ import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { formatToolResultAsText, formatMessageCompact } from './format'
import { applyPreprocessingPipeline } from './preprocessing-pipeline'
import type { PreprocessableMessage } from './types'
import type { PreprocessConfig, PreprocessableMessage } from './types'
const rawMessages: PreprocessableMessage[] = [
{ id: 1, senderName: 'Alice', content: 'hello world', timestamp: 1710000000 },
@@ -103,6 +103,16 @@ describe('applyPreprocessingPipeline CLI extensions', () => {
{ id: 1021, senderName: '老王', content: '报销流程是不是改了?', timestamp: 1710000000 },
{ id: 1022, senderName: '小红', content: '对,三月起走新系统', timestamp: 1710000060 },
]
const mergeConfig: PreprocessConfig = {
dataCleaning: false,
mergeConsecutive: true,
mergeWindowSeconds: 180,
blacklistKeywords: [],
denoise: false,
desensitize: false,
desensitizeRules: [],
anonymizeNames: false,
}
it('threads includeMessageIds and hitIds into the rendered text', () => {
const result = applyPreprocessingPipeline({
@@ -114,6 +124,21 @@ describe('applyPreprocessingPipeline CLI extensions', () => {
assert.ok(result.text.includes('[#1022]'))
})
it('marks merged ranges when the later merged message is a search hit', () => {
const result = applyPreprocessingPipeline({
rawMessages: [
{ id: 1021, senderName: '老王', content: '先看下旧流程', timestamp: 1710000000 },
{ id: 1022, senderName: '老王', content: '新流程在哪里?', timestamp: 1710000060 },
],
preprocessConfig: mergeConfig,
includeMessageIds: true,
hitIds: [1022],
})
assert.ok(result.text.includes('[#1021-1022*]'), result.text)
assert.equal(result.stats.mergeCombined, 1)
})
it('keeps default output free of id markers and exposes pipeline stats', () => {
const result = applyPreprocessingPipeline({ rawMessages: messagesWithIds })
assert.ok(!result.text.includes('[#'))
@@ -39,7 +39,7 @@ export function t(key: TextEntryKey, locale?: string): string | string[] {
const MAX_MESSAGE_CONTENT_LENGTH = 200
export interface FormatMessageOptions {
/** Render an [#id] prefix for message citations (CLI agent format). */
/** Render [#id] / [#id*] prefixes, or display-only [#a-b] ranges for merged blocks. */
includeMessageId?: boolean
/** Message ids that are search hits; rendered as [#id*]. */
hitIds?: Set<number>
@@ -50,7 +50,7 @@ export interface FormatMessageOptions {
/**
* 格式化消息为简洁文本格式
* 输出格式: "2025/3/3 07:25:04 张三: 消息内容"
* includeMessageId 开启时: "[#1021] 2025/3/3 07:25:04 张三: 消息内容"(合并块为 [#start-end])
* includeMessageId 开启时: "[#1021] 2025/3/3 07:25:04 张三: 消息内容"(合并块显示为 [#start-end])
*/
export function formatMessageCompact(
msg: {
@@ -59,6 +59,7 @@ export function formatMessageCompact(
content: string | null
timestamp: number
mergedEndId?: number
mergedIds?: number[]
},
locale?: string,
options?: FormatMessageOptions
@@ -75,13 +76,19 @@ export function formatMessageCompact(
let idPrefix = ''
if (options?.includeMessageId && msg.id != null) {
const range = msg.mergedEndId != null && msg.mergedEndId !== msg.id ? `${msg.id}-${msg.mergedEndId}` : `${msg.id}`
const hitMark = options.hitIds?.has(msg.id) ? '*' : ''
const hitMark = isHitMessage(msg, options.hitIds) ? '*' : ''
idPrefix = `[#${range}${hitMark}] `
}
return `${idPrefix}${time} ${msg.senderName}: ${content}`
}
function isHitMessage(msg: { id?: number; mergedIds?: number[] }, hitIds?: Set<number>): boolean {
if (!hitIds) return false
if (msg.id != null && hitIds.has(msg.id)) return true
return msg.mergedIds?.some((id) => hitIds.has(id)) ?? false
}
/**
* 格式化时间范围用于返回结果
*/
@@ -129,7 +136,7 @@ export function formatToolResultAsText(details: Record<string, unknown>): string
lines.push('')
let lastDate = ''
for (const msg of messages) {
// [#id] citation prefixes (CLI agent format) must survive date grouping at line start
// [#id] prefixes (CLI agent format) must survive date grouping at line start.
const prefixMatch = /^(\[#[^\]]+\] )/.exec(msg)
const prefix = prefixMatch ? prefixMatch[1] : ''
const body = prefix ? msg.slice(prefix.length) : msg
@@ -231,11 +231,16 @@ function applyMergeConsecutive<T extends PreprocessableMessage>(messages: T[], w
const withinWindow = Math.abs(msg.timestamp - current.timestamp) <= windowSeconds
if (sameSender && withinWindow) {
const mergedIds = [
...(current.mergedIds ?? (current.id != null ? [current.id] : [])),
...(msg.mergedIds ?? (msg.id != null ? [msg.id] : [])),
]
current = {
...current,
content: [current.content, msg.content].filter(Boolean).join('\n'),
// keep first message's id, track last id for [#start-end] range citations
mergedEndId: msg.id ?? current.mergedEndId,
...(mergedIds.length > 0 ? { mergedIds } : {}),
}
} else {
merged.push(current)
@@ -53,6 +53,8 @@ export interface PreprocessableMessage {
replyToMessageId?: string | null
/** 合并连续发言后,块内最后一条消息的 id(用于 [#start-end] 区间引用) */
mergedEndId?: number
/** 合并连续发言后,块内所有消息 id(用于判断搜索命中标记) */
mergedIds?: number[]
}
export type TruncationStrategy = 'keep_first' | 'keep_last'
@@ -7,6 +7,7 @@ import { getMessageContextTool } from './get-message-context'
import { getConversationBetweenTool } from './get-conversation-between'
import { recentMessagesTool } from './recent-messages'
import { searchMessagesTool } from './search-messages'
import { deepSearchMessagesTool } from './deep-search-messages'
import { schemaTool, sqlQueryTool } from './sql-query'
import { timeStatsTool } from './time-stats'
import { SQL_TOOL_DEFS, createSqlToolDefinition } from '../sql'
@@ -96,29 +97,42 @@ describe('high-risk analysis tool definitions', () => {
assert.equal(searchCalls[0]?.options?.limit, 12)
})
it('search_messages keeps expanded context within the effective limit', async () => {
it('search tools keep expanded context within the effective limit without dropping hits', async () => {
const expandedMessages: RawMessage[] = [
{ id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000001 },
{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 },
{ id: 12, senderName: 'Alice', content: 'after', timestamp: 1710000003 },
]
const context = createContext({
async searchMessages() {
return {
total: 1,
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
}
},
async getSearchMessageContext() {
return expandedMessages
},
})
const result = await searchMessagesTool.handler({ keywords: ['alpha'], limit: 2 }, context)
const data = result.data as { returned: number }
for (const tool of [searchMessagesTool, deepSearchMessagesTool]) {
const context = createContext({
async searchMessages() {
return {
total: 1,
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
}
},
async deepSearchMessages() {
return {
total: 1,
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
}
},
async getSearchMessageContext() {
return expandedMessages
},
})
assert.equal(data.returned, 2)
assert.equal(result.rawMessages?.length, 2)
const result = await tool.handler({ keywords: ['alpha'], limit: 1 }, context)
const data = result.data as { returned: number }
assert.equal(data.returned, 1, tool.name)
assert.deepEqual(
result.rawMessages?.map((message) => message.id),
[11],
tool.name
)
}
})
it('get_recent_messages respects a user limit smaller than maxMessagesLimit', async () => {
@@ -230,6 +244,24 @@ describe('high-risk analysis tool definitions', () => {
assert.equal((result.data as { contextSize: number }).contextSize, 1)
})
it('get_message_context clamps negative context_size before applying maxMessagesLimit', async () => {
const calls: Array<{ ids: number[]; contextSize: number }> = []
const context = createContext(
{
async getMessageContext(ids, contextSize) {
calls.push({ ids, contextSize })
return [{ id: 10, senderName: 'Alice', content: 'hit', timestamp: 1710000001 }]
},
},
{ maxMessagesLimit: 12 }
)
const result = await getMessageContextTool.handler({ message_ids: [10], context_size: -1 }, context)
assert.deepEqual(calls, [{ ids: [10], contextSize: 0 }])
assert.equal((result.data as { contextSize: number }).contextSize, 0)
})
it('get_segment_summaries filters empty and non-matching summaries after over-fetching', async () => {
const calls: Array<{ limit?: number; timeFilter?: ToolTimeRange }> = []
const contextFilter: ToolTimeRange = { startTs: 1704067200, endTs: 1704153600 }
@@ -9,6 +9,7 @@ import { parseExtendedTimeParams } from '../utils/time-params'
import { formatTimeRange } from '../utils/format'
import { timeParamProperties } from '../utils/schemas'
import { resolveMessageLimit } from '../utils/limits'
import { trimMessagesPreservingHits } from './search-context'
const inputSchema: JsonSchema = {
type: 'object',
@@ -36,14 +37,14 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
const contextBefore = context.searchContextBefore ?? 2
const contextAfter = context.searchContextAfter ?? 2
let finalMessages = result.messages
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
if ((contextBefore > 0 || contextAfter > 0) && result.messages.length > 0) {
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
if (hitIds.length > 0) {
finalMessages = await context.dataProvider!.getSearchMessageContext(hitIds, contextBefore, contextAfter)
}
}
finalMessages = finalMessages.slice(0, limit)
finalMessages = trimMessagesPreservingHits(finalMessages, hitIds, limit)
const data = {
total: result.total,
@@ -22,7 +22,11 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
const requestedMessageIds = params.message_ids as number[]
const messageIds =
maxMessagesLimit && maxMessagesLimit > 0 ? requestedMessageIds.slice(0, maxMessagesLimit) : requestedMessageIds
let contextSize = (params.context_size as number) || 20
const requestedContextSize = params.context_size
let contextSize =
typeof requestedContextSize === 'number' && Number.isFinite(requestedContextSize)
? Math.max(0, Math.floor(requestedContextSize))
: 20
if (maxMessagesLimit && maxMessagesLimit > 0 && messageIds.length > 0) {
const maxMessagesPerId = Math.max(1, Math.floor(maxMessagesLimit / messageIds.length))
@@ -0,0 +1,30 @@
import type { RawMessage } from '../types'
export function trimMessagesPreservingHits(messages: RawMessage[], hitIds: number[], limit: number): RawMessage[] {
if (limit <= 0) return []
if (messages.length <= limit) return messages
const requiredIds = new Set(hitIds)
if (requiredIds.size === 0) return messages.slice(0, limit)
const trimmed = messages.slice(0, limit)
const presentIds = new Set(trimmed.map((message) => message.id).filter((id): id is number => id != null))
const missingHits = messages.filter(
(message) => message.id != null && requiredIds.has(message.id) && !presentIds.has(message.id)
)
for (const hit of missingHits) {
let replaceIndex = -1
for (let i = trimmed.length - 1; i >= 0; i--) {
const id = trimmed[i].id
if (id == null || !requiredIds.has(id)) {
replaceIndex = i
break
}
}
if (replaceIndex === -1) break
trimmed[replaceIndex] = hit
}
return trimmed.sort((a, b) => messages.indexOf(a) - messages.indexOf(b))
}
@@ -9,6 +9,7 @@ import { parseExtendedTimeParams } from '../utils/time-params'
import { formatTimeRange } from '../utils/format'
import { timeParamProperties } from '../utils/schemas'
import { resolveMessageLimit } from '../utils/limits'
import { trimMessagesPreservingHits } from './search-context'
const inputSchema: JsonSchema = {
type: 'object',
@@ -36,14 +37,14 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
const contextBefore = context.searchContextBefore ?? 2
const contextAfter = context.searchContextAfter ?? 2
let finalMessages = result.messages
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
if ((contextBefore > 0 || contextAfter > 0) && result.messages.length > 0) {
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
if (hitIds.length > 0) {
finalMessages = await context.dataProvider!.getSearchMessageContext(hitIds, contextBefore, contextAfter)
}
}
finalMessages = finalMessages.slice(0, limit)
finalMessages = trimMessagesPreservingHits(finalMessages, hitIds, limit)
const data = {
total: result.total,
@@ -146,6 +146,18 @@ describe('CoreDataProvider keyword search', () => {
assert.equal(r.messages.length, 1)
})
it('getRecentMessages honors limits above the default queryMessages cap', async () => {
const ins = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
for (let i = 0; i < 1005; i++) {
ins.run(1000 + i, 1, T0 + 1000 + i, 0, `bulk recent ${i}`)
}
const r = await provider.getRecentMessages({ limit: 1005 })
assert.equal(r.total, 1009)
assert.equal(r.messages.length, 1005)
})
it('passes sort asc through to the core query (default stays desc)', async () => {
const asc = await provider.searchMessages(['麻将'], { sort: 'asc' })
assert.deepEqual(
@@ -42,6 +42,8 @@ import type {
RawMessage,
} from '../types'
const TOOL_MESSAGE_LIMIT_CAP = 50000
export class CoreDataProvider implements ToolDataProvider {
constructor(private db: DatabaseAdapter) {}
@@ -84,6 +86,7 @@ export class CoreDataProvider implements ToolDataProvider {
async getRecentMessages(options?: { timeFilter?: ToolTimeRange; limit?: number }): Promise<SearchMessagesResult> {
const result = queryMessages(this.db, {
limit: options?.limit ?? 50,
maxLimit: TOOL_MESSAGE_LIMIT_CAP,
startTs: options?.timeFilter?.startTs,
endTs: options?.timeFilter?.endTs,
})