mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
fix(cli): harden agent query safeguards
Tighten agent-facing query validation, privacy guards, SQL result handling, config writes, and LIKE search semantics for the CLI query workflow.
This commit is contained in:
1 parent
b33cbd065d
commit
801e7fc66a
32 files changed
+880
-97
No files matched your search
@@ -0,0 +1,71 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import {
|
||||
assertContextAnchorsPresent,
|
||||
capExpandedSearchMessages,
|
||||
parseContextIds,
|
||||
parseSearchKeywords,
|
||||
searchTruncationStrategy,
|
||||
} from './commands-messages'
|
||||
import type { MessageLike } from './messages-output'
|
||||
|
||||
describe('capExpandedSearchMessages', () => {
|
||||
it('keeps hit messages when the cap is smaller than pre-context', () => {
|
||||
const messages: MessageLike[] = [
|
||||
{ id: 10, senderName: 'Alice', content: 'before 1', timestamp: 1710000000 },
|
||||
{ id: 11, senderName: 'Alice', content: 'before 2', timestamp: 1710000001 },
|
||||
{ id: 12, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 },
|
||||
]
|
||||
|
||||
const capped = capExpandedSearchMessages(messages, new Set([12]), 1)
|
||||
|
||||
assert.deepEqual(
|
||||
capped.map((message) => message.id),
|
||||
[12]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseContextIds', () => {
|
||||
it('rejects blank id tokens instead of converting them to 0', () => {
|
||||
assert.throws(() => parseContextIds('1021,'), /Invalid --id value/)
|
||||
assert.throws(() => parseContextIds('1021,,1058'), /Invalid --id value/)
|
||||
})
|
||||
|
||||
it('requires positive numeric message ids', () => {
|
||||
assert.deepEqual(parseContextIds('1021,1058'), [1021, 1058])
|
||||
assert.throws(() => parseContextIds('0'), /Invalid --id value/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseSearchKeywords', () => {
|
||||
it('trims keywords and rejects blank-only searches', () => {
|
||||
assert.deepEqual(parseSearchKeywords([' alpha ', 'beta']), ['alpha', 'beta'])
|
||||
assert.throws(() => parseSearchKeywords([' ']), /Invalid search keywords/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('searchTruncationStrategy', () => {
|
||||
it('keeps newest chronological messages when search sort is descending', () => {
|
||||
assert.equal(searchTruncationStrategy('desc'), 'keep_last')
|
||||
assert.equal(searchTruncationStrategy('asc'), 'keep_first')
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertContextAnchorsPresent', () => {
|
||||
it('rejects context results that do not include every requested anchor id', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
assertContextAnchorsPresent(
|
||||
[999],
|
||||
[
|
||||
{ id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000000 },
|
||||
{ id: 11, senderName: 'Bob', content: 'after', timestamp: 1710000001 },
|
||||
],
|
||||
'999'
|
||||
),
|
||||
/No messages found/
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
import { runQuery } from './runner'
|
||||
import { createQueryContext, assertRawAllowed } from './context'
|
||||
import { resolveMember } from './resolve'
|
||||
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor } from './parse'
|
||||
import { parseLimit, queryFingerprint, encodeCursor, decodeCursor, resolveTimeOptionsForCursor } from './parse'
|
||||
import { QueryError } from './envelope'
|
||||
import { buildMessagesResult, assertRawFormatCompatible, type MessageLike } from './messages-output'
|
||||
|
||||
@@ -37,6 +37,73 @@ interface CommonMessageOptions {
|
||||
verbose?: boolean
|
||||
}
|
||||
|
||||
export function capExpandedSearchMessages(
|
||||
messages: MessageLike[],
|
||||
hitIds: ReadonlySet<number>,
|
||||
maxMessages: number
|
||||
): MessageLike[] {
|
||||
if (maxMessages <= 0) return []
|
||||
if (messages.length <= maxMessages) return messages
|
||||
|
||||
const capped = messages.slice(0, maxMessages)
|
||||
const includedIds = new Set(capped.map((message) => message.id))
|
||||
const missingHits = messages.filter((message) => hitIds.has(message.id) && !includedIds.has(message.id))
|
||||
|
||||
for (const hit of missingHits) {
|
||||
let replaceIndex = -1
|
||||
for (let i = capped.length - 1; i >= 0; i--) {
|
||||
if (!hitIds.has(capped[i].id)) {
|
||||
replaceIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (replaceIndex === -1) break
|
||||
capped[replaceIndex] = hit
|
||||
}
|
||||
|
||||
return capped.sort((a, b) => messages.indexOf(a) - messages.indexOf(b))
|
||||
}
|
||||
|
||||
export function parseContextIds(value: string): number[] {
|
||||
const tokens = value.split(',').map((token) => token.trim())
|
||||
const ids = tokens.map((token) => (token.length > 0 ? Number(token) : Number.NaN))
|
||||
if (ids.length === 0 || ids.some((id) => !Number.isInteger(id) || id <= 0)) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid --id value: ${value}`,
|
||||
hint: 'Pass positive numeric message ids, e.g. --id 1021 or --id 1021,1058',
|
||||
})
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
export function parseSearchKeywords(values: string[]): string[] {
|
||||
const keywords = values.map((value) => value.trim()).filter(Boolean)
|
||||
if (keywords.length === 0) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'Invalid search keywords',
|
||||
hint: 'Pass at least one non-empty search keyword',
|
||||
})
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
export function searchTruncationStrategy(sort: string | undefined): 'keep_first' | 'keep_last' {
|
||||
return sort === 'desc' ? 'keep_last' : 'keep_first'
|
||||
}
|
||||
|
||||
export function assertContextAnchorsPresent(ids: number[], messages: MessageLike[], rawValue: string): void {
|
||||
const returnedIds = new Set(messages.map((message) => message.id))
|
||||
if (ids.some((id) => !returnedIds.has(id))) {
|
||||
throw new QueryError({
|
||||
code: 'MESSAGE_NOT_FOUND',
|
||||
message: `No messages found for id(s) ${rawValue}`,
|
||||
hint: 'Use a single numeric id from [#id]/[#id*] markers; merged [#a-b] ranges are display-only',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function addSharedOptions(cmd: Command): Command {
|
||||
return cmd
|
||||
.option('--session <ref>', 'Session id or unique name (auto-selected when only one exists)')
|
||||
@@ -70,9 +137,9 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
|
||||
const fingerprint = queryFingerprint({
|
||||
@@ -104,7 +171,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
totalHits: total,
|
||||
returnedHits: result.messages.length,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + result.messages.length, fingerprint, { time }) } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, result.messages, meta, {
|
||||
@@ -142,6 +209,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
) => {
|
||||
await runQuery('messages.search', options, async (format) => {
|
||||
assertRawFormatCompatible(format, options)
|
||||
const searchKeywords = parseSearchKeywords(keywords)
|
||||
if (options.match !== 'any' && options.match !== 'all') {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
@@ -159,9 +227,9 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const member = options.member ? resolveMember(ctx.db, options.member) : undefined
|
||||
const limit = parseLimit(options.limit, 20, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 20, 500, '--limit', 1)
|
||||
const context = parseLimit(options.context, 0, 50, '--context')
|
||||
const maxMessages = parseLimit(options.maxMessages, 200, 2000, '--max-messages')
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
@@ -169,7 +237,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
const fingerprint = queryFingerprint({
|
||||
command: 'messages.search',
|
||||
session: ctx.session.id,
|
||||
keywords,
|
||||
keywords: searchKeywords,
|
||||
match: options.match,
|
||||
sort: options.sort,
|
||||
startTs: time.startTs ?? null,
|
||||
@@ -179,7 +247,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
})
|
||||
const offset = options.cursor ? decodeCursor(options.cursor, fingerprint) : 0
|
||||
|
||||
const result = searchMessagesByKeywords(ctx.db, keywords, {
|
||||
const result = searchMessagesByKeywords(ctx.db, searchKeywords, {
|
||||
startTs: time.startTs,
|
||||
endTs: time.endTs,
|
||||
senderId: member?.id,
|
||||
@@ -215,7 +283,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
const tail = hits.filter((m) => !keptHits.has(m.id))
|
||||
pipelineMessages = [...expanded, ...tail]
|
||||
if (pipelineMessages.length > maxMessages) {
|
||||
pipelineMessages = pipelineMessages.slice(0, maxMessages)
|
||||
pipelineMessages = capExpandedSearchMessages(pipelineMessages, hitIds, maxMessages)
|
||||
warnings.push(`response capped at ${maxMessages} messages`)
|
||||
}
|
||||
}
|
||||
@@ -225,18 +293,18 @@ export function registerMessageCommands(program: Command): void {
|
||||
const meta: Record<string, unknown> = {
|
||||
session: ctx.session,
|
||||
timeRange: time.meta,
|
||||
query: { keywords, match: options.match, engine: 'like', sort: options.sort, context },
|
||||
query: { keywords: searchKeywords, match: options.match, engine: 'like', sort: options.sort, context },
|
||||
...(member ? { member } : {}),
|
||||
totalHits: total,
|
||||
returnedHits: hits.length,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + hits.length, fingerprint, { time }) } : {}),
|
||||
...(warnings.length > 0 ? { warnings } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, pipelineMessages, meta, {
|
||||
hitIds,
|
||||
strategy: 'keep_first',
|
||||
strategy: searchTruncationStrategy(options.sort),
|
||||
defaultMaxChars: 120,
|
||||
})
|
||||
} finally {
|
||||
@@ -249,33 +317,20 @@ export function registerMessageCommands(program: Command): void {
|
||||
// ---------- messages context ----------
|
||||
const contextCmd = messagesCmd
|
||||
.command('context')
|
||||
.description('Show messages around specific message ids (evidence follow-up)')
|
||||
.description('Show messages around specific numeric message ids')
|
||||
.requiredOption('--id <ids>', 'Message id(s), comma-separated')
|
||||
.option('--window <n>', 'Messages before/after each id (default 10, max 100)')
|
||||
addSharedOptions(contextCmd)
|
||||
contextCmd.action(async (options: CommonMessageOptions & { id: string; window?: string }) => {
|
||||
await runQuery('messages.context', options, async (format) => {
|
||||
assertRawFormatCompatible(format, options)
|
||||
const ids = options.id.split(',').map((s) => Number(s.trim()))
|
||||
if (ids.length === 0 || ids.some((n) => !Number.isInteger(n) || n < 0)) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid --id value: ${options.id}`,
|
||||
hint: 'Pass numeric message ids, e.g. --id 1021 or --id 1021,1058',
|
||||
})
|
||||
}
|
||||
const ids = parseContextIds(options.id)
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const window = parseLimit(options.window, 10, 100, '--window')
|
||||
const messages = getMessageContext(ctx.db, ids, window)
|
||||
if (messages.length === 0) {
|
||||
throw new QueryError({
|
||||
code: 'MESSAGE_NOT_FOUND',
|
||||
message: `No messages found for id(s) ${options.id}`,
|
||||
hint: 'Message ids come from [#id] markers in search results',
|
||||
})
|
||||
}
|
||||
assertContextAnchorsPresent(ids, messages, options.id)
|
||||
const meta: Record<string, unknown> = {
|
||||
session: ctx.session,
|
||||
anchorIds: ids,
|
||||
@@ -312,10 +367,10 @@ export function registerMessageCommands(program: Command): void {
|
||||
const ctx = createQueryContext(options)
|
||||
try {
|
||||
assertRawAllowed(ctx, options)
|
||||
const time = parseTimeOptions(options)
|
||||
const time = resolveTimeOptionsForCursor(options)
|
||||
const memberA = resolveMember(ctx.db, options.member[0])
|
||||
const memberB = resolveMember(ctx.db, options.member[1])
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit')
|
||||
const limit = parseLimit(options.limit, 50, 500, '--limit', 1)
|
||||
const excludeKeywords = options.raw ? undefined : ctx.preprocessConfig.blacklistKeywords
|
||||
|
||||
const fingerprint = queryFingerprint({
|
||||
@@ -346,7 +401,7 @@ export function registerMessageCommands(program: Command): void {
|
||||
totalHits: result.total,
|
||||
returnedHits: returned,
|
||||
hasMore,
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint) } : {}),
|
||||
...(hasMore ? { nextCursor: encodeCursor(offset + returned, fingerprint, { time }) } : {}),
|
||||
}
|
||||
|
||||
return buildMessagesResult(format, ctx, options, result.messages, meta, {
|
||||
|
||||
@@ -6,7 +6,14 @@
|
||||
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { topicsAgentText, type TopicListItem } from './commands-topics-sql'
|
||||
import {
|
||||
assertSqlPrivacyAllowed,
|
||||
assertSqlResultPrivacyAllowed,
|
||||
parseSqlRowLimit,
|
||||
topicsAgentText,
|
||||
type TopicListItem,
|
||||
} from './commands-topics-sql'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
const items: TopicListItem[] = [
|
||||
{
|
||||
@@ -41,3 +48,55 @@ describe('topicsAgentText', () => {
|
||||
assert.equal(topicsAgentText([]), 'No summaries.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseSqlRowLimit', () => {
|
||||
it('rejects zero because SQL maxRows=0 means unlimited', () => {
|
||||
assert.throws(
|
||||
() => parseSqlRowLimit('0'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSqlPrivacyAllowed', () => {
|
||||
it('requires raw mode before SQL can read message content', () => {
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT * FROM message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT * FROM main.message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT msg.* FROM message msg LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT "message".* FROM message LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertSqlPrivacyAllowed('SELECT "m".* FROM message AS "m" LIMIT 10', false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT COUNT(*) AS total FROM message', false))
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT * FROM sqlite_master', false))
|
||||
assert.doesNotThrow(() => assertSqlPrivacyAllowed('SELECT hex(content) AS content FROM message', true))
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSqlResultPrivacyAllowed', () => {
|
||||
it('requires raw mode when SQL result columns expose message content', () => {
|
||||
assert.throws(
|
||||
() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], false),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id', 'content'], true))
|
||||
assert.doesNotThrow(() => assertSqlResultPrivacyAllowed(['id', 'sender_id'], false))
|
||||
})
|
||||
})
|
||||
@@ -2,9 +2,9 @@
|
||||
* topics / sql / schema query commands.
|
||||
*
|
||||
* Derived-text privacy (design §6.2): topic summaries are LLM-generated from raw
|
||||
* content, and SQL results can carry message bodies — both are desensitized by
|
||||
* default, with blacklist-matching rows dropped and counted into meta.warnings.
|
||||
* --raw (user-gated) and cli.allow_sql are the only escape hatches.
|
||||
* content and are desensitized by default. SQL string cells are desensitized,
|
||||
* but reading the message content column requires --raw so SQL expressions
|
||||
* cannot encode raw bodies before the sanitizer sees them.
|
||||
*/
|
||||
|
||||
import type { Command } from 'commander'
|
||||
@@ -20,6 +20,99 @@ function enabledRules(ctx: QueryContext): DesensitizeRule[] {
|
||||
return ctx.preprocessConfig.desensitize ? ctx.preprocessConfig.desensitizeRules.filter((r) => r.enabled) : []
|
||||
}
|
||||
|
||||
export function parseSqlRowLimit(value: string | undefined): number {
|
||||
return parseLimit(value, 100, 1000, '--limit', 1)
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
||||
const SQL_IDENTIFIER = String.raw`(?:"[^"]+"|'[^']+'|` + '`[^`]+`' + String.raw`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_]*)`
|
||||
const SQL_TABLE_REF_END = String.raw`(?=\s|$|[,;])`
|
||||
const SQL_KEYWORDS = new Set([
|
||||
'where',
|
||||
'join',
|
||||
'left',
|
||||
'right',
|
||||
'inner',
|
||||
'outer',
|
||||
'cross',
|
||||
'order',
|
||||
'group',
|
||||
'limit',
|
||||
'on',
|
||||
'using',
|
||||
])
|
||||
|
||||
function sqlNamePattern(name: string): string {
|
||||
const escaped = escapeRegExp(name)
|
||||
return String.raw`(?:"${escaped}"|'${escaped}'|` + '`' + escaped + '`' + String.raw`|\[${escaped}\]|${escaped})`
|
||||
}
|
||||
|
||||
function unquoteSqlIdentifier(value: string): string {
|
||||
const trimmed = value.trim()
|
||||
if (trimmed.startsWith('[') && trimmed.endsWith(']')) return trimmed.slice(1, -1)
|
||||
const quote = trimmed[0]
|
||||
if ((quote === '"' || quote === "'" || quote === '`') && trimmed.endsWith(quote)) {
|
||||
return trimmed.slice(1, -1)
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
function messageTablePattern(): string {
|
||||
const schemaPrefix = String.raw`(?:${SQL_IDENTIFIER}\s*\.\s*)?`
|
||||
return `${schemaPrefix}${sqlNamePattern('message')}`
|
||||
}
|
||||
|
||||
function selectsMessageStar(query: string): boolean {
|
||||
const selectMatch = query.match(/\bselect\b([\s\S]+?)\bfrom\b/i)
|
||||
if (!selectMatch) return false
|
||||
|
||||
const projection = selectMatch[1]
|
||||
const messageTable = messageTablePattern()
|
||||
if (/(^|[,\s])\*(?=$|[,\s])/i.test(projection)) {
|
||||
return new RegExp(String.raw`\b(?:from|join)\s+${messageTable}${SQL_TABLE_REF_END}`, 'i').test(query)
|
||||
}
|
||||
|
||||
const aliasMatches = query.matchAll(
|
||||
new RegExp(String.raw`\b(?:from|join)\s+${messageTable}(?:\s+(?:as\s+)?(${SQL_IDENTIFIER}))?`, 'gi')
|
||||
)
|
||||
const aliases = new Set(['message'])
|
||||
for (const match of aliasMatches) {
|
||||
const alias = match[1] ? unquoteSqlIdentifier(match[1]).toLowerCase() : undefined
|
||||
if (alias && !SQL_KEYWORDS.has(alias)) aliases.add(alias)
|
||||
}
|
||||
|
||||
return [...aliases].some((alias) =>
|
||||
new RegExp(`${sqlNamePattern(alias)}\\s*\\.\\s*\\*(?=$|[\\s,])`, 'i').test(projection)
|
||||
)
|
||||
}
|
||||
|
||||
export function assertSqlPrivacyAllowed(query: string, raw: boolean): void {
|
||||
if (raw) return
|
||||
if (!/(^|[^A-Za-z0-9_])["'`[]?content["'`\]]?(?![A-Za-z0-9_])/i.test(query) && !selectsMessageStar(query)) {
|
||||
return
|
||||
}
|
||||
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'SQL queries that read message content require --raw',
|
||||
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
|
||||
})
|
||||
}
|
||||
|
||||
export function assertSqlResultPrivacyAllowed(columns: readonly string[], raw: boolean): void {
|
||||
if (raw) return
|
||||
if (!columns.some((column) => unquoteSqlIdentifier(column).toLowerCase() === 'content')) return
|
||||
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: 'SQL results that include message content require --raw',
|
||||
hint: 'Use dedicated message commands for privacy-processed content, or enable raw mode and pass --raw for debugging',
|
||||
})
|
||||
}
|
||||
|
||||
export interface TopicListItem {
|
||||
id: number
|
||||
since: string
|
||||
@@ -209,7 +302,7 @@ export function registerTopicCommands(program: Command): void {
|
||||
export function registerSqlCommands(program: Command): void {
|
||||
program
|
||||
.command('sql')
|
||||
.description('Read-only SQL fallback (string cells are desensitized by default)')
|
||||
.description('Read-only SQL fallback (string cells are desensitized; message content requires --raw)')
|
||||
.argument('<query>', 'SELECT / WITH statement')
|
||||
.option('--session <ref>', 'Session id or unique name')
|
||||
.option('--limit <n>', 'Max rows (default 100, max 1000)')
|
||||
@@ -227,7 +320,8 @@ export function registerSqlCommands(program: Command): void {
|
||||
})
|
||||
}
|
||||
assertRawAllowed(ctx, options)
|
||||
const limit = parseLimit(options.limit, 100, 1000, '--limit')
|
||||
assertSqlPrivacyAllowed(query, !!options.raw)
|
||||
const limit = parseSqlRowLimit(options.limit)
|
||||
|
||||
let result
|
||||
try {
|
||||
@@ -239,6 +333,7 @@ export function registerSqlCommands(program: Command): void {
|
||||
hint: 'Run `chatlab schema` to inspect available tables and columns',
|
||||
})
|
||||
}
|
||||
assertSqlResultPrivacyAllowed(result.columns, !!options.raw)
|
||||
|
||||
let rows = result.rows
|
||||
let removed = 0
|
||||
|
||||
@@ -66,11 +66,11 @@ const EXAMPLES: ManifestExample[] = [
|
||||
command: 'chatlab messages search 旅游 --since 2026-01-01 --limit 20 --format agent',
|
||||
},
|
||||
{
|
||||
task: 'Who mentioned it first (evidence ids)?',
|
||||
task: 'Who mentioned it first (message ids)?',
|
||||
command: 'chatlab messages search 报销 --sort asc --limit 5 --format agent',
|
||||
},
|
||||
{
|
||||
task: 'Follow up a [#id] citation with context',
|
||||
task: 'Follow up a single [#id] marker with context',
|
||||
command: 'chatlab messages context --id 1021 --window 10 --format agent',
|
||||
},
|
||||
{
|
||||
@@ -92,7 +92,7 @@ const NOTES: string[] = [
|
||||
'stdout carries exactly one JSON envelope in agent/json mode; logs go to stderr.',
|
||||
'Success envelope: { ok, command, data, meta }; failure: { ok: false, command, error }.',
|
||||
'Pagination: follow meta.nextCursor with --cursor; cursors are bound to the exact query.',
|
||||
'[#id] markers in agent text are message ids usable with `messages context --id`.',
|
||||
'Single-message [#id] and [#id*] markers are usable with `messages context --id`; merged [#a-b] ranges are display-only.',
|
||||
'Privacy preprocessing (desensitize/blacklist) is always on by default.',
|
||||
]
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
* Shared output shaping for message-carrying query commands.
|
||||
*
|
||||
* agent format: AI-facing text from the shared preprocessing pipeline with
|
||||
* [#id] citations (design §6.2). json format: structured items with privacy
|
||||
* [#id] / [#id*] message markers and display-only [#a-b] merged ranges.
|
||||
* json format: structured items with privacy
|
||||
* steps applied (cleaning/blacklist/desensitize) but no merging/denoising.
|
||||
*/
|
||||
|
||||
|
||||
@@ -6,7 +6,15 @@
|
||||
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { parseTimeOptions, parseLimit, queryFingerprint, encodeCursor, decodeCursor, epochToIso } from './parse'
|
||||
import {
|
||||
parseTimeOptions,
|
||||
parseLimit,
|
||||
queryFingerprint,
|
||||
encodeCursor,
|
||||
decodeCursor,
|
||||
epochToIso,
|
||||
resolveTimeOptionsForCursor,
|
||||
} from './parse'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
// Fixed reference: 2026-07-03 15:30:00 local time
|
||||
@@ -60,6 +68,22 @@ describe('parseTimeOptions', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects invalid calendar dates instead of normalizing them', () => {
|
||||
for (const bad of [
|
||||
{ since: '2026-02-31' },
|
||||
{ until: '2026-13-01' },
|
||||
{ since: '2026-02-31 08:30' },
|
||||
{ since: '2026-06-01 25:00' },
|
||||
{ until: '2026-02-31T10:00:00+08:00' },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => parseTimeOptions(bad, NOW),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT',
|
||||
JSON.stringify(bad)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects inverted ranges', () => {
|
||||
assert.throws(
|
||||
() => parseTimeOptions({ since: '2026-07-01', until: '2026-06-01' }, NOW),
|
||||
@@ -90,6 +114,14 @@ describe('parseLimit', () => {
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('can reject zero for cursor-paginated page sizes', () => {
|
||||
assert.equal(parseLimit('0', 0, 50, '--context'), 0)
|
||||
assert.throws(
|
||||
() => parseLimit('0', 50, 500, '--limit', 1),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('cursor', () => {
|
||||
@@ -116,4 +148,19 @@ describe('cursor', () => {
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'CURSOR_INVALID'
|
||||
)
|
||||
})
|
||||
|
||||
it('reuses frozen time bounds embedded in cursor tokens', () => {
|
||||
const firstRange = resolveTimeOptionsForCursor({ last: '7d' }, NOW)
|
||||
const fp = queryFingerprint({
|
||||
command: 'messages.list',
|
||||
startTs: firstRange.startTs ?? null,
|
||||
endTs: firstRange.endTs ?? null,
|
||||
})
|
||||
const token = encodeCursor(50, fp, { time: firstRange })
|
||||
|
||||
const later = new Date(NOW.getTime() + 60_000)
|
||||
const secondRange = resolveTimeOptionsForCursor({ last: '7d', cursor: token }, later)
|
||||
assert.deepEqual(secondRange, firstRange)
|
||||
assert.equal(decodeCursor(token, fp), 50)
|
||||
})
|
||||
})
|
||||
+145
-9
@@ -23,6 +23,10 @@ export interface ResolvedTimeRange {
|
||||
meta: { since: string | null; until: string | null }
|
||||
}
|
||||
|
||||
export interface CursorSnapshot {
|
||||
time?: ResolvedTimeRange
|
||||
}
|
||||
|
||||
/** Format a Date as ISO 8601 with the local UTC offset (sortable, self-describing). */
|
||||
export function toIsoWithOffset(date: Date): string {
|
||||
const pad = (n: number, w = 2) => String(Math.abs(n)).padStart(w, '0')
|
||||
@@ -43,6 +47,7 @@ export function epochToIso(ts: number): string {
|
||||
|
||||
const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/
|
||||
const DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?$/
|
||||
const ISO_DATE_PREFIX = /^(\d{4})-(\d{2})-(\d{2})T/
|
||||
const LAST_DURATION = /^(\d+)([hdw])$/
|
||||
|
||||
function startOfDay(base: Date, dayOffset = 0): Date {
|
||||
@@ -57,6 +62,14 @@ function invalidTime(flag: string, value: string): QueryError {
|
||||
})
|
||||
}
|
||||
|
||||
function isSameLocalDate(date: Date, y: number, m: number, d: number): boolean {
|
||||
return date.getFullYear() === y && date.getMonth() === m - 1 && date.getDate() === d
|
||||
}
|
||||
|
||||
function isSameLocalDateTime(date: Date, y: number, m: number, d: number, h: number, min: number, s: number): boolean {
|
||||
return isSameLocalDate(date, y, m, d) && date.getHours() === h && date.getMinutes() === min && date.getSeconds() === s
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single time value. Date-only values resolve to the start of the day;
|
||||
* when `boundary` is 'end' they resolve to the end of the day (inclusive whole-day).
|
||||
@@ -71,23 +84,43 @@ function parseTimeValue(value: string, boundary: 'start' | 'end', flag: string,
|
||||
const dateOnly = DATE_ONLY.exec(value)
|
||||
if (dateOnly) {
|
||||
const [, y, m, d] = dateOnly
|
||||
const base = new Date(Number(y), Number(m) - 1, Number(d))
|
||||
if (Number.isNaN(base.getTime())) throw invalidTime(flag, value)
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const base = new Date(year, month - 1, day)
|
||||
if (Number.isNaN(base.getTime()) || !isSameLocalDate(base, year, month, day)) throw invalidTime(flag, value)
|
||||
if (boundary === 'start') return base
|
||||
return new Date(new Date(Number(y), Number(m) - 1, Number(d) + 1).getTime() - 1000)
|
||||
return new Date(new Date(year, month - 1, day + 1).getTime() - 1000)
|
||||
}
|
||||
|
||||
const dateTime = DATE_TIME.exec(value)
|
||||
if (dateTime) {
|
||||
const [, y, m, d, h, min, s] = dateTime
|
||||
const parsed = new Date(Number(y), Number(m) - 1, Number(d), Number(h), Number(min), Number(s ?? 0))
|
||||
if (Number.isNaN(parsed.getTime())) throw invalidTime(flag, value)
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const hour = Number(h)
|
||||
const minute = Number(min)
|
||||
const second = Number(s ?? 0)
|
||||
const parsed = new Date(year, month - 1, day, hour, minute, second)
|
||||
if (Number.isNaN(parsed.getTime()) || !isSameLocalDateTime(parsed, year, month, day, hour, minute, second)) {
|
||||
throw invalidTime(flag, value)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// full ISO 8601 (with timezone offset or Z)
|
||||
const parsed = new Date(value)
|
||||
if (!Number.isNaN(parsed.getTime()) && /\d{4}-\d{2}-\d{2}T/.test(value)) {
|
||||
const isoDate = ISO_DATE_PREFIX.exec(value)
|
||||
if (isoDate) {
|
||||
const [, y, m, d] = isoDate
|
||||
const year = Number(y)
|
||||
const month = Number(m)
|
||||
const day = Number(d)
|
||||
const localDate = new Date(year, month - 1, day)
|
||||
if (!isSameLocalDate(localDate, year, month, day)) throw invalidTime(flag, value)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
@@ -161,15 +194,19 @@ export function parseLimit(
|
||||
value: string | number | undefined,
|
||||
defaultValue: number,
|
||||
cap: number,
|
||||
flag: string
|
||||
flag: string,
|
||||
min = 0
|
||||
): number {
|
||||
if (value === undefined) return Math.min(defaultValue, cap)
|
||||
const n = typeof value === 'number' ? value : Number(value)
|
||||
if (!Number.isInteger(n) || n < 0) {
|
||||
if (!Number.isInteger(n) || n < min) {
|
||||
throw new QueryError({
|
||||
code: 'INVALID_ARGUMENT',
|
||||
message: `Invalid ${flag} value: ${value}`,
|
||||
hint: `${flag} must be a non-negative integer (max ${cap})`,
|
||||
hint:
|
||||
min > 0
|
||||
? `${flag} must be an integer from ${min} to ${cap}`
|
||||
: `${flag} must be a non-negative integer (max ${cap})`,
|
||||
})
|
||||
}
|
||||
return Math.min(n, cap)
|
||||
@@ -183,12 +220,111 @@ export function queryFingerprint(conditions: Record<string, unknown>): string {
|
||||
return createHash('sha256').update(canonical).digest('hex').slice(0, 12)
|
||||
}
|
||||
|
||||
export function encodeCursor(offset: number, fingerprint: string): string {
|
||||
export function freezeTimeRangeForCursor(time: ResolvedTimeRange): ResolvedTimeRange {
|
||||
if (time.endTs !== undefined || time.meta.until === null) return time
|
||||
const endMs = new Date(time.meta.until).getTime()
|
||||
if (Number.isNaN(endMs)) return time
|
||||
return { ...time, endTs: Math.floor(endMs / 1000) }
|
||||
}
|
||||
|
||||
export function resolveTimeOptionsForCursor(
|
||||
options: TimeCliOptions & { cursor?: string },
|
||||
now = new Date()
|
||||
): ResolvedTimeRange {
|
||||
const snapshot = options.cursor ? getCursorSnapshot(options.cursor) : undefined
|
||||
return snapshot?.time ?? freezeTimeRangeForCursor(parseTimeOptions(options, now))
|
||||
}
|
||||
|
||||
export function encodeCursor(offset: number, fingerprint: string, snapshot?: CursorSnapshot): string {
|
||||
if (snapshot?.time) {
|
||||
return Buffer.from(JSON.stringify({ v: 2, fp: fingerprint, offset, time: snapshot.time }), 'utf8').toString(
|
||||
'base64url'
|
||||
)
|
||||
}
|
||||
return Buffer.from(`${fingerprint}:${offset}`, 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
function asObject(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null
|
||||
}
|
||||
|
||||
function decodeCursorEnvelope(
|
||||
token: string
|
||||
): { fingerprint: string; offset: number; snapshot?: CursorSnapshot } | null {
|
||||
let decoded = ''
|
||||
try {
|
||||
decoded = Buffer.from(token, 'base64url').toString('utf8')
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
let payload: unknown
|
||||
try {
|
||||
payload = JSON.parse(decoded)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
const object = asObject(payload)
|
||||
if (
|
||||
!object ||
|
||||
object.v !== 2 ||
|
||||
typeof object.fp !== 'string' ||
|
||||
typeof object.offset !== 'number' ||
|
||||
!Number.isInteger(object.offset) ||
|
||||
object.offset < 0
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
const snapshot = decodeCursorSnapshot(object)
|
||||
return { fingerprint: object.fp, offset: object.offset, ...(snapshot ? { snapshot } : {}) }
|
||||
}
|
||||
|
||||
function decodeCursorSnapshot(payload: Record<string, unknown>): CursorSnapshot | undefined {
|
||||
const timeObject = asObject(payload.time)
|
||||
if (!timeObject) return undefined
|
||||
const metaObject = asObject(timeObject.meta)
|
||||
if (!metaObject) return undefined
|
||||
|
||||
const startTs = timeObject.startTs
|
||||
const endTs = timeObject.endTs
|
||||
const since = metaObject.since
|
||||
const until = metaObject.until
|
||||
if (startTs !== undefined && typeof startTs !== 'number') return undefined
|
||||
if (endTs !== undefined && typeof endTs !== 'number') return undefined
|
||||
if (since !== null && typeof since !== 'string') return undefined
|
||||
if (until !== null && typeof until !== 'string') return undefined
|
||||
|
||||
return {
|
||||
time: {
|
||||
...(startTs !== undefined ? { startTs } : {}),
|
||||
...(endTs !== undefined ? { endTs } : {}),
|
||||
meta: { since, until },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function getCursorSnapshot(token: string): CursorSnapshot | undefined {
|
||||
return decodeCursorEnvelope(token)?.snapshot
|
||||
}
|
||||
|
||||
/** Decode a cursor and validate it against the current query fingerprint. */
|
||||
export function decodeCursor(token: string, fingerprint: string): number {
|
||||
const envelope = decodeCursorEnvelope(token)
|
||||
if (envelope) {
|
||||
if (envelope.fingerprint !== fingerprint) {
|
||||
throw new QueryError({
|
||||
code: 'CURSOR_INVALID',
|
||||
message: 'Cursor does not match the current query conditions',
|
||||
hint: 'Cursors are only valid for the exact query that produced them; re-run the query without --cursor',
|
||||
})
|
||||
}
|
||||
return envelope.offset
|
||||
}
|
||||
|
||||
let decoded = ''
|
||||
try {
|
||||
decoded = Buffer.from(token, 'base64url').toString('utf8')
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import type { ChatLabConfig } from '@openchatlab/config'
|
||||
|
||||
import { assertLegacySqlAllowed } from './register'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
function configWithSql(allowSql: boolean): ChatLabConfig {
|
||||
return {
|
||||
llm: { provider: '', model: '', base_url: '' },
|
||||
data: { user_data_dir: '', electron_migration_done: false },
|
||||
api: { port: 3110, host: '127.0.0.1', token: '', require_auth: false },
|
||||
locale: { lang: '' },
|
||||
ui: { default_session_tab: 'overview', session_gap_threshold: 1800, summary_strategy: 'standard' },
|
||||
cli: { allow_raw: false, allow_sql: allowSql },
|
||||
}
|
||||
}
|
||||
|
||||
describe('legacy query alias SQL gate', () => {
|
||||
it('rejects SQL when cli.allow_sql is disabled', () => {
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(false)),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'SQL_DISABLED'
|
||||
)
|
||||
})
|
||||
|
||||
it('allows SQL when cli.allow_sql is enabled', () => {
|
||||
assert.doesNotThrow(() => assertLegacySqlAllowed(configWithSql(true)))
|
||||
})
|
||||
|
||||
it('rejects message content SQL because the legacy alias has no raw mode', () => {
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT content FROM message'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
assert.throws(
|
||||
() => assertLegacySqlAllowed(configWithSql(true), 'SELECT * FROM message'),
|
||||
(err: unknown) => err instanceof QueryError && err.code === 'INVALID_ARGUMENT'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -4,12 +4,19 @@
|
||||
*/
|
||||
|
||||
import type { Command } from 'commander'
|
||||
import type { ChatLabConfig } from '@openchatlab/config'
|
||||
import { searchMessagesLike, executeReadonlySql } from '@openchatlab/core'
|
||||
import { initRuntime } from '../runtime'
|
||||
import { registerSessionCommands, registerMemberCommands } from './commands-sessions-members'
|
||||
import { registerMessageCommands } from './commands-messages'
|
||||
import { registerStatsCommands } from './commands-stats'
|
||||
import { registerTopicCommands, registerSqlCommands } from './commands-topics-sql'
|
||||
import {
|
||||
registerTopicCommands,
|
||||
registerSqlCommands,
|
||||
assertSqlPrivacyAllowed,
|
||||
assertSqlResultPrivacyAllowed,
|
||||
} from './commands-topics-sql'
|
||||
import { QueryError } from './envelope'
|
||||
|
||||
export function registerQueryCommands(program: Command): void {
|
||||
registerSessionCommands(program)
|
||||
@@ -21,6 +28,17 @@ export function registerQueryCommands(program: Command): void {
|
||||
registerLegacyAliases(program)
|
||||
}
|
||||
|
||||
export function assertLegacySqlAllowed(config: Pick<ChatLabConfig, 'cli'>, sql?: string): void {
|
||||
if (!config.cli.allow_sql) {
|
||||
throw new QueryError({
|
||||
code: 'SQL_DISABLED',
|
||||
message: 'The sql command is disabled by configuration',
|
||||
hint: 'The user can re-enable it with `chatlab config set cli.allow_sql true`',
|
||||
})
|
||||
}
|
||||
if (sql) assertSqlPrivacyAllowed(sql, false)
|
||||
}
|
||||
|
||||
// ==================== Hidden legacy aliases (output kept byte-compatible) ====================
|
||||
|
||||
function registerLegacyAliases(program: Command): void {
|
||||
@@ -67,7 +85,7 @@ function registerLegacyAliases(program: Command): void {
|
||||
console.error(
|
||||
'[Deprecated] `chatlab query` — use `chatlab sql "<SELECT ...>" --session <id>`; alias kept for one major version'
|
||||
)
|
||||
const { dbManager } = initRuntime()
|
||||
const { config, dbManager } = initRuntime()
|
||||
const db = dbManager.open(sessionId)
|
||||
if (!db) {
|
||||
console.error(`Session ${sessionId} not found`)
|
||||
@@ -75,7 +93,9 @@ function registerLegacyAliases(program: Command): void {
|
||||
}
|
||||
|
||||
try {
|
||||
assertLegacySqlAllowed(config, options.sql)
|
||||
const result = executeReadonlySql(db, options.sql)
|
||||
assertSqlResultPrivacyAllowed(result.columns, false)
|
||||
if (options.format === 'json') {
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
} else {
|
||||
@@ -87,6 +107,11 @@ function registerLegacyAliases(program: Command): void {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof QueryError) {
|
||||
console.error(`${err.code}: ${err.message}`)
|
||||
if (err.hint) console.error(`Hint: ${err.hint}`)
|
||||
process.exit(2)
|
||||
}
|
||||
console.error(`SQL error: ${err instanceof Error ? err.message : err}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { adaptWorkerSqlResult } from './worker-sql-result'
|
||||
|
||||
describe('adaptWorkerSqlResult', () => {
|
||||
it('maps the worker limited flag to the shared truncated contract', () => {
|
||||
const result = adaptWorkerSqlResult({
|
||||
columns: ['id'],
|
||||
rows: [[1]],
|
||||
rowCount: 1,
|
||||
duration: 1,
|
||||
limited: true,
|
||||
})
|
||||
|
||||
assert.equal(result.truncated, true)
|
||||
})
|
||||
})
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
SegmentSummaryItem,
|
||||
RawMessage,
|
||||
} from '@openchatlab/tools'
|
||||
import { adaptWorkerSqlResult } from './worker-sql-result'
|
||||
|
||||
function mapSearchMessages(messages: workerManager.SearchMessageResult[]): RawMessage[] {
|
||||
return messages.map((m) => ({
|
||||
@@ -186,7 +187,8 @@ export class WorkerDataProvider implements ToolDataProvider {
|
||||
}
|
||||
|
||||
async executeSql(sql: string, options?: { maxRows?: number }): Promise<unknown> {
|
||||
return this.run(() => workerManager.executeRawSQL(this.sessionId, sql, options?.maxRows))
|
||||
const result = await this.run(() => workerManager.executeRawSQL(this.sessionId, sql, options?.maxRows))
|
||||
return adaptWorkerSqlResult(result)
|
||||
}
|
||||
|
||||
async executeParameterizedSql<T = Record<string, unknown>>(
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
interface WorkerSqlResultLike {
|
||||
limited?: boolean
|
||||
truncated?: boolean
|
||||
}
|
||||
|
||||
export function adaptWorkerSqlResult<T extends WorkerSqlResultLike>(result: T): T & { truncated?: boolean } {
|
||||
return {
|
||||
...result,
|
||||
truncated: result.truncated ?? result.limited,
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ ChatLab 可以作为 Codex、Claude Code、HermesAgent 等外部 Agent 的本地
|
||||
|
||||
- 你已经在使用 Codex、Claude Code、HermesAgent 等外部 Agent,并希望复用它们的订阅、token、模型能力和工作流。
|
||||
- 你希望外部 Agent 能按需分析 ChatLab 里的聊天记录,但不想把原始数据库或完整聊天导出直接交给 Agent。
|
||||
- 你希望先由 ChatLab 完成脱敏、黑名单过滤、清洗、合并和截断,再把适合分析的文本与证据 id 提供给 Agent。
|
||||
- 你希望先由 ChatLab 完成脱敏、黑名单过滤、清洗、合并和截断,再把适合分析的文本与可追溯的单条消息 id 提供给 Agent。
|
||||
|
||||
`chatlab` CLI 提供一组面向 AI Agent 优化、同时人类可读的查询命令。所有查询命令只读,默认应用你在 ChatLab 中配置的隐私预处理(脱敏、黑名单)。
|
||||
|
||||
@@ -47,7 +47,7 @@ chatlab manifest # 机读命令清单(给 Agent 一次读
|
||||
|
||||
通过 `--format` 显式指定:
|
||||
|
||||
- **`agent`**:推荐给 AI Agent 的格式。返回 JSON 信封,正文是经过完整预处理管道(清洗、黑名单、去噪、合并连续发言、脱敏、token 截断)的紧凑文本,每条消息带 `[#id]` 引用标记,搜索命中标 `[#id*]`。
|
||||
- **`agent`**:推荐给 AI Agent 的格式。返回 JSON 信封,正文是经过完整预处理管道(清洗、黑名单、去噪、合并连续发言、脱敏、token 截断)的紧凑文本;单条消息标记 `[#id]` / `[#id*]` 可用于 `messages context --id`,合并块范围 `[#a-b]` 仅用于展示。
|
||||
- **`json`**:程序化解析格式。返回结构化消息数组,仍应用隐私步骤(清洗、黑名单、脱敏),但不合并、不去噪。适合 `--no-content`、`--fields` 之类的结构化勘察。
|
||||
- **`text`**:人类可读输出(终端默认)。
|
||||
|
||||
@@ -73,7 +73,7 @@ agent/json 模式下 stdout 只包含一个 JSON 响应信封,日志一律走
|
||||
|
||||
## 隐私边界
|
||||
|
||||
- 所有查询命令默认应用你在 ChatLab 设置中的脱敏规则与黑名单,包括 `stats keywords` 的词表、`topics list` 的摘要和 `sql` 结果中的字符串字段。
|
||||
- 所有查询命令默认应用你在 ChatLab 设置中的脱敏规则与黑名单,包括 `stats keywords` 的词表、`topics list` 的摘要和 `sql` 结果中的字符串字段;`sql` 读取消息 `content` 列需要显式 `--raw`。
|
||||
- `--raw`(绕过预处理)默认禁用,仅当你显式执行 `chatlab config set cli.allow_raw true` 或设置 `CHATLAB_CLI_ALLOW_RAW=1` 后可用。
|
||||
- 不需要 SQL 兜底能力时,可用 `chatlab config set cli.allow_sql false` 关闭 `sql` 命令。
|
||||
|
||||
@@ -93,10 +93,10 @@ chatlab-analyze 帮我分析我和小红的聊天记录
|
||||
|
||||
这个技能会引导 Agent 先执行 `chatlab manifest` 获取机读命令清单,再用安全的 `--format agent/json` 查询聊天记录。
|
||||
|
||||
典型配方——"谁最早提到某个问题?给出证据":
|
||||
典型配方——"谁最早提到某个问题?查看现场":
|
||||
|
||||
```bash
|
||||
chatlab messages search "服务器迁移" --sort asc --limit 5 --context 3 --format agent
|
||||
# 从返回文本的 [#1021*] 标记拿到消息 id,再深挖现场:
|
||||
# 从返回文本的单条消息标记 [#1021*] 拿到消息 id,再深挖现场:
|
||||
chatlab messages context --id 1021 --window 10 --format agent
|
||||
```
|
||||
@@ -12,7 +12,7 @@ This mode does not require opening the ChatLab web UI, and it does not require s
|
||||
|
||||
- You already use an external agent such as Codex, Claude Code, or HermesAgent, and want to reuse its subscription, tokens, model capabilities, and workflow.
|
||||
- You want the agent to analyze ChatLab records on demand, but you do not want to hand it the raw database or a full chat export.
|
||||
- You want ChatLab to handle desensitization, blacklist filtering, cleaning, merging, and truncation before sending analysis-ready text and evidence ids to the agent.
|
||||
- You want ChatLab to handle desensitization, blacklist filtering, cleaning, merging, and truncation before sending analysis-ready text and traceable single-message ids to the agent.
|
||||
|
||||
The `chatlab` CLI ships a set of query commands optimized for AI agents while staying human-readable. All query commands are read-only and apply your ChatLab privacy preprocessing (desensitization, blacklist) by default.
|
||||
|
||||
@@ -47,7 +47,7 @@ With a single session `--session` can be omitted; with multiple sessions, comman
|
||||
|
||||
Pass `--format` explicitly:
|
||||
|
||||
- **`agent`**: recommended for AI agents. A JSON envelope whose body is compact text produced by the full preprocessing pipeline (cleaning, blacklist, denoising, merging consecutive messages, desensitization, token-aware truncation). Every message carries a `[#id]` citation marker; search hits are marked `[#id*]`.
|
||||
- **`agent`**: recommended for AI agents. A JSON envelope whose body is compact text produced by the full preprocessing pipeline (cleaning, blacklist, denoising, merging consecutive messages, desensitization, token-aware truncation). Single-message markers `[#id]` / `[#id*]` can be used with `messages context --id`; merged ranges such as `[#a-b]` are display-only.
|
||||
- **`json`**: for programmatic parsing. Structured message items with privacy steps applied (cleaning, blacklist, desensitization) but no merging or denoising. The right choice for structural scouting with `--no-content` / `--fields`.
|
||||
- **`text`**: human-readable output (TTY default).
|
||||
|
||||
@@ -73,7 +73,7 @@ Failures return `{ "ok": false, "error": { "code", "message", "hint", "candidate
|
||||
|
||||
## Privacy boundaries
|
||||
|
||||
- All query commands apply your ChatLab desensitize rules and blacklist by default — including the `stats keywords` vocabulary, `topics list` summaries, and string cells in `sql` results.
|
||||
- All query commands apply your ChatLab desensitize rules and blacklist by default — including the `stats keywords` vocabulary, `topics list` summaries, and string cells in `sql` results; reading the message `content` column with `sql` requires explicit `--raw`.
|
||||
- `--raw` (bypassing preprocessing) is disabled by default; it only works after you explicitly run `chatlab config set cli.allow_raw true` or set `CHATLAB_CLI_ALLOW_RAW=1`.
|
||||
- If you don't need the SQL fallback, disable it with `chatlab config set cli.allow_sql false`.
|
||||
|
||||
@@ -93,10 +93,10 @@ chatlab-analyze help me analyze my chat history with Alice
|
||||
|
||||
The skill tells the agent to run `chatlab manifest` first, then query chat records safely with explicit `--format agent/json` commands.
|
||||
|
||||
A typical recipe — "who mentioned this first? give evidence":
|
||||
A typical recipe — "who mentioned this first? inspect the surrounding context":
|
||||
|
||||
```bash
|
||||
chatlab messages search "server migration" --sort asc --limit 5 --context 3 --format agent
|
||||
# Take the message id from the [#1021*] marker in the returned text, then dig in:
|
||||
# Take the message id from a single-message [#1021*] marker in the returned text, then dig in:
|
||||
chatlab messages context --id 1021 --window 10 --format agent
|
||||
```
|
||||
@@ -126,6 +126,13 @@ export function writeConfigField(section: string, key: string, value: string | n
|
||||
} catch {
|
||||
// 解析失败时从空开始,旧文件会被覆盖
|
||||
}
|
||||
} else if (fs.existsSync(CONFIG_JSON)) {
|
||||
try {
|
||||
const content = fs.readFileSync(CONFIG_JSON, 'utf-8')
|
||||
existing = JSON.parse(content) as Record<string, Record<string, unknown>>
|
||||
} catch {
|
||||
// Match loadConfigFile: unreadable legacy JSON does not seed TOML writes.
|
||||
}
|
||||
}
|
||||
|
||||
if (!existing[section] || typeof existing[section] !== 'object') {
|
||||
|
||||
@@ -13,10 +13,12 @@ const [{ setConfigField, ConfigSetError }, { loadConfig }] = await Promise.all([
|
||||
])
|
||||
|
||||
const configToml = join(tempHome, '.chatlab', 'config.toml')
|
||||
const configJson = join(tempHome, '.chatlab', 'config.json')
|
||||
|
||||
beforeEach(() => {
|
||||
rmSync(join(tempHome, '.chatlab'), { recursive: true, force: true })
|
||||
delete process.env.CHATLAB_CLI_ALLOW_RAW
|
||||
delete process.env.CHATLAB_API_PORT
|
||||
})
|
||||
|
||||
after(() => {
|
||||
@@ -57,6 +59,39 @@ describe('setConfigField', () => {
|
||||
assert.equal(loadConfig().api.port, 8080)
|
||||
})
|
||||
|
||||
it('validates written file values without environment overrides', () => {
|
||||
process.env.CHATLAB_API_PORT = '3110'
|
||||
|
||||
assert.throws(
|
||||
() => setConfigField('api.port', '70000'),
|
||||
(err: unknown) => {
|
||||
assert.ok(err instanceof ConfigSetError)
|
||||
assert.equal(err.reason, 'invalid_config')
|
||||
return true
|
||||
}
|
||||
)
|
||||
assert.equal(existsSync(configToml), false)
|
||||
})
|
||||
|
||||
it('preserves legacy JSON config when creating TOML', () => {
|
||||
mkdirSync(join(tempHome, '.chatlab'), { recursive: true })
|
||||
writeFileSync(
|
||||
configJson,
|
||||
JSON.stringify({
|
||||
data: { user_data_dir: '/tmp/chatlab-legacy-data' },
|
||||
api: { host: '0.0.0.0' },
|
||||
}),
|
||||
'utf-8'
|
||||
)
|
||||
|
||||
setConfigField('cli.allow_raw', 'true')
|
||||
|
||||
const config = loadConfig()
|
||||
assert.equal(config.cli.allow_raw, true)
|
||||
assert.equal(config.data.user_data_dir, '/tmp/chatlab-legacy-data')
|
||||
assert.equal(config.api.host, '0.0.0.0')
|
||||
})
|
||||
|
||||
it('rejects unknown section or key without touching the file', () => {
|
||||
assert.throws(
|
||||
() => setConfigField('nope.key', 'x'),
|
||||
|
||||
@@ -10,7 +10,7 @@ import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { parse as parseToml } from 'smol-toml'
|
||||
import { configSchema } from './schema'
|
||||
import { loadConfig, writeConfigField, getConfigDir } from './loader'
|
||||
import { writeConfigField, getConfigDir } from './loader'
|
||||
|
||||
export type ConfigSetErrorReason = 'unknown_key' | 'invalid_value' | 'invalid_config' | 'unreadable_config'
|
||||
|
||||
@@ -70,7 +70,7 @@ export function setConfigField(fieldPath: string, rawValue: string): ConfigSetRe
|
||||
writeConfigField(section, key, value)
|
||||
|
||||
try {
|
||||
loadConfig()
|
||||
validateConfigFile(tomlPath)
|
||||
} catch (err) {
|
||||
if (original === null) {
|
||||
fs.unlinkSync(tomlPath)
|
||||
@@ -87,6 +87,12 @@ export function setConfigField(fieldPath: string, rawValue: string): ConfigSetRe
|
||||
return { section, key, value }
|
||||
}
|
||||
|
||||
function validateConfigFile(tomlPath: string): void {
|
||||
const written = fs.readFileSync(tomlPath, 'utf-8')
|
||||
const parsed = parseToml(written) as Record<string, unknown>
|
||||
configSchema.parse(parsed)
|
||||
}
|
||||
|
||||
function parseValueByType(
|
||||
section: string,
|
||||
key: string,
|
||||
|
||||
@@ -118,6 +118,26 @@ describe('search extensions (in-memory SQLite)', () => {
|
||||
assert.equal(wildcard.total, 2)
|
||||
})
|
||||
|
||||
it('search keywords escape SQL wildcards instead of expanding them', () => {
|
||||
raw
|
||||
.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(6, 1, 6000, 0, '包含_下划线和 100% 符号')
|
||||
|
||||
const underscore = searchMessagesByKeywords(db, ['_'], { sort: 'asc' })
|
||||
assert.equal(underscore.total, 1)
|
||||
assert.deepEqual(
|
||||
underscore.messages.map((m) => m.id),
|
||||
[6]
|
||||
)
|
||||
|
||||
const percent = searchMessagesByKeywords(db, ['%'], { sort: 'asc' })
|
||||
assert.equal(percent.total, 2)
|
||||
assert.deepEqual(
|
||||
percent.messages.map((m) => m.id),
|
||||
[4, 6]
|
||||
)
|
||||
})
|
||||
|
||||
it('excludeKeywords keeps NULL-content rows (parity with pipeline blacklist)', () => {
|
||||
const { clause, params } = buildMsgConditions({ excludeKeywords: ['机密'] })
|
||||
const row = db
|
||||
@@ -139,6 +159,24 @@ describe('search extensions (in-memory SQLite)', () => {
|
||||
[3, 2]
|
||||
)
|
||||
})
|
||||
|
||||
it('uses id as a tie-breaker for same-second search paging', () => {
|
||||
const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
|
||||
insert.run(6, 1, 7000, 0, '同秒消息')
|
||||
insert.run(7, 2, 7000, 0, '同秒消息')
|
||||
|
||||
const asc = searchMessagesByKeywords(db, ['同秒'], { sort: 'asc' })
|
||||
assert.deepEqual(
|
||||
asc.messages.map((m) => m.id),
|
||||
[6, 7]
|
||||
)
|
||||
|
||||
const desc = searchMessagesByKeywords(db, ['同秒'])
|
||||
assert.deepEqual(
|
||||
desc.messages.map((m) => m.id),
|
||||
[7, 6]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getConversationBetween pagination and blacklist (in-memory SQLite)', () => {
|
||||
@@ -181,6 +219,18 @@ describe('getConversationBetween pagination and blacklist (in-memory SQLite)', (
|
||||
)
|
||||
})
|
||||
|
||||
it('uses id as a tie-breaker for same-second between paging', () => {
|
||||
const insert = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
|
||||
insert.run(7, 1, 6000, 0, '同秒 A')
|
||||
insert.run(8, 2, 6000, 0, '同秒 B')
|
||||
|
||||
const page = getConversationBetween(db, 1, 2, undefined, 2)
|
||||
assert.deepEqual(
|
||||
page.messages.map((m) => m.id),
|
||||
[7, 8]
|
||||
)
|
||||
})
|
||||
|
||||
it('excludeKeywords removes rows from both page and total (visible counting)', () => {
|
||||
const result = getConversationBetween(db, 1, 2, undefined, 10, { excludeKeywords: ['机密'] })
|
||||
assert.equal(result.total, 4)
|
||||
|
||||
@@ -39,6 +39,7 @@ export interface QueryMessagesOptions {
|
||||
endTs?: number
|
||||
senderId?: number
|
||||
limit?: number
|
||||
maxLimit?: number
|
||||
offset?: number
|
||||
}
|
||||
|
||||
@@ -55,7 +56,8 @@ export interface QueryMessagesResult {
|
||||
* 支持关键词、时间范围、发送者过滤
|
||||
*/
|
||||
export function queryMessages(db: DatabaseAdapter, options?: QueryMessagesOptions): QueryMessagesResult {
|
||||
const limit = Math.min(1000, Math.max(1, options?.limit ?? 100))
|
||||
const maxLimit = Math.max(1, options?.maxLimit ?? 1000)
|
||||
const limit = Math.min(maxLimit, Math.max(1, options?.limit ?? 100))
|
||||
const offset = options?.offset ?? 0
|
||||
const page = Math.floor(offset / limit) + 1
|
||||
|
||||
@@ -240,7 +242,7 @@ export function searchMessagesByKeywords(
|
||||
.get(...params) as { total: number }
|
||||
|
||||
const rows = db
|
||||
.prepare(`${FULL_MSG_SELECT} WHERE 1=1 ${clause} ORDER BY msg.ts ${order} LIMIT ? OFFSET ?`)
|
||||
.prepare(`${FULL_MSG_SELECT} WHERE 1=1 ${clause} ORDER BY msg.ts ${order}, msg.id ${order} LIMIT ? OFFSET ?`)
|
||||
.all(...params, limit + 1, offset) as unknown as FullMessageRow[]
|
||||
|
||||
const hasMore = rows.length > limit
|
||||
@@ -752,7 +754,7 @@ export function getConversationBetween(
|
||||
WHERE msg.sender_id IN (?, ?) ${timeCondition}
|
||||
AND msg.content IS NOT NULL AND msg.content != ''
|
||||
${excludeCondition}
|
||||
ORDER BY msg.ts DESC LIMIT ? OFFSET ?
|
||||
ORDER BY msg.ts DESC, msg.id DESC LIMIT ? OFFSET ?
|
||||
`
|
||||
const rows = db
|
||||
.prepare(sql)
|
||||
|
||||
@@ -163,9 +163,9 @@ export function buildMsgConditions(options?: {
|
||||
}
|
||||
if (options?.keywords && options.keywords.length > 0) {
|
||||
const joiner = options.matchMode === 'all' ? ' AND ' : ' OR '
|
||||
const kwConds = options.keywords.map(() => 'msg.content LIKE ?')
|
||||
const kwConds = options.keywords.map(() => "msg.content LIKE ? ESCAPE '\\'")
|
||||
conds.push(`(${kwConds.join(joiner)})`)
|
||||
params.push(...options.keywords.map((k) => `%${k}%`))
|
||||
params.push(...options.keywords.map((k) => `%${escapeLikePattern(k)}%`))
|
||||
}
|
||||
if (options?.excludeKeywords && options.excludeKeywords.length > 0) {
|
||||
const exclude = buildExcludeKeywordsConditions(options.excludeKeywords)
|
||||
|
||||
@@ -13,7 +13,7 @@ import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { formatToolResultAsText, formatMessageCompact } from './format'
|
||||
import { applyPreprocessingPipeline } from './preprocessing-pipeline'
|
||||
import type { PreprocessableMessage } from './types'
|
||||
import type { PreprocessConfig, PreprocessableMessage } from './types'
|
||||
|
||||
const rawMessages: PreprocessableMessage[] = [
|
||||
{ id: 1, senderName: 'Alice', content: 'hello world', timestamp: 1710000000 },
|
||||
@@ -103,6 +103,16 @@ describe('applyPreprocessingPipeline CLI extensions', () => {
|
||||
{ id: 1021, senderName: '老王', content: '报销流程是不是改了?', timestamp: 1710000000 },
|
||||
{ id: 1022, senderName: '小红', content: '对,三月起走新系统', timestamp: 1710000060 },
|
||||
]
|
||||
const mergeConfig: PreprocessConfig = {
|
||||
dataCleaning: false,
|
||||
mergeConsecutive: true,
|
||||
mergeWindowSeconds: 180,
|
||||
blacklistKeywords: [],
|
||||
denoise: false,
|
||||
desensitize: false,
|
||||
desensitizeRules: [],
|
||||
anonymizeNames: false,
|
||||
}
|
||||
|
||||
it('threads includeMessageIds and hitIds into the rendered text', () => {
|
||||
const result = applyPreprocessingPipeline({
|
||||
@@ -114,6 +124,21 @@ describe('applyPreprocessingPipeline CLI extensions', () => {
|
||||
assert.ok(result.text.includes('[#1022]'))
|
||||
})
|
||||
|
||||
it('marks merged ranges when the later merged message is a search hit', () => {
|
||||
const result = applyPreprocessingPipeline({
|
||||
rawMessages: [
|
||||
{ id: 1021, senderName: '老王', content: '先看下旧流程', timestamp: 1710000000 },
|
||||
{ id: 1022, senderName: '老王', content: '新流程在哪里?', timestamp: 1710000060 },
|
||||
],
|
||||
preprocessConfig: mergeConfig,
|
||||
includeMessageIds: true,
|
||||
hitIds: [1022],
|
||||
})
|
||||
|
||||
assert.ok(result.text.includes('[#1021-1022*]'), result.text)
|
||||
assert.equal(result.stats.mergeCombined, 1)
|
||||
})
|
||||
|
||||
it('keeps default output free of id markers and exposes pipeline stats', () => {
|
||||
const result = applyPreprocessingPipeline({ rawMessages: messagesWithIds })
|
||||
assert.ok(!result.text.includes('[#'))
|
||||
|
||||
@@ -39,7 +39,7 @@ export function t(key: TextEntryKey, locale?: string): string | string[] {
|
||||
const MAX_MESSAGE_CONTENT_LENGTH = 200
|
||||
|
||||
export interface FormatMessageOptions {
|
||||
/** Render an [#id] prefix for message citations (CLI agent format). */
|
||||
/** Render [#id] / [#id*] prefixes, or display-only [#a-b] ranges for merged blocks. */
|
||||
includeMessageId?: boolean
|
||||
/** Message ids that are search hits; rendered as [#id*]. */
|
||||
hitIds?: Set<number>
|
||||
@@ -50,7 +50,7 @@ export interface FormatMessageOptions {
|
||||
/**
|
||||
* 格式化消息为简洁文本格式
|
||||
* 输出格式: "2025/3/3 07:25:04 张三: 消息内容"
|
||||
* includeMessageId 开启时: "[#1021] 2025/3/3 07:25:04 张三: 消息内容"(合并块为 [#start-end])
|
||||
* includeMessageId 开启时: "[#1021] 2025/3/3 07:25:04 张三: 消息内容"(合并块显示为 [#start-end])
|
||||
*/
|
||||
export function formatMessageCompact(
|
||||
msg: {
|
||||
@@ -59,6 +59,7 @@ export function formatMessageCompact(
|
||||
content: string | null
|
||||
timestamp: number
|
||||
mergedEndId?: number
|
||||
mergedIds?: number[]
|
||||
},
|
||||
locale?: string,
|
||||
options?: FormatMessageOptions
|
||||
@@ -75,13 +76,19 @@ export function formatMessageCompact(
|
||||
let idPrefix = ''
|
||||
if (options?.includeMessageId && msg.id != null) {
|
||||
const range = msg.mergedEndId != null && msg.mergedEndId !== msg.id ? `${msg.id}-${msg.mergedEndId}` : `${msg.id}`
|
||||
const hitMark = options.hitIds?.has(msg.id) ? '*' : ''
|
||||
const hitMark = isHitMessage(msg, options.hitIds) ? '*' : ''
|
||||
idPrefix = `[#${range}${hitMark}] `
|
||||
}
|
||||
|
||||
return `${idPrefix}${time} ${msg.senderName}: ${content}`
|
||||
}
|
||||
|
||||
function isHitMessage(msg: { id?: number; mergedIds?: number[] }, hitIds?: Set<number>): boolean {
|
||||
if (!hitIds) return false
|
||||
if (msg.id != null && hitIds.has(msg.id)) return true
|
||||
return msg.mergedIds?.some((id) => hitIds.has(id)) ?? false
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化时间范围用于返回结果
|
||||
*/
|
||||
@@ -129,7 +136,7 @@ export function formatToolResultAsText(details: Record<string, unknown>): string
|
||||
lines.push('')
|
||||
let lastDate = ''
|
||||
for (const msg of messages) {
|
||||
// [#id] citation prefixes (CLI agent format) must survive date grouping at line start
|
||||
// [#id] prefixes (CLI agent format) must survive date grouping at line start.
|
||||
const prefixMatch = /^(\[#[^\]]+\] )/.exec(msg)
|
||||
const prefix = prefixMatch ? prefixMatch[1] : ''
|
||||
const body = prefix ? msg.slice(prefix.length) : msg
|
||||
|
||||
@@ -231,11 +231,16 @@ function applyMergeConsecutive<T extends PreprocessableMessage>(messages: T[], w
|
||||
const withinWindow = Math.abs(msg.timestamp - current.timestamp) <= windowSeconds
|
||||
|
||||
if (sameSender && withinWindow) {
|
||||
const mergedIds = [
|
||||
...(current.mergedIds ?? (current.id != null ? [current.id] : [])),
|
||||
...(msg.mergedIds ?? (msg.id != null ? [msg.id] : [])),
|
||||
]
|
||||
current = {
|
||||
...current,
|
||||
content: [current.content, msg.content].filter(Boolean).join('\n'),
|
||||
// keep first message's id, track last id for [#start-end] range citations
|
||||
mergedEndId: msg.id ?? current.mergedEndId,
|
||||
...(mergedIds.length > 0 ? { mergedIds } : {}),
|
||||
}
|
||||
} else {
|
||||
merged.push(current)
|
||||
|
||||
@@ -53,6 +53,8 @@ export interface PreprocessableMessage {
|
||||
replyToMessageId?: string | null
|
||||
/** 合并连续发言后,块内最后一条消息的 id(用于 [#start-end] 区间引用) */
|
||||
mergedEndId?: number
|
||||
/** 合并连续发言后,块内所有消息 id(用于判断搜索命中标记) */
|
||||
mergedIds?: number[]
|
||||
}
|
||||
|
||||
export type TruncationStrategy = 'keep_first' | 'keep_last'
|
||||
@@ -7,6 +7,7 @@ import { getMessageContextTool } from './get-message-context'
|
||||
import { getConversationBetweenTool } from './get-conversation-between'
|
||||
import { recentMessagesTool } from './recent-messages'
|
||||
import { searchMessagesTool } from './search-messages'
|
||||
import { deepSearchMessagesTool } from './deep-search-messages'
|
||||
import { schemaTool, sqlQueryTool } from './sql-query'
|
||||
import { timeStatsTool } from './time-stats'
|
||||
import { SQL_TOOL_DEFS, createSqlToolDefinition } from '../sql'
|
||||
@@ -96,29 +97,42 @@ describe('high-risk analysis tool definitions', () => {
|
||||
assert.equal(searchCalls[0]?.options?.limit, 12)
|
||||
})
|
||||
|
||||
it('search_messages keeps expanded context within the effective limit', async () => {
|
||||
it('search tools keep expanded context within the effective limit without dropping hits', async () => {
|
||||
const expandedMessages: RawMessage[] = [
|
||||
{ id: 10, senderName: 'Alice', content: 'before', timestamp: 1710000001 },
|
||||
{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 },
|
||||
{ id: 12, senderName: 'Alice', content: 'after', timestamp: 1710000003 },
|
||||
]
|
||||
const context = createContext({
|
||||
async searchMessages() {
|
||||
return {
|
||||
total: 1,
|
||||
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
|
||||
}
|
||||
},
|
||||
async getSearchMessageContext() {
|
||||
return expandedMessages
|
||||
},
|
||||
})
|
||||
|
||||
const result = await searchMessagesTool.handler({ keywords: ['alpha'], limit: 2 }, context)
|
||||
const data = result.data as { returned: number }
|
||||
for (const tool of [searchMessagesTool, deepSearchMessagesTool]) {
|
||||
const context = createContext({
|
||||
async searchMessages() {
|
||||
return {
|
||||
total: 1,
|
||||
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
|
||||
}
|
||||
},
|
||||
async deepSearchMessages() {
|
||||
return {
|
||||
total: 1,
|
||||
messages: [{ id: 11, senderName: 'Bob', content: 'alpha hit', timestamp: 1710000002 }],
|
||||
}
|
||||
},
|
||||
async getSearchMessageContext() {
|
||||
return expandedMessages
|
||||
},
|
||||
})
|
||||
|
||||
assert.equal(data.returned, 2)
|
||||
assert.equal(result.rawMessages?.length, 2)
|
||||
const result = await tool.handler({ keywords: ['alpha'], limit: 1 }, context)
|
||||
const data = result.data as { returned: number }
|
||||
|
||||
assert.equal(data.returned, 1, tool.name)
|
||||
assert.deepEqual(
|
||||
result.rawMessages?.map((message) => message.id),
|
||||
[11],
|
||||
tool.name
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('get_recent_messages respects a user limit smaller than maxMessagesLimit', async () => {
|
||||
@@ -230,6 +244,24 @@ describe('high-risk analysis tool definitions', () => {
|
||||
assert.equal((result.data as { contextSize: number }).contextSize, 1)
|
||||
})
|
||||
|
||||
it('get_message_context clamps negative context_size before applying maxMessagesLimit', async () => {
|
||||
const calls: Array<{ ids: number[]; contextSize: number }> = []
|
||||
const context = createContext(
|
||||
{
|
||||
async getMessageContext(ids, contextSize) {
|
||||
calls.push({ ids, contextSize })
|
||||
return [{ id: 10, senderName: 'Alice', content: 'hit', timestamp: 1710000001 }]
|
||||
},
|
||||
},
|
||||
{ maxMessagesLimit: 12 }
|
||||
)
|
||||
|
||||
const result = await getMessageContextTool.handler({ message_ids: [10], context_size: -1 }, context)
|
||||
|
||||
assert.deepEqual(calls, [{ ids: [10], contextSize: 0 }])
|
||||
assert.equal((result.data as { contextSize: number }).contextSize, 0)
|
||||
})
|
||||
|
||||
it('get_segment_summaries filters empty and non-matching summaries after over-fetching', async () => {
|
||||
const calls: Array<{ limit?: number; timeFilter?: ToolTimeRange }> = []
|
||||
const contextFilter: ToolTimeRange = { startTs: 1704067200, endTs: 1704153600 }
|
||||
|
||||
@@ -9,6 +9,7 @@ import { parseExtendedTimeParams } from '../utils/time-params'
|
||||
import { formatTimeRange } from '../utils/format'
|
||||
import { timeParamProperties } from '../utils/schemas'
|
||||
import { resolveMessageLimit } from '../utils/limits'
|
||||
import { trimMessagesPreservingHits } from './search-context'
|
||||
|
||||
const inputSchema: JsonSchema = {
|
||||
type: 'object',
|
||||
@@ -36,14 +37,14 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
|
||||
const contextBefore = context.searchContextBefore ?? 2
|
||||
const contextAfter = context.searchContextAfter ?? 2
|
||||
let finalMessages = result.messages
|
||||
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
|
||||
|
||||
if ((contextBefore > 0 || contextAfter > 0) && result.messages.length > 0) {
|
||||
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
|
||||
if (hitIds.length > 0) {
|
||||
finalMessages = await context.dataProvider!.getSearchMessageContext(hitIds, contextBefore, contextAfter)
|
||||
}
|
||||
}
|
||||
finalMessages = finalMessages.slice(0, limit)
|
||||
finalMessages = trimMessagesPreservingHits(finalMessages, hitIds, limit)
|
||||
|
||||
const data = {
|
||||
total: result.total,
|
||||
|
||||
@@ -22,7 +22,11 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
|
||||
const requestedMessageIds = params.message_ids as number[]
|
||||
const messageIds =
|
||||
maxMessagesLimit && maxMessagesLimit > 0 ? requestedMessageIds.slice(0, maxMessagesLimit) : requestedMessageIds
|
||||
let contextSize = (params.context_size as number) || 20
|
||||
const requestedContextSize = params.context_size
|
||||
let contextSize =
|
||||
typeof requestedContextSize === 'number' && Number.isFinite(requestedContextSize)
|
||||
? Math.max(0, Math.floor(requestedContextSize))
|
||||
: 20
|
||||
|
||||
if (maxMessagesLimit && maxMessagesLimit > 0 && messageIds.length > 0) {
|
||||
const maxMessagesPerId = Math.max(1, Math.floor(maxMessagesLimit / messageIds.length))
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { RawMessage } from '../types'
|
||||
|
||||
export function trimMessagesPreservingHits(messages: RawMessage[], hitIds: number[], limit: number): RawMessage[] {
|
||||
if (limit <= 0) return []
|
||||
if (messages.length <= limit) return messages
|
||||
|
||||
const requiredIds = new Set(hitIds)
|
||||
if (requiredIds.size === 0) return messages.slice(0, limit)
|
||||
|
||||
const trimmed = messages.slice(0, limit)
|
||||
const presentIds = new Set(trimmed.map((message) => message.id).filter((id): id is number => id != null))
|
||||
const missingHits = messages.filter(
|
||||
(message) => message.id != null && requiredIds.has(message.id) && !presentIds.has(message.id)
|
||||
)
|
||||
|
||||
for (const hit of missingHits) {
|
||||
let replaceIndex = -1
|
||||
for (let i = trimmed.length - 1; i >= 0; i--) {
|
||||
const id = trimmed[i].id
|
||||
if (id == null || !requiredIds.has(id)) {
|
||||
replaceIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (replaceIndex === -1) break
|
||||
trimmed[replaceIndex] = hit
|
||||
}
|
||||
|
||||
return trimmed.sort((a, b) => messages.indexOf(a) - messages.indexOf(b))
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { parseExtendedTimeParams } from '../utils/time-params'
|
||||
import { formatTimeRange } from '../utils/format'
|
||||
import { timeParamProperties } from '../utils/schemas'
|
||||
import { resolveMessageLimit } from '../utils/limits'
|
||||
import { trimMessagesPreservingHits } from './search-context'
|
||||
|
||||
const inputSchema: JsonSchema = {
|
||||
type: 'object',
|
||||
@@ -36,14 +37,14 @@ async function handler(params: Record<string, unknown>, context: ToolExecutionCo
|
||||
const contextBefore = context.searchContextBefore ?? 2
|
||||
const contextAfter = context.searchContextAfter ?? 2
|
||||
let finalMessages = result.messages
|
||||
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
|
||||
|
||||
if ((contextBefore > 0 || contextAfter > 0) && result.messages.length > 0) {
|
||||
const hitIds = result.messages.map((m) => m.id).filter((id): id is number => id != null)
|
||||
if (hitIds.length > 0) {
|
||||
finalMessages = await context.dataProvider!.getSearchMessageContext(hitIds, contextBefore, contextAfter)
|
||||
}
|
||||
}
|
||||
finalMessages = finalMessages.slice(0, limit)
|
||||
finalMessages = trimMessagesPreservingHits(finalMessages, hitIds, limit)
|
||||
|
||||
const data = {
|
||||
total: result.total,
|
||||
|
||||
@@ -146,6 +146,18 @@ describe('CoreDataProvider keyword search', () => {
|
||||
assert.equal(r.messages.length, 1)
|
||||
})
|
||||
|
||||
it('getRecentMessages honors limits above the default queryMessages cap', async () => {
|
||||
const ins = raw.prepare('INSERT INTO message (id, sender_id, ts, type, content) VALUES (?, ?, ?, ?, ?)')
|
||||
for (let i = 0; i < 1005; i++) {
|
||||
ins.run(1000 + i, 1, T0 + 1000 + i, 0, `bulk recent ${i}`)
|
||||
}
|
||||
|
||||
const r = await provider.getRecentMessages({ limit: 1005 })
|
||||
|
||||
assert.equal(r.total, 1009)
|
||||
assert.equal(r.messages.length, 1005)
|
||||
})
|
||||
|
||||
it('passes sort asc through to the core query (default stays desc)', async () => {
|
||||
const asc = await provider.searchMessages(['麻将'], { sort: 'asc' })
|
||||
assert.deepEqual(
|
||||
|
||||
@@ -42,6 +42,8 @@ import type {
|
||||
RawMessage,
|
||||
} from '../types'
|
||||
|
||||
const TOOL_MESSAGE_LIMIT_CAP = 50000
|
||||
|
||||
export class CoreDataProvider implements ToolDataProvider {
|
||||
constructor(private db: DatabaseAdapter) {}
|
||||
|
||||
@@ -84,6 +86,7 @@ export class CoreDataProvider implements ToolDataProvider {
|
||||
async getRecentMessages(options?: { timeFilter?: ToolTimeRange; limit?: number }): Promise<SearchMessagesResult> {
|
||||
const result = queryMessages(this.db, {
|
||||
limit: options?.limit ?? 50,
|
||||
maxLimit: TOOL_MESSAGE_LIMIT_CAP,
|
||||
startTs: options?.timeFilter?.startTs,
|
||||
endTs: options?.timeFilter?.endTs,
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user