refactor(core): extract filter algorithms to @openchatlab/core

Move filterMessagesWithContext and getMultipleSessionsMessages to
packages/core/src/query/filter.ts. Functions now accept DatabaseAdapter
instead of raw better-sqlite3 Database. Types FilterMessage, ContextBlock,
PaginationInfo, FilterResultWithPagination moved to core.
Electron filter.ts is now a thin wrapper that opens DB and delegates.
Added 6 unit tests with mock DatabaseAdapter.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
diguaandCursor committed 2026-05-17 23:22:21 +08:00
1 parent f42c8d71ec
commit abcb1fc28f
9 files changed
+501 -446

No files matched your search

+1 -1
View File
@@ -98,7 +98,7 @@ export type {
SessionSearchResultItem,
SessionMessagesResult,
ContextBlock,
FilterResult,
FilterStats,
FilterMessage,
PaginationInfo,
FilterResultWithPagination,
+26 -316
View File
@@ -1,28 +1,24 @@
/**
* 自定义筛选模块
* 提供按条件筛选消息和获取多会话消息等功能
* Filter module — thin adapter over @openchatlab/core filter algorithms.
* Opens/closes the DB and delegates to core.
*/
import { BetterSqliteAdapter } from '@openchatlab/node-runtime'
import { openReadonlyDatabase } from './core'
import type { FilterMessage, ContextBlock, FilterResultWithPagination } from './types'
import { FULL_MSG_COLUMNS, FULL_MSG_FROM, mapMessageRow, type FullMessageRow } from '@openchatlab/core'
import type { FilterResultWithPagination } from '@openchatlab/core'
import {
filterMessagesWithContext as filterCore,
getMultipleSessionsMessages as getMultiSessionsCore,
} from '@openchatlab/core'
export type { FilterMessage, ContextBlock, FilterResultWithPagination } from '@openchatlab/core'
const EMPTY_RESULT = (page: number, pageSize: number): FilterResultWithPagination => ({
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
})
/**
* 按条件筛选消息并扩充上下文(支持分页)
*
* 两阶段查询架构:
* 1. 第一阶段:轻量级查询获取消息 ID、序号和匹配信息(不加载完整内容)
* 2. 第二阶段:计算上下文范围、合并、分页后只获取当前页的完整消息
*
* @param sessionId 数据库会话ID
* @param keywords 关键词列表(可选,OR 逻辑)
* @param timeFilter 时间过滤器(可选)
* @param senderIds 发送者ID列表(可选)
* @param contextSize 上下文扩展数量(前后各多少条)
* @param page 页码(从 1 开始,默认 1)
* @param pageSize 每页块数(默认 50)
* @returns 筛选结果(带分页信息)
*/
export function filterMessagesWithContext(
sessionId: string,
keywords?: string[],
@@ -32,321 +28,35 @@ export function filterMessagesWithContext(
page: number = 1,
pageSize: number = 50
): FilterResultWithPagination {
const db = openReadonlyDatabase(sessionId)
if (!db) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
}
const rawDb = openReadonlyDatabase(sessionId)
if (!rawDb) return EMPTY_RESULT(page, pageSize)
const db = new BetterSqliteAdapter(rawDb)
try {
// ==================== 第一阶段:轻量级查询 ====================
// 只获取消息的 ID、时间戳、发送者ID、内容(用于匹配)
// 使用 ROW_NUMBER() 计算全局序号,避免一次性加载所有完整数据
const lightweightSql = `
SELECT
id,
ts,
sender_id as senderId,
content
FROM message
${timeFilter ? 'WHERE ts >= ? AND ts <= ?' : ''}
ORDER BY ts ASC, id ASC
`
const params: unknown[] = []
if (timeFilter) {
params.push(timeFilter.startTs, timeFilter.endTs)
}
// 使用 iterate() 流式处理,避免一次性加载所有数据到内存
const stmt = db.prepare(lightweightSql)
const hitIndexes: number[] = []
let totalMessageCount = 0
let estimatedTotalChars = 0
// 流式遍历消息,标记命中的索引
for (const row of stmt.iterate(...params) as Iterable<{
id: number
ts: number
senderId: number
content: string | null
}>) {
let isHit = true
// 关键词匹配(OR 逻辑)
if (keywords && keywords.length > 0) {
const content = (row.content || '').toLowerCase()
isHit = keywords.some((kw) => content.includes(kw.toLowerCase()))
}
// 发送者匹配
if (isHit && senderIds && senderIds.length > 0) {
isHit = senderIds.includes(row.senderId)
}
if (isHit) {
hitIndexes.push(totalMessageCount)
}
totalMessageCount++
}
if (hitIndexes.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
}
// ==================== 计算上下文范围并合并 ====================
const ranges: Array<{ start: number; end: number; hitIndexes: number[] }> = []
for (const hitIndex of hitIndexes) {
const start = Math.max(0, hitIndex - contextSize)
const end = Math.min(totalMessageCount - 1, hitIndex + contextSize)
// 检查是否能与前一个范围合并
if (ranges.length > 0) {
const lastRange = ranges[ranges.length - 1]
if (start <= lastRange.end + 1) {
lastRange.end = Math.max(lastRange.end, end)
lastRange.hitIndexes.push(hitIndex)
continue
}
}
ranges.push({ start, end, hitIndexes: [hitIndex] })
}
const totalBlocks = ranges.length
const totalHits = hitIndexes.length
// ==================== 分页处理 ====================
const startIndex = (page - 1) * pageSize
const endIndex = Math.min(startIndex + pageSize, totalBlocks)
const pageRanges = ranges.slice(startIndex, endIndex)
const hasMore = endIndex < totalBlocks
if (pageRanges.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: totalHits, totalChars: 0 },
pagination: { page, pageSize, totalBlocks, totalHits, hasMore: false },
}
}
// ==================== 第二阶段:获取当前页的完整消息 ====================
// 只为当前页的范围获取完整消息数据
const blocks: ContextBlock[] = []
let totalMessages = 0
let totalChars = 0
for (const range of pageRanges) {
const blockSql = `
SELECT ${FULL_MSG_COLUMNS}
${FULL_MSG_FROM}
${timeFilter ? 'WHERE msg.ts >= ? AND msg.ts <= ?' : ''}
ORDER BY msg.ts ASC, msg.id ASC
LIMIT ? OFFSET ?
`
const blockParams: unknown[] = []
if (timeFilter) {
blockParams.push(timeFilter.startTs, timeFilter.endTs)
}
blockParams.push(range.end - range.start + 1, range.start)
const rows = db.prepare(blockSql).all(...blockParams) as FullMessageRow[]
const mapped = rows.map(mapMessageRow)
const hitIndexSet = new Set(range.hitIndexes.map((idx) => idx - range.start))
const blockMessages: FilterMessage[] = mapped.map((msg, i) => ({
...msg,
isHit: hitIndexSet.has(i),
}))
for (const msg of mapped) {
totalChars += msg.content.length
}
if (blockMessages.length > 0) {
blocks.push({
startTs: blockMessages[0].timestamp,
endTs: blockMessages[blockMessages.length - 1].timestamp,
messages: blockMessages,
hitCount: range.hitIndexes.length,
})
totalMessages += blockMessages.length
}
}
// 如果是第一页,需要估算总字符数(用于统计显示)
// 由于我们不再一次性加载所有数据,这里使用采样估算
if (page === 1 && totalBlocks > pageSize) {
// 估算:当前页的平均字符数 × 总块数
const avgCharsPerBlock = totalChars / blocks.length
estimatedTotalChars = Math.round(avgCharsPerBlock * totalBlocks)
} else if (page === 1) {
estimatedTotalChars = totalChars
}
return {
blocks,
stats: {
totalMessages: page === 1 ? totalMessages : 0, // 只有第一页返回准确的消息数
hitMessages: totalHits,
totalChars: page === 1 ? (totalBlocks > pageSize ? estimatedTotalChars : totalChars) : 0,
},
pagination: {
page,
pageSize,
totalBlocks,
totalHits,
hasMore,
},
}
return filterCore(db, { keywords, timeFilter, senderIds, contextSize, page, pageSize })
} catch (error) {
console.error('filterMessagesWithContext error:', error)
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
return EMPTY_RESULT(page, pageSize)
} finally {
db.close()
}
}
/**
* 获取多个会话的完整消息(用于会话筛选模式,支持分页)
*
* @param sessionId 数据库会话ID
* @param chatSessionIds 要获取的会话ID列表
* @param page 页码(从 1 开始,默认 1)
* @param pageSize 每页块数(默认 50)
* @returns 合并后的上下文块和统计(带分页信息)
*/
export function getMultipleSessionsMessages(
sessionId: string,
chatSessionIds: number[],
page: number = 1,
pageSize: number = 50
): FilterResultWithPagination {
const db = openReadonlyDatabase(sessionId)
if (!db) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
}
const rawDb = openReadonlyDatabase(sessionId)
if (!rawDb) return EMPTY_RESULT(page, pageSize)
const db = new BetterSqliteAdapter(rawDb)
try {
if (chatSessionIds.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
}
// 先获取会话信息,按时间排序
const sessionsSql = `
SELECT id, start_ts as startTs, end_ts as endTs, message_count as messageCount
FROM chat_session
WHERE id IN (${chatSessionIds.map(() => '?').join(',')})
ORDER BY start_ts ASC
`
const allSessions = db.prepare(sessionsSql).all(...chatSessionIds) as Array<{
id: number
startTs: number
endTs: number
messageCount: number
}>
const totalBlocks = allSessions.length
// 分页处理
const startIndex = (page - 1) * pageSize
const endIndex = Math.min(startIndex + pageSize, totalBlocks)
const pageSessions = allSessions.slice(startIndex, endIndex)
const hasMore = endIndex < totalBlocks
if (pageSessions.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore: false },
}
}
const blocks: ContextBlock[] = []
let totalMessages = 0
let totalChars = 0
const messagesSql = `
SELECT ${FULL_MSG_COLUMNS}
FROM message_context mc
JOIN message msg ON msg.id = mc.message_id
JOIN member m ON msg.sender_id = m.id
LEFT JOIN message reply_msg ON msg.reply_to_message_id = reply_msg.platform_message_id
LEFT JOIN member reply_m ON reply_msg.sender_id = reply_m.id
WHERE mc.session_id = ?
ORDER BY msg.ts ASC
`
for (const session of pageSessions) {
const rows = db.prepare(messagesSql).all(session.id) as FullMessageRow[]
const mapped = rows.map(mapMessageRow)
const blockMessages: FilterMessage[] = mapped.map((msg) => ({
...msg,
isHit: false,
}))
for (const msg of mapped) {
totalChars += msg.content.length
}
blocks.push({
startTs: session.startTs,
endTs: session.endTs,
messages: blockMessages,
hitCount: 0,
})
totalMessages += rows.length
}
return {
blocks,
stats: {
totalMessages: page === 1 ? totalMessages : 0, // 只有第一页返回准确的消息数
hitMessages: 0, // 会话模式没有命中概念
totalChars: page === 1 ? totalChars : 0,
},
pagination: {
page,
pageSize,
totalBlocks,
totalHits: 0,
hasMore,
},
}
return getMultiSessionsCore(db, chatSessionIds, page, pageSize)
} catch (error) {
console.error('getMultipleSessionsMessages error:', error)
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
return EMPTY_RESULT(page, pageSize)
} finally {
db.close()
}
+1 -1
View File
@@ -11,7 +11,7 @@ export type {
SessionMessagesResult,
FilterMessage,
ContextBlock,
FilterResult,
FilterStats,
PaginationInfo,
FilterResultWithPagination,
ExportFilterParams,
+8 -71
View File
@@ -10,77 +10,14 @@ export type { ChatSessionItem, SessionIndexStats } from '@openchatlab/core'
// AI tool types — re-exported from core via aiTools.ts
export type { SessionSearchResultItem, SessionMessagesResult } from './aiTools'
/**
* 自定义筛选消息类型(完整信息,兼容 MessageList 组件)
*/
export interface FilterMessage {
id: number
senderName: string
senderPlatformId: string
senderAliases: string[]
senderAvatar: string | null
content: string
timestamp: number
type: number
replyToMessageId: string | null
replyToContent: string | null
replyToSenderName: string | null
/** 是否为命中的消息(关键词匹配) */
isHit: boolean
}
/**
* 上下文块类型(用于自定义筛选)
*/
export interface ContextBlock {
/** 块的时间范围 */
startTs: number
endTs: number
/** 消息列表 */
messages: FilterMessage[]
/** 命中的消息数量 */
hitCount: number
}
/**
* 筛选结果类型
*/
export interface FilterResult {
/** 上下文块列表 */
blocks: ContextBlock[]
/** 统计信息 */
stats: {
/** 总消息数 */
totalMessages: number
/** 命中的消息数 */
hitMessages: number
/** 总字符数 */
totalChars: number
}
}
/**
* 分页信息类型
*/
export interface PaginationInfo {
/** 当前页码(从 1 开始) */
page: number
/** 每页块数 */
pageSize: number
/** 总块数 */
totalBlocks: number
/** 总命中数 */
totalHits: number
/** 是否还有更多 */
hasMore: boolean
}
/**
* 带分页的筛选结果类型
*/
export interface FilterResultWithPagination extends FilterResult {
pagination: PaginationInfo
}
// Filter types — re-exported from @openchatlab/core
export type {
FilterMessage,
ContextBlock,
FilterStats,
PaginationInfo,
FilterResultWithPagination,
} from '@openchatlab/core'
/**
* 导出筛选结果参数
+8 -57
View File
@@ -883,63 +883,14 @@ export async function getSessionSummaries(
// ==================== 自定义筛选 API ====================
/**
* 筛选消息类型(完整信息)
*/
export interface FilterMessage {
id: number
senderName: string
senderPlatformId: string
senderAliases: string[]
senderAvatar: string | null
content: string
timestamp: number
type: number
replyToMessageId: string | null
replyToContent: string | null
replyToSenderName: string | null
isHit: boolean
}
/**
* 上下文块类型
*/
export interface ContextBlock {
startTs: number
endTs: number
messages: FilterMessage[]
hitCount: number
}
/**
* 筛选结果类型
*/
export interface FilterResult {
blocks: ContextBlock[]
stats: {
totalMessages: number
hitMessages: number
totalChars: number
}
}
/**
* 分页信息类型
*/
export interface PaginationInfo {
page: number
pageSize: number
totalBlocks: number
totalHits: number
hasMore: boolean
}
/**
* 带分页的筛选结果类型
*/
export interface FilterResultWithPagination extends FilterResult {
pagination: PaginationInfo
}
import type {
FilterMessage,
ContextBlock,
FilterStats,
PaginationInfo,
FilterResultWithPagination,
} from '@openchatlab/core'
export type { FilterMessage, ContextBlock, FilterStats, PaginationInfo, FilterResultWithPagination }
/**
* 按条件筛选消息并扩充上下文(支持分页)
+8
View File
@@ -89,6 +89,8 @@ export {
fetchAllRecentMessages,
fetchRecentTextMessages,
fetchConversationBetween,
filterMessagesWithContext,
getMultipleSessionsMessages,
} from './query'
// 查询类型
@@ -152,6 +154,12 @@ export type {
AsyncPaginatedMessages,
AsyncMessagesWithTotal,
AsyncConversationData,
FilterMessage,
ContextBlock,
FilterStats,
PaginationInfo,
FilterResultWithPagination,
FilterOptions,
} from './query'
// NLP(平台无关的类型、数据和工具函数)
@@ -0,0 +1,143 @@
/**
* Tests for shared filter algorithms (filterMessagesWithContext, getMultipleSessionsMessages).
*
* Uses a mock DatabaseAdapter to verify the filtering, context expansion,
* range merging, and pagination logic.
*
* Run: npx tsx --test packages/core/src/query/__tests__/filter.test.ts
*/
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import type { DatabaseAdapter, PreparedStatement, RunResult } from '../../interfaces/database-adapter'
import type { FullMessageRow } from '../message-sql'
import { filterMessagesWithContext, getMultipleSessionsMessages } from '../filter'
function makeRow(id: number, content: string, ts: number, senderId: number = 1): FullMessageRow {
return {
id,
senderId,
senderName: `User${senderId}`,
senderPlatformId: `u${senderId}`,
aliasesJson: '[]',
senderAvatar: null,
content,
timestamp: ts,
type: 0,
replyToMessageId: null,
replyToContent: null,
replyToSenderName: null,
}
}
type LightRow = { id: number; ts: number; senderId: number; content: string | null }
function createMockDb(lightRows: LightRow[], fullRows: FullMessageRow[]): DatabaseAdapter {
let callCount = 0
return {
exec() {},
prepare(): PreparedStatement {
callCount++
const isLightQuery = callCount === 1
return {
get(): Record<string, unknown> | undefined {
return undefined
},
all(..._params: unknown[]): Record<string, unknown>[] {
if (isLightQuery) {
return lightRows as unknown as Record<string, unknown>[]
}
const limit = typeof _params[_params.length - 2] === 'number' ? (_params[_params.length - 2] as number) : fullRows.length
const offset = typeof _params[_params.length - 1] === 'number' ? (_params[_params.length - 1] as number) : 0
return fullRows.slice(offset, offset + limit) as unknown as Record<string, unknown>[]
},
run(): RunResult {
return { changes: 0 }
},
}
},
transaction<T>(fn: () => T): T {
return fn()
},
pragma() {
return undefined
},
close() {},
}
}
describe('filterMessagesWithContext', () => {
it('returns empty result when no messages match', () => {
const lightRows: LightRow[] = [
{ id: 1, ts: 100, senderId: 1, content: 'hello' },
{ id: 2, ts: 200, senderId: 2, content: 'world' },
]
const db = createMockDb(lightRows, [])
const result = filterMessagesWithContext(db, { keywords: ['notfound'] })
assert.equal(result.blocks.length, 0)
assert.equal(result.stats.hitMessages, 0)
assert.equal(result.pagination.totalHits, 0)
})
it('matches keywords case-insensitively', () => {
const lightRows: LightRow[] = [
{ id: 1, ts: 100, senderId: 1, content: 'Hello World' },
{ id: 2, ts: 200, senderId: 2, content: 'foo bar' },
]
const fullRows = [makeRow(1, 'Hello World', 100)]
const db = createMockDb(lightRows, fullRows)
const result = filterMessagesWithContext(db, { keywords: ['hello'], contextSize: 0 })
assert.equal(result.pagination.totalHits, 1)
assert.equal(result.blocks.length, 1)
assert.equal(result.blocks[0].messages[0].isHit, true)
})
it('filters by sender IDs', () => {
const lightRows: LightRow[] = [
{ id: 1, ts: 100, senderId: 1, content: 'msg1' },
{ id: 2, ts: 200, senderId: 2, content: 'msg2' },
{ id: 3, ts: 300, senderId: 1, content: 'msg3' },
]
const fullRows = [makeRow(1, 'msg1', 100, 1), makeRow(3, 'msg3', 300, 1)]
const db = createMockDb(lightRows, fullRows)
const result = filterMessagesWithContext(db, { senderIds: [1], contextSize: 0 })
assert.equal(result.pagination.totalHits, 2)
})
it('merges overlapping context ranges', () => {
const lightRows: LightRow[] = Array.from({ length: 10 }, (_, i) => ({
id: i + 1,
ts: (i + 1) * 100,
senderId: 1,
content: i === 2 || i === 4 ? 'MATCH' : 'other',
}))
const fullRows = lightRows.map((r) => makeRow(r.id, r.content ?? '', r.ts))
const db = createMockDb(lightRows, fullRows)
const result = filterMessagesWithContext(db, { keywords: ['match'], contextSize: 2 })
assert.equal(result.pagination.totalHits, 2)
assert.equal(result.blocks.length, 1, 'adjacent hits should merge into one block')
})
it('paginates blocks correctly', () => {
const lightRows: LightRow[] = Array.from({ length: 100 }, (_, i) => ({
id: i + 1,
ts: (i + 1) * 100,
senderId: 1,
content: i % 20 === 0 ? 'HIT' : 'other',
}))
const fullRows = lightRows.map((r) => makeRow(r.id, r.content ?? '', r.ts))
const db = createMockDb(lightRows, fullRows)
const result = filterMessagesWithContext(db, { keywords: ['hit'], contextSize: 0, pageSize: 2, page: 1 })
assert.equal(result.blocks.length, 2)
assert.equal(result.pagination.hasMore, true)
assert.equal(result.pagination.totalBlocks, 5)
})
})
describe('getMultipleSessionsMessages', () => {
it('returns empty result for empty chatSessionIds', () => {
const db = createMockDb([], [])
const result = getMultipleSessionsMessages(db, [])
assert.equal(result.blocks.length, 0)
})
})
+295
View File
@@ -0,0 +1,295 @@
/**
* Message filter with context expansion and pagination.
*
* Two-phase query architecture:
* 1. Lightweight pass: fetch id/ts/senderId/content to identify hits
* 2. Full pass: load complete message data only for the current page's ranges
*/
import type { DatabaseAdapter } from '../interfaces/database-adapter'
import { FULL_MSG_COLUMNS, FULL_MSG_FROM, mapMessageRow, type FullMessageRow, type MappedMessage } from './message-sql'
// ==================== Types ====================
export interface FilterMessage extends MappedMessage {
isHit: boolean
}
export interface ContextBlock {
startTs: number
endTs: number
messages: FilterMessage[]
hitCount: number
}
export interface FilterStats {
totalMessages: number
hitMessages: number
totalChars: number
}
export interface PaginationInfo {
page: number
pageSize: number
totalBlocks: number
totalHits: number
hasMore: boolean
}
export interface FilterResultWithPagination {
blocks: ContextBlock[]
stats: FilterStats
pagination: PaginationInfo
}
export interface FilterOptions {
keywords?: string[]
timeFilter?: { startTs: number; endTs: number }
senderIds?: number[]
contextSize?: number
page?: number
pageSize?: number
}
// ==================== Helpers ====================
function emptyResult(page: number, pageSize: number): FilterResultWithPagination {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
}
}
// ==================== Core filter algorithm ====================
/**
* Filter messages by keyword/sender/time with surrounding context, paginated by blocks.
*/
export function filterMessagesWithContext(
db: DatabaseAdapter,
options: FilterOptions = {}
): FilterResultWithPagination {
const { keywords, timeFilter, senderIds, contextSize = 10, page = 1, pageSize = 50 } = options
// Phase 1: lightweight scan
const lightSql = `
SELECT id, ts, sender_id as senderId, content
FROM message
${timeFilter ? 'WHERE ts >= ? AND ts <= ?' : ''}
ORDER BY ts ASC, id ASC
`
const params: unknown[] = []
if (timeFilter) {
params.push(timeFilter.startTs, timeFilter.endTs)
}
const lightRows = db.prepare(lightSql).all(...params) as Array<{
id: number
ts: number
senderId: number
content: string | null
}>
const hitIndexes: number[] = []
for (let i = 0; i < lightRows.length; i++) {
const row = lightRows[i]
let isHit = true
if (keywords && keywords.length > 0) {
const content = (row.content || '').toLowerCase()
isHit = keywords.some((kw) => content.includes(kw.toLowerCase()))
}
if (isHit && senderIds && senderIds.length > 0) {
isHit = senderIds.includes(row.senderId)
}
if (isHit) {
hitIndexes.push(i)
}
}
if (hitIndexes.length === 0) {
return emptyResult(page, pageSize)
}
// Merge overlapping context ranges
const ranges: Array<{ start: number; end: number; hitIndexes: number[] }> = []
for (const hitIndex of hitIndexes) {
const start = Math.max(0, hitIndex - contextSize)
const end = Math.min(lightRows.length - 1, hitIndex + contextSize)
if (ranges.length > 0) {
const lastRange = ranges[ranges.length - 1]
if (start <= lastRange.end + 1) {
lastRange.end = Math.max(lastRange.end, end)
lastRange.hitIndexes.push(hitIndex)
continue
}
}
ranges.push({ start, end, hitIndexes: [hitIndex] })
}
const totalBlocks = ranges.length
const totalHits = hitIndexes.length
// Paginate blocks
const startIdx = (page - 1) * pageSize
const endIdx = Math.min(startIdx + pageSize, totalBlocks)
const pageRanges = ranges.slice(startIdx, endIdx)
const hasMore = endIdx < totalBlocks
if (pageRanges.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: totalHits, totalChars: 0 },
pagination: { page, pageSize, totalBlocks, totalHits, hasMore: false },
}
}
// Phase 2: load full messages for current page ranges
const blocks: ContextBlock[] = []
let totalMessages = 0
let totalChars = 0
for (const range of pageRanges) {
const blockSql = `
SELECT ${FULL_MSG_COLUMNS}
${FULL_MSG_FROM}
${timeFilter ? 'WHERE msg.ts >= ? AND msg.ts <= ?' : ''}
ORDER BY msg.ts ASC, msg.id ASC
LIMIT ? OFFSET ?
`
const blockParams: unknown[] = []
if (timeFilter) {
blockParams.push(timeFilter.startTs, timeFilter.endTs)
}
blockParams.push(range.end - range.start + 1, range.start)
const rows = db.prepare(blockSql).all(...blockParams) as unknown as FullMessageRow[]
const mapped = rows.map(mapMessageRow)
const hitIndexSet = new Set(range.hitIndexes.map((idx) => idx - range.start))
const blockMessages: FilterMessage[] = mapped.map((msg, i) => ({
...msg,
isHit: hitIndexSet.has(i),
}))
for (const msg of mapped) {
totalChars += msg.content.length
}
if (blockMessages.length > 0) {
blocks.push({
startTs: blockMessages[0].timestamp,
endTs: blockMessages[blockMessages.length - 1].timestamp,
messages: blockMessages,
hitCount: range.hitIndexes.length,
})
totalMessages += blockMessages.length
}
}
let estimatedTotalChars = totalChars
if (page === 1 && totalBlocks > pageSize && blocks.length > 0) {
const avgCharsPerBlock = totalChars / blocks.length
estimatedTotalChars = Math.round(avgCharsPerBlock * totalBlocks)
}
return {
blocks,
stats: {
totalMessages: page === 1 ? totalMessages : 0,
hitMessages: totalHits,
totalChars: page === 1 ? (totalBlocks > pageSize ? estimatedTotalChars : totalChars) : 0,
},
pagination: { page, pageSize, totalBlocks, totalHits, hasMore },
}
}
/**
* Get messages from multiple chat sessions, paginated by session blocks.
*/
export function getMultipleSessionsMessages(
db: DatabaseAdapter,
chatSessionIds: number[],
page: number = 1,
pageSize: number = 50
): FilterResultWithPagination {
if (chatSessionIds.length === 0) {
return emptyResult(page, pageSize)
}
const sessionsSql = `
SELECT id, start_ts as startTs, end_ts as endTs, message_count as messageCount
FROM chat_session
WHERE id IN (${chatSessionIds.map(() => '?').join(',')})
ORDER BY start_ts ASC
`
const allSessions = db.prepare(sessionsSql).all(...chatSessionIds) as Array<{
id: number
startTs: number
endTs: number
messageCount: number
}>
const totalBlocks = allSessions.length
const startIdx = (page - 1) * pageSize
const endIdx = Math.min(startIdx + pageSize, totalBlocks)
const pageSessions = allSessions.slice(startIdx, endIdx)
const hasMore = endIdx < totalBlocks
if (pageSessions.length === 0) {
return {
blocks: [],
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore: false },
}
}
const blocks: ContextBlock[] = []
let totalMessages = 0
let totalChars = 0
const messagesSql = `
SELECT ${FULL_MSG_COLUMNS}
FROM message_context mc
JOIN message msg ON msg.id = mc.message_id
JOIN member m ON msg.sender_id = m.id
LEFT JOIN message reply_msg ON msg.reply_to_message_id = reply_msg.platform_message_id
LEFT JOIN member reply_m ON reply_msg.sender_id = reply_m.id
WHERE mc.session_id = ?
ORDER BY msg.ts ASC
`
for (const session of pageSessions) {
const rows = db.prepare(messagesSql).all(session.id) as unknown as FullMessageRow[]
const mapped = rows.map(mapMessageRow)
const blockMessages: FilterMessage[] = mapped.map((msg) => ({
...msg,
isHit: false,
}))
for (const msg of mapped) {
totalChars += msg.content.length
}
blocks.push({
startTs: session.startTs,
endTs: session.endTs,
messages: blockMessages,
hitCount: 0,
})
totalMessages += rows.length
}
return {
blocks,
stats: {
totalMessages: page === 1 ? totalMessages : 0,
hitMessages: 0,
totalChars: page === 1 ? totalChars : 0,
},
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore },
}
}
+11
View File
@@ -121,6 +121,17 @@ export {
} from './message-query-functions'
export type { AsyncPaginatedMessages, AsyncMessagesWithTotal, AsyncConversationData } from './message-query-functions'
// Filter with context (keyword/sender/time filter + pagination)
export { filterMessagesWithContext, getMultipleSessionsMessages } from './filter'
export type {
FilterMessage,
ContextBlock,
FilterStats,
PaginationInfo,
FilterResultWithPagination,
FilterOptions,
} from './filter'
// Advanced analytics
export {
getCatchphraseAnalysis,