mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
refactor(core): extract filter algorithms to @openchatlab/core
Move filterMessagesWithContext and getMultipleSessionsMessages to packages/core/src/query/filter.ts. Functions now accept DatabaseAdapter instead of raw better-sqlite3 Database. Types FilterMessage, ContextBlock, PaginationInfo, FilterResultWithPagination moved to core. Electron filter.ts is now a thin wrapper that opens DB and delegates. Added 6 unit tests with mock DatabaseAdapter. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
f42c8d71ec
commit
abcb1fc28f
9 files changed
+501
-446
No files matched your search
@@ -98,7 +98,7 @@ export type {
|
||||
SessionSearchResultItem,
|
||||
SessionMessagesResult,
|
||||
ContextBlock,
|
||||
FilterResult,
|
||||
FilterStats,
|
||||
FilterMessage,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
|
||||
@@ -1,28 +1,24 @@
|
||||
/**
|
||||
* 自定义筛选模块
|
||||
* 提供按条件筛选消息和获取多会话消息等功能
|
||||
* Filter module — thin adapter over @openchatlab/core filter algorithms.
|
||||
* Opens/closes the DB and delegates to core.
|
||||
*/
|
||||
|
||||
import { BetterSqliteAdapter } from '@openchatlab/node-runtime'
|
||||
import { openReadonlyDatabase } from './core'
|
||||
import type { FilterMessage, ContextBlock, FilterResultWithPagination } from './types'
|
||||
import { FULL_MSG_COLUMNS, FULL_MSG_FROM, mapMessageRow, type FullMessageRow } from '@openchatlab/core'
|
||||
import type { FilterResultWithPagination } from '@openchatlab/core'
|
||||
import {
|
||||
filterMessagesWithContext as filterCore,
|
||||
getMultipleSessionsMessages as getMultiSessionsCore,
|
||||
} from '@openchatlab/core'
|
||||
|
||||
export type { FilterMessage, ContextBlock, FilterResultWithPagination } from '@openchatlab/core'
|
||||
|
||||
const EMPTY_RESULT = (page: number, pageSize: number): FilterResultWithPagination => ({
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
})
|
||||
|
||||
/**
|
||||
* 按条件筛选消息并扩充上下文(支持分页)
|
||||
*
|
||||
* 两阶段查询架构:
|
||||
* 1. 第一阶段:轻量级查询获取消息 ID、序号和匹配信息(不加载完整内容)
|
||||
* 2. 第二阶段:计算上下文范围、合并、分页后只获取当前页的完整消息
|
||||
*
|
||||
* @param sessionId 数据库会话ID
|
||||
* @param keywords 关键词列表(可选,OR 逻辑)
|
||||
* @param timeFilter 时间过滤器(可选)
|
||||
* @param senderIds 发送者ID列表(可选)
|
||||
* @param contextSize 上下文扩展数量(前后各多少条)
|
||||
* @param page 页码(从 1 开始,默认 1)
|
||||
* @param pageSize 每页块数(默认 50)
|
||||
* @returns 筛选结果(带分页信息)
|
||||
*/
|
||||
export function filterMessagesWithContext(
|
||||
sessionId: string,
|
||||
keywords?: string[],
|
||||
@@ -32,321 +28,35 @@ export function filterMessagesWithContext(
|
||||
page: number = 1,
|
||||
pageSize: number = 50
|
||||
): FilterResultWithPagination {
|
||||
const db = openReadonlyDatabase(sessionId)
|
||||
if (!db) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
const rawDb = openReadonlyDatabase(sessionId)
|
||||
if (!rawDb) return EMPTY_RESULT(page, pageSize)
|
||||
|
||||
const db = new BetterSqliteAdapter(rawDb)
|
||||
try {
|
||||
// ==================== 第一阶段:轻量级查询 ====================
|
||||
// 只获取消息的 ID、时间戳、发送者ID、内容(用于匹配)
|
||||
// 使用 ROW_NUMBER() 计算全局序号,避免一次性加载所有完整数据
|
||||
|
||||
const lightweightSql = `
|
||||
SELECT
|
||||
id,
|
||||
ts,
|
||||
sender_id as senderId,
|
||||
content
|
||||
FROM message
|
||||
${timeFilter ? 'WHERE ts >= ? AND ts <= ?' : ''}
|
||||
ORDER BY ts ASC, id ASC
|
||||
`
|
||||
|
||||
const params: unknown[] = []
|
||||
if (timeFilter) {
|
||||
params.push(timeFilter.startTs, timeFilter.endTs)
|
||||
}
|
||||
|
||||
// 使用 iterate() 流式处理,避免一次性加载所有数据到内存
|
||||
const stmt = db.prepare(lightweightSql)
|
||||
const hitIndexes: number[] = []
|
||||
let totalMessageCount = 0
|
||||
let estimatedTotalChars = 0
|
||||
|
||||
// 流式遍历消息,标记命中的索引
|
||||
for (const row of stmt.iterate(...params) as Iterable<{
|
||||
id: number
|
||||
ts: number
|
||||
senderId: number
|
||||
content: string | null
|
||||
}>) {
|
||||
let isHit = true
|
||||
|
||||
// 关键词匹配(OR 逻辑)
|
||||
if (keywords && keywords.length > 0) {
|
||||
const content = (row.content || '').toLowerCase()
|
||||
isHit = keywords.some((kw) => content.includes(kw.toLowerCase()))
|
||||
}
|
||||
|
||||
// 发送者匹配
|
||||
if (isHit && senderIds && senderIds.length > 0) {
|
||||
isHit = senderIds.includes(row.senderId)
|
||||
}
|
||||
|
||||
if (isHit) {
|
||||
hitIndexes.push(totalMessageCount)
|
||||
}
|
||||
|
||||
totalMessageCount++
|
||||
}
|
||||
|
||||
if (hitIndexes.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 计算上下文范围并合并 ====================
|
||||
const ranges: Array<{ start: number; end: number; hitIndexes: number[] }> = []
|
||||
|
||||
for (const hitIndex of hitIndexes) {
|
||||
const start = Math.max(0, hitIndex - contextSize)
|
||||
const end = Math.min(totalMessageCount - 1, hitIndex + contextSize)
|
||||
|
||||
// 检查是否能与前一个范围合并
|
||||
if (ranges.length > 0) {
|
||||
const lastRange = ranges[ranges.length - 1]
|
||||
if (start <= lastRange.end + 1) {
|
||||
lastRange.end = Math.max(lastRange.end, end)
|
||||
lastRange.hitIndexes.push(hitIndex)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ranges.push({ start, end, hitIndexes: [hitIndex] })
|
||||
}
|
||||
|
||||
const totalBlocks = ranges.length
|
||||
const totalHits = hitIndexes.length
|
||||
|
||||
// ==================== 分页处理 ====================
|
||||
const startIndex = (page - 1) * pageSize
|
||||
const endIndex = Math.min(startIndex + pageSize, totalBlocks)
|
||||
const pageRanges = ranges.slice(startIndex, endIndex)
|
||||
const hasMore = endIndex < totalBlocks
|
||||
|
||||
if (pageRanges.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: totalHits, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks, totalHits, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 第二阶段:获取当前页的完整消息 ====================
|
||||
// 只为当前页的范围获取完整消息数据
|
||||
|
||||
const blocks: ContextBlock[] = []
|
||||
let totalMessages = 0
|
||||
let totalChars = 0
|
||||
|
||||
for (const range of pageRanges) {
|
||||
const blockSql = `
|
||||
SELECT ${FULL_MSG_COLUMNS}
|
||||
${FULL_MSG_FROM}
|
||||
${timeFilter ? 'WHERE msg.ts >= ? AND msg.ts <= ?' : ''}
|
||||
ORDER BY msg.ts ASC, msg.id ASC
|
||||
LIMIT ? OFFSET ?
|
||||
`
|
||||
|
||||
const blockParams: unknown[] = []
|
||||
if (timeFilter) {
|
||||
blockParams.push(timeFilter.startTs, timeFilter.endTs)
|
||||
}
|
||||
blockParams.push(range.end - range.start + 1, range.start)
|
||||
|
||||
const rows = db.prepare(blockSql).all(...blockParams) as FullMessageRow[]
|
||||
const mapped = rows.map(mapMessageRow)
|
||||
|
||||
const hitIndexSet = new Set(range.hitIndexes.map((idx) => idx - range.start))
|
||||
|
||||
const blockMessages: FilterMessage[] = mapped.map((msg, i) => ({
|
||||
...msg,
|
||||
isHit: hitIndexSet.has(i),
|
||||
}))
|
||||
|
||||
for (const msg of mapped) {
|
||||
totalChars += msg.content.length
|
||||
}
|
||||
|
||||
if (blockMessages.length > 0) {
|
||||
blocks.push({
|
||||
startTs: blockMessages[0].timestamp,
|
||||
endTs: blockMessages[blockMessages.length - 1].timestamp,
|
||||
messages: blockMessages,
|
||||
hitCount: range.hitIndexes.length,
|
||||
})
|
||||
totalMessages += blockMessages.length
|
||||
}
|
||||
}
|
||||
|
||||
// 如果是第一页,需要估算总字符数(用于统计显示)
|
||||
// 由于我们不再一次性加载所有数据,这里使用采样估算
|
||||
if (page === 1 && totalBlocks > pageSize) {
|
||||
// 估算:当前页的平均字符数 × 总块数
|
||||
const avgCharsPerBlock = totalChars / blocks.length
|
||||
estimatedTotalChars = Math.round(avgCharsPerBlock * totalBlocks)
|
||||
} else if (page === 1) {
|
||||
estimatedTotalChars = totalChars
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
stats: {
|
||||
totalMessages: page === 1 ? totalMessages : 0, // 只有第一页返回准确的消息数
|
||||
hitMessages: totalHits,
|
||||
totalChars: page === 1 ? (totalBlocks > pageSize ? estimatedTotalChars : totalChars) : 0,
|
||||
},
|
||||
pagination: {
|
||||
page,
|
||||
pageSize,
|
||||
totalBlocks,
|
||||
totalHits,
|
||||
hasMore,
|
||||
},
|
||||
}
|
||||
return filterCore(db, { keywords, timeFilter, senderIds, contextSize, page, pageSize })
|
||||
} catch (error) {
|
||||
console.error('filterMessagesWithContext error:', error)
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
return EMPTY_RESULT(page, pageSize)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取多个会话的完整消息(用于会话筛选模式,支持分页)
|
||||
*
|
||||
* @param sessionId 数据库会话ID
|
||||
* @param chatSessionIds 要获取的会话ID列表
|
||||
* @param page 页码(从 1 开始,默认 1)
|
||||
* @param pageSize 每页块数(默认 50)
|
||||
* @returns 合并后的上下文块和统计(带分页信息)
|
||||
*/
|
||||
export function getMultipleSessionsMessages(
|
||||
sessionId: string,
|
||||
chatSessionIds: number[],
|
||||
page: number = 1,
|
||||
pageSize: number = 50
|
||||
): FilterResultWithPagination {
|
||||
const db = openReadonlyDatabase(sessionId)
|
||||
if (!db) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
const rawDb = openReadonlyDatabase(sessionId)
|
||||
if (!rawDb) return EMPTY_RESULT(page, pageSize)
|
||||
|
||||
const db = new BetterSqliteAdapter(rawDb)
|
||||
try {
|
||||
if (chatSessionIds.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
// 先获取会话信息,按时间排序
|
||||
const sessionsSql = `
|
||||
SELECT id, start_ts as startTs, end_ts as endTs, message_count as messageCount
|
||||
FROM chat_session
|
||||
WHERE id IN (${chatSessionIds.map(() => '?').join(',')})
|
||||
ORDER BY start_ts ASC
|
||||
`
|
||||
const allSessions = db.prepare(sessionsSql).all(...chatSessionIds) as Array<{
|
||||
id: number
|
||||
startTs: number
|
||||
endTs: number
|
||||
messageCount: number
|
||||
}>
|
||||
|
||||
const totalBlocks = allSessions.length
|
||||
|
||||
// 分页处理
|
||||
const startIndex = (page - 1) * pageSize
|
||||
const endIndex = Math.min(startIndex + pageSize, totalBlocks)
|
||||
const pageSessions = allSessions.slice(startIndex, endIndex)
|
||||
const hasMore = endIndex < totalBlocks
|
||||
|
||||
if (pageSessions.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
const blocks: ContextBlock[] = []
|
||||
let totalMessages = 0
|
||||
let totalChars = 0
|
||||
|
||||
const messagesSql = `
|
||||
SELECT ${FULL_MSG_COLUMNS}
|
||||
FROM message_context mc
|
||||
JOIN message msg ON msg.id = mc.message_id
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
LEFT JOIN message reply_msg ON msg.reply_to_message_id = reply_msg.platform_message_id
|
||||
LEFT JOIN member reply_m ON reply_msg.sender_id = reply_m.id
|
||||
WHERE mc.session_id = ?
|
||||
ORDER BY msg.ts ASC
|
||||
`
|
||||
|
||||
for (const session of pageSessions) {
|
||||
const rows = db.prepare(messagesSql).all(session.id) as FullMessageRow[]
|
||||
const mapped = rows.map(mapMessageRow)
|
||||
|
||||
const blockMessages: FilterMessage[] = mapped.map((msg) => ({
|
||||
...msg,
|
||||
isHit: false,
|
||||
}))
|
||||
|
||||
for (const msg of mapped) {
|
||||
totalChars += msg.content.length
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
startTs: session.startTs,
|
||||
endTs: session.endTs,
|
||||
messages: blockMessages,
|
||||
hitCount: 0,
|
||||
})
|
||||
|
||||
totalMessages += rows.length
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
stats: {
|
||||
totalMessages: page === 1 ? totalMessages : 0, // 只有第一页返回准确的消息数
|
||||
hitMessages: 0, // 会话模式没有命中概念
|
||||
totalChars: page === 1 ? totalChars : 0,
|
||||
},
|
||||
pagination: {
|
||||
page,
|
||||
pageSize,
|
||||
totalBlocks,
|
||||
totalHits: 0,
|
||||
hasMore,
|
||||
},
|
||||
}
|
||||
return getMultiSessionsCore(db, chatSessionIds, page, pageSize)
|
||||
} catch (error) {
|
||||
console.error('getMultipleSessionsMessages error:', error)
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
return EMPTY_RESULT(page, pageSize)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ export type {
|
||||
SessionMessagesResult,
|
||||
FilterMessage,
|
||||
ContextBlock,
|
||||
FilterResult,
|
||||
FilterStats,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
ExportFilterParams,
|
||||
|
||||
@@ -10,77 +10,14 @@ export type { ChatSessionItem, SessionIndexStats } from '@openchatlab/core'
|
||||
// AI tool types — re-exported from core via aiTools.ts
|
||||
export type { SessionSearchResultItem, SessionMessagesResult } from './aiTools'
|
||||
|
||||
/**
|
||||
* 自定义筛选消息类型(完整信息,兼容 MessageList 组件)
|
||||
*/
|
||||
export interface FilterMessage {
|
||||
id: number
|
||||
senderName: string
|
||||
senderPlatformId: string
|
||||
senderAliases: string[]
|
||||
senderAvatar: string | null
|
||||
content: string
|
||||
timestamp: number
|
||||
type: number
|
||||
replyToMessageId: string | null
|
||||
replyToContent: string | null
|
||||
replyToSenderName: string | null
|
||||
/** 是否为命中的消息(关键词匹配) */
|
||||
isHit: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 上下文块类型(用于自定义筛选)
|
||||
*/
|
||||
export interface ContextBlock {
|
||||
/** 块的时间范围 */
|
||||
startTs: number
|
||||
endTs: number
|
||||
/** 消息列表 */
|
||||
messages: FilterMessage[]
|
||||
/** 命中的消息数量 */
|
||||
hitCount: number
|
||||
}
|
||||
|
||||
/**
|
||||
* 筛选结果类型
|
||||
*/
|
||||
export interface FilterResult {
|
||||
/** 上下文块列表 */
|
||||
blocks: ContextBlock[]
|
||||
/** 统计信息 */
|
||||
stats: {
|
||||
/** 总消息数 */
|
||||
totalMessages: number
|
||||
/** 命中的消息数 */
|
||||
hitMessages: number
|
||||
/** 总字符数 */
|
||||
totalChars: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页信息类型
|
||||
*/
|
||||
export interface PaginationInfo {
|
||||
/** 当前页码(从 1 开始) */
|
||||
page: number
|
||||
/** 每页块数 */
|
||||
pageSize: number
|
||||
/** 总块数 */
|
||||
totalBlocks: number
|
||||
/** 总命中数 */
|
||||
totalHits: number
|
||||
/** 是否还有更多 */
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 带分页的筛选结果类型
|
||||
*/
|
||||
export interface FilterResultWithPagination extends FilterResult {
|
||||
pagination: PaginationInfo
|
||||
}
|
||||
// Filter types — re-exported from @openchatlab/core
|
||||
export type {
|
||||
FilterMessage,
|
||||
ContextBlock,
|
||||
FilterStats,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
} from '@openchatlab/core'
|
||||
|
||||
/**
|
||||
* 导出筛选结果参数
|
||||
|
||||
@@ -883,63 +883,14 @@ export async function getSessionSummaries(
|
||||
|
||||
// ==================== 自定义筛选 API ====================
|
||||
|
||||
/**
|
||||
* 筛选消息类型(完整信息)
|
||||
*/
|
||||
export interface FilterMessage {
|
||||
id: number
|
||||
senderName: string
|
||||
senderPlatformId: string
|
||||
senderAliases: string[]
|
||||
senderAvatar: string | null
|
||||
content: string
|
||||
timestamp: number
|
||||
type: number
|
||||
replyToMessageId: string | null
|
||||
replyToContent: string | null
|
||||
replyToSenderName: string | null
|
||||
isHit: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 上下文块类型
|
||||
*/
|
||||
export interface ContextBlock {
|
||||
startTs: number
|
||||
endTs: number
|
||||
messages: FilterMessage[]
|
||||
hitCount: number
|
||||
}
|
||||
|
||||
/**
|
||||
* 筛选结果类型
|
||||
*/
|
||||
export interface FilterResult {
|
||||
blocks: ContextBlock[]
|
||||
stats: {
|
||||
totalMessages: number
|
||||
hitMessages: number
|
||||
totalChars: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页信息类型
|
||||
*/
|
||||
export interface PaginationInfo {
|
||||
page: number
|
||||
pageSize: number
|
||||
totalBlocks: number
|
||||
totalHits: number
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 带分页的筛选结果类型
|
||||
*/
|
||||
export interface FilterResultWithPagination extends FilterResult {
|
||||
pagination: PaginationInfo
|
||||
}
|
||||
import type {
|
||||
FilterMessage,
|
||||
ContextBlock,
|
||||
FilterStats,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
} from '@openchatlab/core'
|
||||
export type { FilterMessage, ContextBlock, FilterStats, PaginationInfo, FilterResultWithPagination }
|
||||
|
||||
/**
|
||||
* 按条件筛选消息并扩充上下文(支持分页)
|
||||
|
||||
@@ -89,6 +89,8 @@ export {
|
||||
fetchAllRecentMessages,
|
||||
fetchRecentTextMessages,
|
||||
fetchConversationBetween,
|
||||
filterMessagesWithContext,
|
||||
getMultipleSessionsMessages,
|
||||
} from './query'
|
||||
|
||||
// 查询类型
|
||||
@@ -152,6 +154,12 @@ export type {
|
||||
AsyncPaginatedMessages,
|
||||
AsyncMessagesWithTotal,
|
||||
AsyncConversationData,
|
||||
FilterMessage,
|
||||
ContextBlock,
|
||||
FilterStats,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
FilterOptions,
|
||||
} from './query'
|
||||
|
||||
// NLP(平台无关的类型、数据和工具函数)
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Tests for shared filter algorithms (filterMessagesWithContext, getMultipleSessionsMessages).
|
||||
*
|
||||
* Uses a mock DatabaseAdapter to verify the filtering, context expansion,
|
||||
* range merging, and pagination logic.
|
||||
*
|
||||
* Run: npx tsx --test packages/core/src/query/__tests__/filter.test.ts
|
||||
*/
|
||||
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import type { DatabaseAdapter, PreparedStatement, RunResult } from '../../interfaces/database-adapter'
|
||||
import type { FullMessageRow } from '../message-sql'
|
||||
import { filterMessagesWithContext, getMultipleSessionsMessages } from '../filter'
|
||||
|
||||
function makeRow(id: number, content: string, ts: number, senderId: number = 1): FullMessageRow {
|
||||
return {
|
||||
id,
|
||||
senderId,
|
||||
senderName: `User${senderId}`,
|
||||
senderPlatformId: `u${senderId}`,
|
||||
aliasesJson: '[]',
|
||||
senderAvatar: null,
|
||||
content,
|
||||
timestamp: ts,
|
||||
type: 0,
|
||||
replyToMessageId: null,
|
||||
replyToContent: null,
|
||||
replyToSenderName: null,
|
||||
}
|
||||
}
|
||||
|
||||
type LightRow = { id: number; ts: number; senderId: number; content: string | null }
|
||||
|
||||
function createMockDb(lightRows: LightRow[], fullRows: FullMessageRow[]): DatabaseAdapter {
|
||||
let callCount = 0
|
||||
return {
|
||||
exec() {},
|
||||
prepare(): PreparedStatement {
|
||||
callCount++
|
||||
const isLightQuery = callCount === 1
|
||||
return {
|
||||
get(): Record<string, unknown> | undefined {
|
||||
return undefined
|
||||
},
|
||||
all(..._params: unknown[]): Record<string, unknown>[] {
|
||||
if (isLightQuery) {
|
||||
return lightRows as unknown as Record<string, unknown>[]
|
||||
}
|
||||
const limit = typeof _params[_params.length - 2] === 'number' ? (_params[_params.length - 2] as number) : fullRows.length
|
||||
const offset = typeof _params[_params.length - 1] === 'number' ? (_params[_params.length - 1] as number) : 0
|
||||
return fullRows.slice(offset, offset + limit) as unknown as Record<string, unknown>[]
|
||||
},
|
||||
run(): RunResult {
|
||||
return { changes: 0 }
|
||||
},
|
||||
}
|
||||
},
|
||||
transaction<T>(fn: () => T): T {
|
||||
return fn()
|
||||
},
|
||||
pragma() {
|
||||
return undefined
|
||||
},
|
||||
close() {},
|
||||
}
|
||||
}
|
||||
|
||||
describe('filterMessagesWithContext', () => {
|
||||
it('returns empty result when no messages match', () => {
|
||||
const lightRows: LightRow[] = [
|
||||
{ id: 1, ts: 100, senderId: 1, content: 'hello' },
|
||||
{ id: 2, ts: 200, senderId: 2, content: 'world' },
|
||||
]
|
||||
const db = createMockDb(lightRows, [])
|
||||
const result = filterMessagesWithContext(db, { keywords: ['notfound'] })
|
||||
assert.equal(result.blocks.length, 0)
|
||||
assert.equal(result.stats.hitMessages, 0)
|
||||
assert.equal(result.pagination.totalHits, 0)
|
||||
})
|
||||
|
||||
it('matches keywords case-insensitively', () => {
|
||||
const lightRows: LightRow[] = [
|
||||
{ id: 1, ts: 100, senderId: 1, content: 'Hello World' },
|
||||
{ id: 2, ts: 200, senderId: 2, content: 'foo bar' },
|
||||
]
|
||||
const fullRows = [makeRow(1, 'Hello World', 100)]
|
||||
const db = createMockDb(lightRows, fullRows)
|
||||
const result = filterMessagesWithContext(db, { keywords: ['hello'], contextSize: 0 })
|
||||
assert.equal(result.pagination.totalHits, 1)
|
||||
assert.equal(result.blocks.length, 1)
|
||||
assert.equal(result.blocks[0].messages[0].isHit, true)
|
||||
})
|
||||
|
||||
it('filters by sender IDs', () => {
|
||||
const lightRows: LightRow[] = [
|
||||
{ id: 1, ts: 100, senderId: 1, content: 'msg1' },
|
||||
{ id: 2, ts: 200, senderId: 2, content: 'msg2' },
|
||||
{ id: 3, ts: 300, senderId: 1, content: 'msg3' },
|
||||
]
|
||||
const fullRows = [makeRow(1, 'msg1', 100, 1), makeRow(3, 'msg3', 300, 1)]
|
||||
const db = createMockDb(lightRows, fullRows)
|
||||
const result = filterMessagesWithContext(db, { senderIds: [1], contextSize: 0 })
|
||||
assert.equal(result.pagination.totalHits, 2)
|
||||
})
|
||||
|
||||
it('merges overlapping context ranges', () => {
|
||||
const lightRows: LightRow[] = Array.from({ length: 10 }, (_, i) => ({
|
||||
id: i + 1,
|
||||
ts: (i + 1) * 100,
|
||||
senderId: 1,
|
||||
content: i === 2 || i === 4 ? 'MATCH' : 'other',
|
||||
}))
|
||||
const fullRows = lightRows.map((r) => makeRow(r.id, r.content ?? '', r.ts))
|
||||
const db = createMockDb(lightRows, fullRows)
|
||||
const result = filterMessagesWithContext(db, { keywords: ['match'], contextSize: 2 })
|
||||
assert.equal(result.pagination.totalHits, 2)
|
||||
assert.equal(result.blocks.length, 1, 'adjacent hits should merge into one block')
|
||||
})
|
||||
|
||||
it('paginates blocks correctly', () => {
|
||||
const lightRows: LightRow[] = Array.from({ length: 100 }, (_, i) => ({
|
||||
id: i + 1,
|
||||
ts: (i + 1) * 100,
|
||||
senderId: 1,
|
||||
content: i % 20 === 0 ? 'HIT' : 'other',
|
||||
}))
|
||||
const fullRows = lightRows.map((r) => makeRow(r.id, r.content ?? '', r.ts))
|
||||
const db = createMockDb(lightRows, fullRows)
|
||||
const result = filterMessagesWithContext(db, { keywords: ['hit'], contextSize: 0, pageSize: 2, page: 1 })
|
||||
assert.equal(result.blocks.length, 2)
|
||||
assert.equal(result.pagination.hasMore, true)
|
||||
assert.equal(result.pagination.totalBlocks, 5)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getMultipleSessionsMessages', () => {
|
||||
it('returns empty result for empty chatSessionIds', () => {
|
||||
const db = createMockDb([], [])
|
||||
const result = getMultipleSessionsMessages(db, [])
|
||||
assert.equal(result.blocks.length, 0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* Message filter with context expansion and pagination.
|
||||
*
|
||||
* Two-phase query architecture:
|
||||
* 1. Lightweight pass: fetch id/ts/senderId/content to identify hits
|
||||
* 2. Full pass: load complete message data only for the current page's ranges
|
||||
*/
|
||||
|
||||
import type { DatabaseAdapter } from '../interfaces/database-adapter'
|
||||
import { FULL_MSG_COLUMNS, FULL_MSG_FROM, mapMessageRow, type FullMessageRow, type MappedMessage } from './message-sql'
|
||||
|
||||
// ==================== Types ====================
|
||||
|
||||
export interface FilterMessage extends MappedMessage {
|
||||
isHit: boolean
|
||||
}
|
||||
|
||||
export interface ContextBlock {
|
||||
startTs: number
|
||||
endTs: number
|
||||
messages: FilterMessage[]
|
||||
hitCount: number
|
||||
}
|
||||
|
||||
export interface FilterStats {
|
||||
totalMessages: number
|
||||
hitMessages: number
|
||||
totalChars: number
|
||||
}
|
||||
|
||||
export interface PaginationInfo {
|
||||
page: number
|
||||
pageSize: number
|
||||
totalBlocks: number
|
||||
totalHits: number
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
export interface FilterResultWithPagination {
|
||||
blocks: ContextBlock[]
|
||||
stats: FilterStats
|
||||
pagination: PaginationInfo
|
||||
}
|
||||
|
||||
export interface FilterOptions {
|
||||
keywords?: string[]
|
||||
timeFilter?: { startTs: number; endTs: number }
|
||||
senderIds?: number[]
|
||||
contextSize?: number
|
||||
page?: number
|
||||
pageSize?: number
|
||||
}
|
||||
|
||||
// ==================== Helpers ====================
|
||||
|
||||
function emptyResult(page: number, pageSize: number): FilterResultWithPagination {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks: 0, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== Core filter algorithm ====================
|
||||
|
||||
/**
|
||||
* Filter messages by keyword/sender/time with surrounding context, paginated by blocks.
|
||||
*/
|
||||
export function filterMessagesWithContext(
|
||||
db: DatabaseAdapter,
|
||||
options: FilterOptions = {}
|
||||
): FilterResultWithPagination {
|
||||
const { keywords, timeFilter, senderIds, contextSize = 10, page = 1, pageSize = 50 } = options
|
||||
|
||||
// Phase 1: lightweight scan
|
||||
const lightSql = `
|
||||
SELECT id, ts, sender_id as senderId, content
|
||||
FROM message
|
||||
${timeFilter ? 'WHERE ts >= ? AND ts <= ?' : ''}
|
||||
ORDER BY ts ASC, id ASC
|
||||
`
|
||||
const params: unknown[] = []
|
||||
if (timeFilter) {
|
||||
params.push(timeFilter.startTs, timeFilter.endTs)
|
||||
}
|
||||
|
||||
const lightRows = db.prepare(lightSql).all(...params) as Array<{
|
||||
id: number
|
||||
ts: number
|
||||
senderId: number
|
||||
content: string | null
|
||||
}>
|
||||
|
||||
const hitIndexes: number[] = []
|
||||
for (let i = 0; i < lightRows.length; i++) {
|
||||
const row = lightRows[i]
|
||||
let isHit = true
|
||||
|
||||
if (keywords && keywords.length > 0) {
|
||||
const content = (row.content || '').toLowerCase()
|
||||
isHit = keywords.some((kw) => content.includes(kw.toLowerCase()))
|
||||
}
|
||||
if (isHit && senderIds && senderIds.length > 0) {
|
||||
isHit = senderIds.includes(row.senderId)
|
||||
}
|
||||
if (isHit) {
|
||||
hitIndexes.push(i)
|
||||
}
|
||||
}
|
||||
|
||||
if (hitIndexes.length === 0) {
|
||||
return emptyResult(page, pageSize)
|
||||
}
|
||||
|
||||
// Merge overlapping context ranges
|
||||
const ranges: Array<{ start: number; end: number; hitIndexes: number[] }> = []
|
||||
for (const hitIndex of hitIndexes) {
|
||||
const start = Math.max(0, hitIndex - contextSize)
|
||||
const end = Math.min(lightRows.length - 1, hitIndex + contextSize)
|
||||
|
||||
if (ranges.length > 0) {
|
||||
const lastRange = ranges[ranges.length - 1]
|
||||
if (start <= lastRange.end + 1) {
|
||||
lastRange.end = Math.max(lastRange.end, end)
|
||||
lastRange.hitIndexes.push(hitIndex)
|
||||
continue
|
||||
}
|
||||
}
|
||||
ranges.push({ start, end, hitIndexes: [hitIndex] })
|
||||
}
|
||||
|
||||
const totalBlocks = ranges.length
|
||||
const totalHits = hitIndexes.length
|
||||
|
||||
// Paginate blocks
|
||||
const startIdx = (page - 1) * pageSize
|
||||
const endIdx = Math.min(startIdx + pageSize, totalBlocks)
|
||||
const pageRanges = ranges.slice(startIdx, endIdx)
|
||||
const hasMore = endIdx < totalBlocks
|
||||
|
||||
if (pageRanges.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: totalHits, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks, totalHits, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: load full messages for current page ranges
|
||||
const blocks: ContextBlock[] = []
|
||||
let totalMessages = 0
|
||||
let totalChars = 0
|
||||
|
||||
for (const range of pageRanges) {
|
||||
const blockSql = `
|
||||
SELECT ${FULL_MSG_COLUMNS}
|
||||
${FULL_MSG_FROM}
|
||||
${timeFilter ? 'WHERE msg.ts >= ? AND msg.ts <= ?' : ''}
|
||||
ORDER BY msg.ts ASC, msg.id ASC
|
||||
LIMIT ? OFFSET ?
|
||||
`
|
||||
const blockParams: unknown[] = []
|
||||
if (timeFilter) {
|
||||
blockParams.push(timeFilter.startTs, timeFilter.endTs)
|
||||
}
|
||||
blockParams.push(range.end - range.start + 1, range.start)
|
||||
|
||||
const rows = db.prepare(blockSql).all(...blockParams) as unknown as FullMessageRow[]
|
||||
const mapped = rows.map(mapMessageRow)
|
||||
|
||||
const hitIndexSet = new Set(range.hitIndexes.map((idx) => idx - range.start))
|
||||
const blockMessages: FilterMessage[] = mapped.map((msg, i) => ({
|
||||
...msg,
|
||||
isHit: hitIndexSet.has(i),
|
||||
}))
|
||||
|
||||
for (const msg of mapped) {
|
||||
totalChars += msg.content.length
|
||||
}
|
||||
|
||||
if (blockMessages.length > 0) {
|
||||
blocks.push({
|
||||
startTs: blockMessages[0].timestamp,
|
||||
endTs: blockMessages[blockMessages.length - 1].timestamp,
|
||||
messages: blockMessages,
|
||||
hitCount: range.hitIndexes.length,
|
||||
})
|
||||
totalMessages += blockMessages.length
|
||||
}
|
||||
}
|
||||
|
||||
let estimatedTotalChars = totalChars
|
||||
if (page === 1 && totalBlocks > pageSize && blocks.length > 0) {
|
||||
const avgCharsPerBlock = totalChars / blocks.length
|
||||
estimatedTotalChars = Math.round(avgCharsPerBlock * totalBlocks)
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
stats: {
|
||||
totalMessages: page === 1 ? totalMessages : 0,
|
||||
hitMessages: totalHits,
|
||||
totalChars: page === 1 ? (totalBlocks > pageSize ? estimatedTotalChars : totalChars) : 0,
|
||||
},
|
||||
pagination: { page, pageSize, totalBlocks, totalHits, hasMore },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get messages from multiple chat sessions, paginated by session blocks.
|
||||
*/
|
||||
export function getMultipleSessionsMessages(
|
||||
db: DatabaseAdapter,
|
||||
chatSessionIds: number[],
|
||||
page: number = 1,
|
||||
pageSize: number = 50
|
||||
): FilterResultWithPagination {
|
||||
if (chatSessionIds.length === 0) {
|
||||
return emptyResult(page, pageSize)
|
||||
}
|
||||
|
||||
const sessionsSql = `
|
||||
SELECT id, start_ts as startTs, end_ts as endTs, message_count as messageCount
|
||||
FROM chat_session
|
||||
WHERE id IN (${chatSessionIds.map(() => '?').join(',')})
|
||||
ORDER BY start_ts ASC
|
||||
`
|
||||
const allSessions = db.prepare(sessionsSql).all(...chatSessionIds) as Array<{
|
||||
id: number
|
||||
startTs: number
|
||||
endTs: number
|
||||
messageCount: number
|
||||
}>
|
||||
|
||||
const totalBlocks = allSessions.length
|
||||
const startIdx = (page - 1) * pageSize
|
||||
const endIdx = Math.min(startIdx + pageSize, totalBlocks)
|
||||
const pageSessions = allSessions.slice(startIdx, endIdx)
|
||||
const hasMore = endIdx < totalBlocks
|
||||
|
||||
if (pageSessions.length === 0) {
|
||||
return {
|
||||
blocks: [],
|
||||
stats: { totalMessages: 0, hitMessages: 0, totalChars: 0 },
|
||||
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore: false },
|
||||
}
|
||||
}
|
||||
|
||||
const blocks: ContextBlock[] = []
|
||||
let totalMessages = 0
|
||||
let totalChars = 0
|
||||
|
||||
const messagesSql = `
|
||||
SELECT ${FULL_MSG_COLUMNS}
|
||||
FROM message_context mc
|
||||
JOIN message msg ON msg.id = mc.message_id
|
||||
JOIN member m ON msg.sender_id = m.id
|
||||
LEFT JOIN message reply_msg ON msg.reply_to_message_id = reply_msg.platform_message_id
|
||||
LEFT JOIN member reply_m ON reply_msg.sender_id = reply_m.id
|
||||
WHERE mc.session_id = ?
|
||||
ORDER BY msg.ts ASC
|
||||
`
|
||||
|
||||
for (const session of pageSessions) {
|
||||
const rows = db.prepare(messagesSql).all(session.id) as unknown as FullMessageRow[]
|
||||
const mapped = rows.map(mapMessageRow)
|
||||
|
||||
const blockMessages: FilterMessage[] = mapped.map((msg) => ({
|
||||
...msg,
|
||||
isHit: false,
|
||||
}))
|
||||
|
||||
for (const msg of mapped) {
|
||||
totalChars += msg.content.length
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
startTs: session.startTs,
|
||||
endTs: session.endTs,
|
||||
messages: blockMessages,
|
||||
hitCount: 0,
|
||||
})
|
||||
totalMessages += rows.length
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
stats: {
|
||||
totalMessages: page === 1 ? totalMessages : 0,
|
||||
hitMessages: 0,
|
||||
totalChars: page === 1 ? totalChars : 0,
|
||||
},
|
||||
pagination: { page, pageSize, totalBlocks, totalHits: 0, hasMore },
|
||||
}
|
||||
}
|
||||
@@ -121,6 +121,17 @@ export {
|
||||
} from './message-query-functions'
|
||||
export type { AsyncPaginatedMessages, AsyncMessagesWithTotal, AsyncConversationData } from './message-query-functions'
|
||||
|
||||
// Filter with context (keyword/sender/time filter + pagination)
|
||||
export { filterMessagesWithContext, getMultipleSessionsMessages } from './filter'
|
||||
export type {
|
||||
FilterMessage,
|
||||
ContextBlock,
|
||||
FilterStats,
|
||||
PaginationInfo,
|
||||
FilterResultWithPagination,
|
||||
FilterOptions,
|
||||
} from './filter'
|
||||
|
||||
// Advanced analytics
|
||||
export {
|
||||
getCatchphraseAnalysis,
|
||||
|
||||
Reference in new issue
Block a user