refactor(ai): unify formatAIError and extract FTS search SQL to core

- Extract formatAIError to @openchatlab/node-runtime, eliminate duplicate in Electron and Server
- Extract stripAvatarFields to @openchatlab/core
- Extract FTS message search SQL to core (searchMessagesWithFtsAsync)
- Remove redundant sync searchMessagesWithLike from Electron worker
This commit is contained in:
digua committed 2026-05-18 23:28:39 +08:00
1 parent 0d454d8a52
commit 53fa23f488
10 files changed
+170 -313

No files matched your search

+9 -132
View File
@@ -13,6 +13,7 @@ import { getDefaultGeneralAssistantId } from '../ai/assistant/defaultGeneral'
import { getDefaultAssistantConfig, buildPiModel, findModelDefinition } from '../ai/llm'
import type { AIServiceConfig } from '../ai/llm/types'
import { countMessagesTokens } from '@openchatlab/node-runtime'
import { stripAvatarFields } from '@openchatlab/core'
import * as assistantManager from '../ai/assistant'
import type { AssistantConfig } from '../ai/assistant/types'
import * as skillManager from '../ai/skills'
@@ -23,6 +24,7 @@ import {
type CompressionLlmAdapter,
completeSimple,
runSimpleLlmStream,
formatAIError,
type PiMessage,
type PiTextContent,
} from '@openchatlab/node-runtime'
@@ -88,137 +90,9 @@ function toPiSimpleMessages(messages: Array<{ role: string; content: string }>,
// 用于跟踪活跃的 Agent 请求,支持中止操作
const activeAgentRequests = new Map<string, AbortController>()
/**
* 格式化 AI 报错信息,输出更友好的提示
*/
function formatAIError(error: unknown, provider?: llm.LLMProvider): string {
const candidates: unknown[] = []
if (error) {
candidates.push(error)
}
const errorObj = error as {
lastError?: unknown
errors?: unknown[]
}
if (errorObj?.lastError) {
candidates.push(errorObj.lastError)
}
if (Array.isArray(errorObj?.errors)) {
candidates.push(...errorObj.errors)
}
let rawMessage = ''
let statusCode: number | undefined
let retrySeconds: number | undefined
for (const candidate of candidates) {
if (!candidate || typeof candidate !== 'object') {
if (!rawMessage && typeof candidate === 'string') {
rawMessage = candidate
}
continue
}
const record = candidate as Record<string, unknown>
if (typeof record.statusCode === 'number') {
statusCode = record.statusCode
}
if (!rawMessage && typeof record.message === 'string') {
rawMessage = record.message
}
if (!rawMessage && record.data && typeof record.data === 'object') {
const data = record.data as { error?: { message?: string } }
if (data.error?.message) {
rawMessage = data.error.message
}
}
if (record.responseBody && typeof record.responseBody === 'string') {
const responseBody = record.responseBody
try {
const parsed = JSON.parse(responseBody) as { error?: { message?: string } }
if (!rawMessage && parsed.error?.message) {
rawMessage = parsed.error.message
}
} catch {
if (!rawMessage) {
rawMessage = responseBody
}
}
}
if (rawMessage) {
const retryMatch = rawMessage.match(/retry in ([0-9.]+)s/i)
if (retryMatch) {
retrySeconds = Math.ceil(Number(retryMatch[1]))
}
}
}
const fallbackMessage = rawMessage || String(error)
const lowerMessage = fallbackMessage.toLowerCase()
const providerName =
provider === 'openai-compatible'
? t('llm.genericProviderName')
: provider
? llm.getProviderInfo(provider)?.name || provider
: t('llm.genericProviderName')
let friendlyMessage = ''
if (statusCode === 429 || lowerMessage.includes('quota') || lowerMessage.includes('resource_exhausted')) {
friendlyMessage = retrySeconds
? `${providerName} quota exhausted, please retry after ${retrySeconds}s or upgrade your quota.`
: `${providerName} quota exhausted, please retry later or upgrade your quota.`
} else if (
statusCode === 403 &&
(lowerMessage.includes('quota') || lowerMessage.includes('not enough') || lowerMessage.includes('insufficient'))
) {
friendlyMessage = `${providerName} rejected the request due to insufficient quota or balance.`
} else if (statusCode === 503 || lowerMessage.includes('overloaded') || lowerMessage.includes('unavailable')) {
friendlyMessage = `${providerName} model is overloaded, please retry later.`
} else if (fallbackMessage.length > 300) {
friendlyMessage = `${fallbackMessage.slice(0, 300)}...`
} else {
friendlyMessage = fallbackMessage
}
const details = [statusCode ? `status=${statusCode}` : null, fallbackMessage].filter(Boolean).join('; ')
// 同时返回封装提示和原始错误详情,便于用户定位第三方平台问题。
if (friendlyMessage !== fallbackMessage) {
return `${friendlyMessage}\n\n${t('llm.rawErrorLabel')}: ${details}`
}
return friendlyMessage
}
/**
* 递归剥离对象中的 avatar/senderAvatar 字段(base64 大数据)
* 用于工具测试场景,避免传输和序列化大量无用头像数据
*/
function stripAvatarFields(obj: unknown): void {
if (!obj || typeof obj !== 'object') return
if (Array.isArray(obj)) {
for (const item of obj) stripAvatarFields(item)
return
}
const record = obj as Record<string, unknown>
for (const key of Object.keys(record)) {
if ((key === 'avatar' || key === 'senderAvatar') && typeof record[key] === 'string') {
const val = record[key] as string
if (val.length > 200) {
record[key] = '[stripped]'
}
} else if (typeof record[key] === 'object' && record[key] !== null) {
stripAvatarFields(record[key])
}
}
function resolveProviderName(provider?: llm.LLMProvider): string {
if (provider === 'openai-compatible') return t('llm.genericProviderName')
return provider ? llm.getProviderInfo(provider)?.name || provider : t('llm.genericProviderName')
}
export function registerAIHandlers({ win }: IpcContext): void {
@@ -1305,7 +1179,10 @@ export function registerAIHandlers({ win }: IpcContext): void {
return
}
const serializedError = serializeError(error, activeAIConfig.provider)
serializedError.friendlyMessage = formatAIError(error, activeAIConfig.provider)
serializedError.friendlyMessage = formatAIError(error, {
providerName: resolveProviderName(activeAIConfig.provider),
rawErrorLabel: t('llm.rawErrorLabel'),
})
if (!serializedError.url && activeAIConfig.baseUrl) {
serializedError.url = activeAIConfig.baseUrl
}
+11 -105
View File
@@ -3,23 +3,21 @@
*
* Thin wrapper that delegates to shared async query functions from @openchatlab/core.
* The better-sqlite3 sync calls are wrapped as an AsyncSqlExecutor.
* FTS search remains Electron-specific (depends on @node-rs/jieba tokenizer).
* FTS tokenization depends on @node-rs/jieba (platform-specific);
* the SQL query itself is shared via core's searchMessagesWithFtsAsync.
*/
import { openDatabase, buildTimeFilter, type TimeFilter } from '../core'
import { openDatabase, type TimeFilter } from '../core'
import { ensureAvatarColumn } from './basic'
import { hasFtsIndex } from './fts'
import { tokenizeQueryForFts } from '@openchatlab/node-runtime'
import {
FULL_MSG_SELECT,
FULL_MSG_FROM,
mapMessageRow,
type FullMessageRow,
type MappedMessage,
type AsyncSqlExecutor,
fetchMessagesBefore,
fetchMessagesAfter,
searchMessagesLikeAsync,
searchMessagesWithFtsAsync,
fetchMessageContext,
fetchSearchMessageContext,
fetchAllRecentMessages,
@@ -99,9 +97,8 @@ export async function searchMessages(
senderId?: number
): Promise<MessagesWithTotal> {
ensureAvatarColumn(sessionId)
const db = openDatabase(sessionId)
if (!db) return { messages: [], total: 0 }
const executor = createSyncExecutor(sessionId)
if (!executor) return { messages: [], total: 0 }
const useFts = keywords.length > 0 && hasFtsIndex(sessionId)
let matchQuery = ''
@@ -110,107 +107,16 @@ export async function searchMessages(
}
if (useFts && matchQuery) {
return searchMessagesWithFts(db, sessionId, matchQuery, filter, limit, offset, senderId)
try {
return await searchMessagesWithFtsAsync(executor, matchQuery, filter, limit, offset, senderId)
} catch (error) {
console.error('[FTS] searchMessages FTS path failed, falling back to LIKE:', error)
}
}
const executor = createSyncExecutor(sessionId)!
return searchMessagesLikeAsync(executor, keywords, filter, limit, offset, senderId)
}
/**
* FTS5 search path (Electron-specific, depends on platform tokenizer).
*/
function searchMessagesWithFts(
db: ReturnType<typeof openDatabase> & object,
_sessionId: string,
matchQuery: string,
filter?: TimeFilter,
limit: number = 20,
offset: number = 0,
senderId?: number
): MessagesWithTotal {
const { clause: timeClause, params: timeParams } = buildTimeFilter(filter, 'msg')
const timeCondition = timeClause ? timeClause.replace('WHERE', 'AND') : ''
const senderCondition = senderId !== undefined ? 'AND msg.sender_id = ?' : ''
const senderParams = senderId !== undefined ? [senderId] : []
try {
const countSql = `
SELECT COUNT(*) as total
${FULL_MSG_FROM}
WHERE msg.id IN (SELECT rowid FROM message_fts WHERE content MATCH ?)
${timeCondition}
${senderCondition}
`
const totalRow = db.prepare(countSql).get(matchQuery, ...timeParams, ...senderParams) as { total: number }
const total = totalRow?.total || 0
const sql = `
${FULL_MSG_SELECT}
WHERE msg.id IN (SELECT rowid FROM message_fts WHERE content MATCH ?)
${timeCondition}
${senderCondition}
ORDER BY msg.ts DESC
LIMIT ? OFFSET ?
`
const rows = db.prepare(sql).all(matchQuery, ...timeParams, ...senderParams, limit, offset) as FullMessageRow[]
return { messages: rows.map(mapMessageRow), total }
} catch (error) {
console.error('[FTS] searchMessages FTS path failed, falling back to LIKE:', error)
return searchMessagesWithLike(db, [], filter, limit, offset, senderId)
}
}
/**
* LIKE search path (fallback or deep_search).
* Kept as a synchronous helper for the FTS fallback path.
*/
export function searchMessagesWithLike(
db: ReturnType<typeof openDatabase> & object,
keywords: string[],
filter?: TimeFilter,
limit: number = 20,
offset: number = 0,
senderId?: number
): MessagesWithTotal {
let keywordCondition = '1=1'
const keywordParams: string[] = []
if (keywords.length > 0) {
keywordCondition = `(${keywords.map(() => `msg.content LIKE ?`).join(' OR ')})`
keywordParams.push(...keywords.map((k) => `%${k}%`))
}
const { clause: timeClause, params: timeParams } = buildTimeFilter(filter, 'msg')
const timeCondition = timeClause ? timeClause.replace('WHERE', 'AND') : ''
const senderCondition = senderId !== undefined ? 'AND msg.sender_id = ?' : ''
const senderParams = senderId !== undefined ? [senderId] : []
const countSql = `
SELECT COUNT(*) as total
${FULL_MSG_FROM}
WHERE ${keywordCondition}
${timeCondition}
${senderCondition}
`
const totalRow = db.prepare(countSql).get(...keywordParams, ...timeParams, ...senderParams) as { total: number }
const total = totalRow?.total || 0
const sql = `
${FULL_MSG_SELECT}
WHERE ${keywordCondition}
${timeCondition}
${senderCondition}
ORDER BY msg.ts DESC
LIMIT ? OFFSET ?
`
const rows = db.prepare(sql).all(...keywordParams, ...timeParams, ...senderParams, limit, offset) as FullMessageRow[]
return { messages: rows.map(mapMessageRow), total }
}
/**
* Deep search (always LIKE, no FTS).
*/
+22
View File
@@ -30,3 +30,25 @@ export function extractThinkingContent(content: string): { thinking: string; cle
export function stripToolCallTags(content: string): string {
return content.replace(/<tool_call>[\s\S]*?<\/tool_call>/gi, '').trim()
}
/**
* Recursively strip large avatar/senderAvatar base64 strings from objects.
* Used when serializing tool results to avoid transmitting large image data.
*/
export function stripAvatarFields(obj: unknown): void {
if (!obj || typeof obj !== 'object') return
if (Array.isArray(obj)) {
for (const item of obj) stripAvatarFields(item)
return
}
const record = obj as Record<string, unknown>
for (const key of Object.keys(record)) {
if ((key === 'avatar' || key === 'senderAvatar') && typeof record[key] === 'string') {
if ((record[key] as string).length > 200) {
record[key] = '[stripped]'
}
} else if (typeof record[key] === 'object' && record[key] !== null) {
stripAvatarFields(record[key])
}
}
}
+2 -2
View File
@@ -23,5 +23,5 @@ export { BUILTIN_PROVIDERS, getBuiltinProviderById } from './provider-registry'
// Model Catalog(内置模型目录)
export { BUILTIN_MODELS, getBuiltinModelsByProvider, getBuiltinModelById } from './model-catalog'
// Content parsing (thinking-tag extraction, tool-call stripping)
export { extractThinkingContent, stripToolCallTags } from './content-parser'
// Content parsing (thinking-tag extraction, tool-call stripping, avatar sanitization)
export { extractThinkingContent, stripToolCallTags, stripAvatarFields } from './content-parser'
+2
View File
@@ -88,6 +88,7 @@ export {
fetchMessagesBefore,
fetchMessagesAfter,
searchMessagesLikeAsync,
searchMessagesWithFtsAsync,
fetchMessageContext,
fetchSearchMessageContext,
fetchAllRecentMessages,
@@ -223,6 +224,7 @@ export {
getBuiltinModelById,
extractThinkingContent,
stripToolCallTags,
stripAvatarFields,
} from './ai'
// Import utilities
@@ -102,6 +102,28 @@ export async function fetchMessagesAfter(
* FTS search is NOT included here — platforms should handle FTS themselves
* and fall back to this function when FTS is unavailable.
*/
/**
* FTS5-based message search. The matchQuery must be pre-tokenized for the platform's FTS tokenizer.
*/
export async function searchMessagesWithFtsAsync(
executor: AsyncSqlExecutor,
matchQuery: string,
filter?: TimeFilter,
limit: number = 20,
offset: number = 0,
senderId?: number
): Promise<AsyncMessagesWithTotal> {
const { clause, params } = filterConditions(filter, senderId)
const countSql = `SELECT COUNT(*) as total ${MSG_COUNT_FROM} WHERE msg.id IN (SELECT rowid FROM message_fts WHERE content MATCH ?) ${clause}`
const countRow = await executor.get<{ total: number }>(countSql, [matchQuery, ...params])
const total = countRow?.total ?? 0
const sql = `${FULL_MSG_SELECT} WHERE msg.id IN (SELECT rowid FROM message_fts WHERE content MATCH ?) ${clause} ORDER BY msg.ts DESC LIMIT ? OFFSET ?`
const rows = await executor.all<FullMessageRow>(sql, [matchQuery, ...params, limit, offset])
return { messages: rows.map(mapMessageRow), total }
}
export async function searchMessagesLikeAsync(
executor: AsyncSqlExecutor,
keywords: string[],
@@ -0,0 +1,94 @@
export interface FormatAIErrorOptions {
providerName?: string
rawErrorLabel?: string
}
/**
* Parse error candidates from an LLM API error, extracting statusCode, message, and retry info.
*/
function parseErrorCandidates(error: unknown): {
rawMessage: string
statusCode: number | undefined
retrySeconds: number | undefined
} {
const candidates: unknown[] = []
if (error) candidates.push(error)
const errorObj = error as { lastError?: unknown; errors?: unknown[] }
if (errorObj?.lastError) candidates.push(errorObj.lastError)
if (Array.isArray(errorObj?.errors)) candidates.push(...errorObj.errors)
let rawMessage = ''
let statusCode: number | undefined
let retrySeconds: number | undefined
for (const candidate of candidates) {
if (!candidate || typeof candidate !== 'object') {
if (!rawMessage && typeof candidate === 'string') rawMessage = candidate
continue
}
const record = candidate as Record<string, unknown>
if (typeof record.statusCode === 'number') statusCode = record.statusCode
if (!rawMessage && typeof record.message === 'string') rawMessage = record.message
if (!rawMessage && record.data && typeof record.data === 'object') {
const data = record.data as { error?: { message?: string } }
if (data.error?.message) rawMessage = data.error.message
}
if (record.responseBody && typeof record.responseBody === 'string') {
try {
const parsed = JSON.parse(record.responseBody) as { error?: { message?: string } }
if (!rawMessage && parsed.error?.message) rawMessage = parsed.error.message
} catch {
if (!rawMessage) rawMessage = record.responseBody
}
}
if (rawMessage) {
const retryMatch = rawMessage.match(/retry in ([0-9.]+)s/i)
if (retryMatch) retrySeconds = Math.ceil(Number(retryMatch[1]))
}
}
return { rawMessage: rawMessage || String(error), statusCode, retrySeconds }
}
/**
* Format LLM API errors into user-friendly messages.
* Shared between Electron and Server — provider name and i18n labels are injected via options.
*/
export function formatAIError(error: unknown, options?: FormatAIErrorOptions): string {
const { rawMessage: fallbackMessage, statusCode, retrySeconds } = parseErrorCandidates(error)
const lowerMessage = fallbackMessage.toLowerCase()
const providerName = options?.providerName || 'API'
let friendlyMessage = ''
if (statusCode === 429 || lowerMessage.includes('quota') || lowerMessage.includes('resource_exhausted')) {
friendlyMessage = retrySeconds
? `${providerName} quota exhausted, please retry after ${retrySeconds}s or upgrade your quota.`
: `${providerName} quota exhausted, please retry later or upgrade your quota.`
} else if (
statusCode === 403 &&
(lowerMessage.includes('quota') || lowerMessage.includes('not enough') || lowerMessage.includes('insufficient'))
) {
friendlyMessage = `${providerName} rejected the request due to insufficient quota or balance.`
} else if (statusCode === 503 || lowerMessage.includes('overloaded') || lowerMessage.includes('unavailable')) {
friendlyMessage = `${providerName} model is overloaded, please retry later.`
} else if (fallbackMessage.length > 300) {
friendlyMessage = `${fallbackMessage.slice(0, 300)}...`
} else {
friendlyMessage = fallbackMessage
}
const rawErrorLabel = options?.rawErrorLabel || 'Raw error'
const details = [statusCode ? `status=${statusCode}` : null, fallbackMessage].filter(Boolean).join('; ')
if (friendlyMessage !== fallbackMessage) {
return `${friendlyMessage}\n\n${rawErrorLabel}: ${details}`
}
return friendlyMessage
}
+4
View File
@@ -7,6 +7,10 @@
// AI Logger
export { AiLogger, extractErrorInfo, extractErrorStack } from './ai-logger'
// Error formatting
export { formatAIError } from './error-formatter'
export type { FormatAIErrorOptions } from './error-formatter'
export type { AssistantConfig, AssistantSummary, SkillDef, SkillSummary } from './types'
export { parseAssistantFile, serializeAssistant } from './assistant-parser'
+3 -2
View File
@@ -49,8 +49,9 @@ export {
// FTS5 full-text search operations
export { hasFtsTable, createFtsTable, buildFtsIndex, rebuildFtsIndex, insertFtsEntries, searchByFts } from './fts'
// AI Logger
export { AiLogger, extractErrorInfo, extractErrorStack } from './ai'
// AI Logger & Error formatting
export { AiLogger, extractErrorInfo, extractErrorStack, formatAIError } from './ai'
export type { FormatAIErrorOptions } from './ai'
export { NodePathProvider } from './node-path-provider'
export { DatabaseManager } from './database-manager'
export { createJiebaNlpProvider } from './jieba-nlp-provider'
+1 -72
View File
@@ -12,6 +12,7 @@ import {
buildSystemPrompt,
createAiTranslate,
AgentEventHandler,
formatAIError,
type AgentStreamChunk,
type PiMessage,
type SimpleHistoryMessage,
@@ -48,78 +49,6 @@ export interface RunAgentOptions {
dataSnapshot?: DataSnapshot
}
/**
* Format AI errors with user-friendly messages (mirrors Electron's formatAIError)
*/
function formatAIError(error: unknown): string {
const candidates: unknown[] = []
if (error) candidates.push(error)
const errorObj = error as { lastError?: unknown; errors?: unknown[] }
if (errorObj?.lastError) candidates.push(errorObj.lastError)
if (Array.isArray(errorObj?.errors)) candidates.push(...errorObj.errors)
let rawMessage = ''
let statusCode: number | undefined
let retrySeconds: number | undefined
for (const candidate of candidates) {
if (!candidate || typeof candidate !== 'object') {
if (!rawMessage && typeof candidate === 'string') rawMessage = candidate
continue
}
const record = candidate as Record<string, unknown>
if (typeof record.statusCode === 'number') statusCode = record.statusCode
if (!rawMessage && typeof record.message === 'string') rawMessage = record.message
if (!rawMessage && record.data && typeof record.data === 'object') {
const data = record.data as { error?: { message?: string } }
if (data.error?.message) rawMessage = data.error.message
}
if (record.responseBody && typeof record.responseBody === 'string') {
try {
const parsed = JSON.parse(record.responseBody) as { error?: { message?: string } }
if (!rawMessage && parsed.error?.message) rawMessage = parsed.error.message
} catch {
if (!rawMessage) rawMessage = record.responseBody
}
}
if (rawMessage) {
const retryMatch = rawMessage.match(/retry in ([0-9.]+)s/i)
if (retryMatch) retrySeconds = Math.ceil(Number(retryMatch[1]))
}
}
const fallbackMessage = rawMessage || String(error)
const lowerMessage = fallbackMessage.toLowerCase()
if (statusCode === 429 || lowerMessage.includes('quota') || lowerMessage.includes('resource_exhausted')) {
return retrySeconds
? `API quota exhausted, please retry after ${retrySeconds}s or upgrade your quota.`
: `API quota exhausted, please retry later or upgrade your quota.`
}
if (
statusCode === 403 &&
(lowerMessage.includes('quota') || lowerMessage.includes('not enough') || lowerMessage.includes('insufficient'))
) {
return `API rejected the request due to insufficient quota or balance.`
}
if (statusCode === 503 || lowerMessage.includes('overloaded') || lowerMessage.includes('unavailable')) {
return `Model is overloaded, please retry later.`
}
if (fallbackMessage.length > 300) {
return `${fallbackMessage.slice(0, 300)}...`
}
return fallbackMessage
}
export async function runServerAgent(options: RunAgentOptions): Promise<void> {
const {
userMessage,