mirror of
https://github.com/hicccc77/WeFlow.git
synced 2026-03-21 23:24:06 +08:00
fix: 修复一些代码报错; 移除了好友复刻的功能
This commit is contained in:
@@ -1,306 +0,0 @@
|
||||
import { parentPort, workerData } from 'worker_threads'
|
||||
import { join } from 'path'
|
||||
import { mkdirSync } from 'fs'
|
||||
import * as lancedb from '@lancedb/lancedb'
|
||||
import { pipeline, env } from '@xenova/transformers'
|
||||
import { wcdbService } from './services/wcdbService'
|
||||
import { mapRowToCloneMessage, CloneMessage, CloneRole } from './services/cloneMessageUtils'
|
||||
|
||||
interface WorkerConfig {
|
||||
resourcesPath?: string
|
||||
userDataPath?: string
|
||||
logEnabled?: boolean
|
||||
embeddingModel?: string
|
||||
}
|
||||
|
||||
type WorkerRequest =
|
||||
| { id: string; type: 'index'; payload: IndexPayload }
|
||||
| { id: string; type: 'query'; payload: QueryPayload }
|
||||
|
||||
interface IndexPayload {
|
||||
sessionId: string
|
||||
dbPath: string
|
||||
decryptKey: string
|
||||
myWxid: string
|
||||
batchSize?: number
|
||||
chunkGapSeconds?: number
|
||||
maxChunkChars?: number
|
||||
maxChunkMessages?: number
|
||||
reset?: boolean
|
||||
}
|
||||
|
||||
interface QueryPayload {
|
||||
sessionId: string
|
||||
keyword: string
|
||||
topK?: number
|
||||
roleFilter?: CloneRole
|
||||
}
|
||||
|
||||
const config = workerData as WorkerConfig
|
||||
process.env.WEFLOW_WORKER = '1'
|
||||
if (config.resourcesPath) {
|
||||
process.env.WCDB_RESOURCES_PATH = config.resourcesPath
|
||||
}
|
||||
|
||||
wcdbService.setPaths(config.resourcesPath || '', config.userDataPath || '')
|
||||
wcdbService.setLogEnabled(config.logEnabled === true)
|
||||
|
||||
env.allowRemoteModels = true
|
||||
if (env.backends?.onnx) {
|
||||
env.backends.onnx.wasm.enabled = false
|
||||
}
|
||||
|
||||
const embeddingModel = config.embeddingModel || 'Xenova/bge-small-zh-v1.5'
|
||||
let embedder: any | null = null
|
||||
|
||||
async function ensureEmbedder() {
|
||||
if (embedder) return embedder
|
||||
if (config.userDataPath) {
|
||||
env.cacheDir = join(config.userDataPath, 'transformers')
|
||||
}
|
||||
embedder = await pipeline('feature-extraction', embeddingModel)
|
||||
return embedder
|
||||
}
|
||||
|
||||
function getMemoryDir(sessionId: string): string {
|
||||
const safeId = sessionId.replace(/[\\/:"*?<>|]+/g, '_')
|
||||
const base = config.userDataPath || process.cwd()
|
||||
const dir = join(base, 'clone_memory', safeId)
|
||||
mkdirSync(dir, { recursive: true })
|
||||
return dir
|
||||
}
|
||||
|
||||
async function getTable(sessionId: string, reset?: boolean) {
|
||||
const dir = getMemoryDir(sessionId)
|
||||
const db = await lancedb.connect(dir)
|
||||
const tables = await db.tableNames()
|
||||
if (reset && tables.includes('messages')) {
|
||||
await db.dropTable('messages')
|
||||
}
|
||||
const hasTable = tables.includes('messages') && !reset
|
||||
return { db, hasTable }
|
||||
}
|
||||
|
||||
function shouldSkipContent(text: string): boolean {
|
||||
if (!text) return true
|
||||
if (text === '[图片]' || text === '[语音]' || text === '[视频]' || text === '[表情]' || text === '[分享]') {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function chunkMessages(
|
||||
messages: CloneMessage[],
|
||||
gapSeconds: number,
|
||||
maxChars: number,
|
||||
maxMessages: number
|
||||
) {
|
||||
const chunks: Array<{
|
||||
role: CloneRole
|
||||
content: string
|
||||
tsStart: number
|
||||
tsEnd: number
|
||||
messageCount: number
|
||||
}> = []
|
||||
let current: typeof chunks[number] | null = null
|
||||
|
||||
for (const msg of messages) {
|
||||
if (shouldSkipContent(msg.content)) continue
|
||||
if (!current) {
|
||||
current = {
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
tsStart: msg.createTime,
|
||||
tsEnd: msg.createTime,
|
||||
messageCount: 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
const gap = msg.createTime - current.tsEnd
|
||||
const nextContent = `${current.content}\n${msg.content}`
|
||||
const roleChanged = msg.role !== current.role
|
||||
if (roleChanged || gap > gapSeconds || nextContent.length > maxChars || current.messageCount >= maxMessages) {
|
||||
chunks.push(current)
|
||||
current = {
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
tsStart: msg.createTime,
|
||||
tsEnd: msg.createTime,
|
||||
messageCount: 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
current.content = nextContent
|
||||
current.tsEnd = msg.createTime
|
||||
current.messageCount += 1
|
||||
}
|
||||
|
||||
if (current) {
|
||||
chunks.push(current)
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
async function embedTexts(texts: string[]) {
|
||||
const model = await ensureEmbedder()
|
||||
const output = await model(texts, { pooling: 'mean', normalize: true })
|
||||
if (Array.isArray(output)) return output
|
||||
if (output?.tolist) return output.tolist()
|
||||
return []
|
||||
}
|
||||
|
||||
async function gatherDebugInfo(table: any) {
|
||||
try {
|
||||
const rowCount = await table.countRows()
|
||||
const sample = await table.limit(3).toArray()
|
||||
return { rowCount, sample }
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
async function handleIndex(requestId: string, payload: IndexPayload) {
|
||||
const {
|
||||
sessionId,
|
||||
dbPath,
|
||||
decryptKey,
|
||||
myWxid,
|
||||
batchSize = 200,
|
||||
chunkGapSeconds = 600,
|
||||
maxChunkChars = 400,
|
||||
maxChunkMessages = 20,
|
||||
reset = false
|
||||
} = payload
|
||||
|
||||
const openOk = await wcdbService.open(dbPath, decryptKey, myWxid)
|
||||
if (!openOk) {
|
||||
throw new Error('WCDB open failed')
|
||||
}
|
||||
|
||||
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, batchSize, true, 0, 0)
|
||||
if (!cursorResult.success || !cursorResult.cursor) {
|
||||
throw new Error(cursorResult.error || 'cursor open failed')
|
||||
}
|
||||
|
||||
const { db, hasTable } = await getTable(sessionId, reset)
|
||||
let table = hasTable ? await db.openTable('messages') : null
|
||||
let cursor = cursorResult.cursor
|
||||
let hasMore = true
|
||||
let chunkId = 0
|
||||
let totalMessages = 0
|
||||
let totalChunks = 0
|
||||
|
||||
try {
|
||||
while (hasMore) {
|
||||
const batchResult = await wcdbService.fetchMessageBatch(cursor)
|
||||
if (!batchResult.success || !batchResult.rows) {
|
||||
throw new Error(batchResult.error || 'fetch batch failed')
|
||||
}
|
||||
|
||||
totalMessages += batchResult.rows.length
|
||||
const messages: CloneMessage[] = []
|
||||
for (const row of batchResult.rows) {
|
||||
const msg = mapRowToCloneMessage(row, myWxid)
|
||||
if (msg) messages.push(msg)
|
||||
}
|
||||
|
||||
const chunks = chunkMessages(messages, chunkGapSeconds, maxChunkChars, maxChunkMessages)
|
||||
if (chunks.length > 0) {
|
||||
const embeddings = await embedTexts(chunks.map((c) => c.content))
|
||||
if (embeddings.length !== chunks.length) {
|
||||
throw new Error('embedding size mismatch')
|
||||
}
|
||||
const rows = chunks.map((chunk, idx) => ({
|
||||
id: `${sessionId}-${chunkId + idx}`,
|
||||
sessionId,
|
||||
role: chunk.role,
|
||||
content: chunk.content,
|
||||
tsStart: chunk.tsStart,
|
||||
tsEnd: chunk.tsEnd,
|
||||
messageCount: chunk.messageCount,
|
||||
embedding: new Float32Array(embeddings[idx] || [])
|
||||
}))
|
||||
if (!table) {
|
||||
table = await db.createTable('messages', rows)
|
||||
} else {
|
||||
await table.add(rows)
|
||||
}
|
||||
chunkId += chunks.length
|
||||
totalChunks += chunks.length
|
||||
}
|
||||
|
||||
hasMore = batchResult.hasMore === true
|
||||
parentPort?.postMessage({
|
||||
type: 'event',
|
||||
event: 'clone:indexProgress',
|
||||
data: { requestId, totalMessages, totalChunks, hasMore }
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
await wcdbService.closeMessageCursor(cursor)
|
||||
wcdbService.close()
|
||||
}
|
||||
|
||||
const debug = await gatherDebugInfo(table)
|
||||
return { success: true, totalMessages, totalChunks, debug }
|
||||
}
|
||||
|
||||
async function handleQuery(payload: QueryPayload) {
|
||||
const { sessionId, keyword, topK = 5, roleFilter } = payload
|
||||
const { db, hasTable } = await getTable(sessionId, false)
|
||||
if (!hasTable) {
|
||||
return { success: false, error: 'memory table not found' }
|
||||
}
|
||||
const table = await db.openTable('messages')
|
||||
const embeddings = await embedTexts([keyword])
|
||||
if (!embeddings.length || !embeddings[0]) {
|
||||
return { success: false, error: 'embedding failed' }
|
||||
}
|
||||
const query = table.search(new Float32Array(embeddings[0] || [])).limit(topK)
|
||||
const filtered = roleFilter ? query.where(`role = '${roleFilter}'`) : query
|
||||
let rows = await filtered.toArray()
|
||||
let usedFallback = false
|
||||
|
||||
if (rows.length === 0) {
|
||||
try {
|
||||
usedFallback = true
|
||||
const lowerKeyword = keyword.trim().toLowerCase()
|
||||
const all = await table.toArray()
|
||||
rows = all.filter((row) => {
|
||||
const content = String(row.content || '').toLowerCase()
|
||||
return content.includes(lowerKeyword)
|
||||
}).slice(0, topK)
|
||||
} catch {
|
||||
// fallback remain empty
|
||||
}
|
||||
}
|
||||
|
||||
const debug = {
|
||||
rowsFound: rows.length,
|
||||
usedFallback,
|
||||
sample: rows.slice(0, 2)
|
||||
}
|
||||
|
||||
return { success: true, results: rows, debug }
|
||||
}
|
||||
|
||||
parentPort?.on('message', async (request: WorkerRequest) => {
|
||||
try {
|
||||
if (request.type === 'index') {
|
||||
const data = await handleIndex(request.id, request.payload)
|
||||
parentPort?.postMessage({ type: 'response', id: request.id, ok: true, data })
|
||||
return
|
||||
}
|
||||
if (request.type === 'query') {
|
||||
const data = await handleQuery(request.payload)
|
||||
parentPort?.postMessage({ type: 'response', id: request.id, ok: true, data })
|
||||
return
|
||||
}
|
||||
parentPort?.postMessage({ type: 'response', id: request.id, ok: false, error: 'unknown request' })
|
||||
} catch (err) {
|
||||
parentPort?.postMessage({ type: 'response', id: request.id, ok: false, error: String(err) })
|
||||
}
|
||||
})
|
||||
@@ -15,7 +15,7 @@ import { groupAnalyticsService } from './services/groupAnalyticsService'
|
||||
import { annualReportService } from './services/annualReportService'
|
||||
import { exportService, ExportOptions } from './services/exportService'
|
||||
import { KeyService } from './services/keyService'
|
||||
import { cloneService } from './services/cloneService'
|
||||
|
||||
|
||||
// 配置自动更新
|
||||
autoUpdater.autoDownload = false
|
||||
@@ -445,27 +445,7 @@ function registerIpcHandlers() {
|
||||
})
|
||||
|
||||
// 私聊克隆
|
||||
ipcMain.handle('clone:indexSession', async (_, sessionId: string, options?: any) => {
|
||||
return await cloneService.indexSession(sessionId, options, (payload) => {
|
||||
mainWindow?.webContents.send('clone:indexProgress', payload)
|
||||
})
|
||||
})
|
||||
|
||||
ipcMain.handle('clone:query', async (_, payload: { sessionId: string; keyword: string; options?: any }) => {
|
||||
return await cloneService.queryMemory(payload.sessionId, payload.keyword, payload.options || {})
|
||||
})
|
||||
|
||||
ipcMain.handle('clone:getToneGuide', async (_, sessionId: string) => {
|
||||
return await cloneService.getToneGuide(sessionId)
|
||||
})
|
||||
|
||||
ipcMain.handle('clone:generateToneGuide', async (_, sessionId: string, sampleSize?: number) => {
|
||||
return await cloneService.generateToneGuide(sessionId, sampleSize || 500)
|
||||
})
|
||||
|
||||
ipcMain.handle('clone:chat', async (_, payload: { sessionId: string; message: string; topK?: number }) => {
|
||||
return await cloneService.chat(payload)
|
||||
})
|
||||
|
||||
ipcMain.handle('image:decrypt', async (_, payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; force?: boolean }) => {
|
||||
return imageDecryptService.decryptImage(payload)
|
||||
|
||||
@@ -108,19 +108,7 @@ contextBridge.exposeInMainWorld('electronAPI', {
|
||||
getVoiceData: (sessionId: string, msgId: string) => ipcRenderer.invoke('chat:getVoiceData', sessionId, msgId)
|
||||
},
|
||||
|
||||
// 私聊克隆
|
||||
clone: {
|
||||
indexSession: (sessionId: string, options?: any) => ipcRenderer.invoke('clone:indexSession', sessionId, options),
|
||||
query: (payload: { sessionId: string; keyword: string; options?: any }) => ipcRenderer.invoke('clone:query', payload),
|
||||
getToneGuide: (sessionId: string) => ipcRenderer.invoke('clone:getToneGuide', sessionId),
|
||||
generateToneGuide: (sessionId: string, sampleSize?: number) =>
|
||||
ipcRenderer.invoke('clone:generateToneGuide', sessionId, sampleSize),
|
||||
chat: (payload: { sessionId: string; message: string; topK?: number }) => ipcRenderer.invoke('clone:chat', payload),
|
||||
onIndexProgress: (callback: (payload: { requestId: string; totalMessages: number; totalChunks: number; hasMore: boolean }) => void) => {
|
||||
ipcRenderer.on('clone:indexProgress', (_, payload) => callback(payload))
|
||||
return () => ipcRenderer.removeAllListeners('clone:indexProgress')
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
// 图片解密
|
||||
image: {
|
||||
|
||||
@@ -58,6 +58,7 @@ export interface Message {
|
||||
aesKey?: string
|
||||
encrypVer?: number
|
||||
cdnThumbUrl?: string
|
||||
voiceDurationSeconds?: number
|
||||
}
|
||||
|
||||
export interface Contact {
|
||||
@@ -2495,12 +2496,12 @@ class ChatService {
|
||||
}
|
||||
|
||||
const aesData = payload.subarray(0, alignedAesSize)
|
||||
let unpadded = Buffer.alloc(0)
|
||||
let unpadded: Buffer = Buffer.alloc(0)
|
||||
if (aesData.length > 0) {
|
||||
const decipher = crypto.createDecipheriv('aes-128-ecb', aesKey, Buffer.alloc(0))
|
||||
decipher.setAutoPadding(false)
|
||||
const decrypted = Buffer.concat([decipher.update(aesData), decipher.final()])
|
||||
unpadded = this.strictRemovePadding(decrypted)
|
||||
unpadded = this.strictRemovePadding(decrypted) as Buffer
|
||||
}
|
||||
|
||||
const remaining = payload.subarray(alignedAesSize)
|
||||
@@ -2508,21 +2509,21 @@ class ChatService {
|
||||
throw new Error('文件格式异常:XOR 数据长度不合法')
|
||||
}
|
||||
|
||||
let rawData = Buffer.alloc(0)
|
||||
let xoredData = Buffer.alloc(0)
|
||||
let rawData: Buffer = Buffer.alloc(0)
|
||||
let xoredData: Buffer = Buffer.alloc(0)
|
||||
if (xorSize > 0) {
|
||||
const rawLength = remaining.length - xorSize
|
||||
if (rawLength < 0) {
|
||||
throw new Error('文件格式异常:原始数据长度小于XOR长度')
|
||||
}
|
||||
rawData = remaining.subarray(0, rawLength)
|
||||
rawData = remaining.subarray(0, rawLength) as Buffer
|
||||
const xorData = remaining.subarray(rawLength)
|
||||
xoredData = Buffer.alloc(xorData.length)
|
||||
for (let i = 0; i < xorData.length; i++) {
|
||||
xoredData[i] = xorData[i] ^ xorKey
|
||||
}
|
||||
} else {
|
||||
rawData = remaining
|
||||
rawData = remaining as Buffer
|
||||
xoredData = Buffer.alloc(0)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
export type CloneRole = 'target' | 'me'
|
||||
|
||||
export interface CloneMessage {
|
||||
role: CloneRole
|
||||
content: string
|
||||
createTime: number
|
||||
}
|
||||
|
||||
const CONTENT_FIELDS = [
|
||||
'message_content',
|
||||
'messageContent',
|
||||
'content',
|
||||
'msg_content',
|
||||
'msgContent',
|
||||
'WCDB_CT_message_content',
|
||||
'WCDB_CT_messageContent'
|
||||
]
|
||||
const COMPRESS_FIELDS = [
|
||||
'compress_content',
|
||||
'compressContent',
|
||||
'compressed_content',
|
||||
'WCDB_CT_compress_content',
|
||||
'WCDB_CT_compressContent'
|
||||
]
|
||||
const LOCAL_TYPE_FIELDS = ['local_type', 'localType', 'type', 'msg_type', 'msgType', 'WCDB_CT_local_type']
|
||||
const IS_SEND_FIELDS = ['computed_is_send', 'computedIsSend', 'is_send', 'isSend', 'WCDB_CT_is_send']
|
||||
const SENDER_FIELDS = ['sender_username', 'senderUsername', 'sender', 'WCDB_CT_sender_username']
|
||||
const CREATE_TIME_FIELDS = [
|
||||
'create_time',
|
||||
'createTime',
|
||||
'createtime',
|
||||
'msg_create_time',
|
||||
'msgCreateTime',
|
||||
'msg_time',
|
||||
'msgTime',
|
||||
'time',
|
||||
'WCDB_CT_create_time'
|
||||
]
|
||||
|
||||
const TYPE_LABELS: Record<number, string> = {
|
||||
1: '',
|
||||
3: '[图片]',
|
||||
34: '[语音]',
|
||||
43: '[视频]',
|
||||
47: '[表情]',
|
||||
49: '[分享]',
|
||||
62: '[小视频]',
|
||||
10000: '[系统消息]'
|
||||
}
|
||||
|
||||
export function mapRowToCloneMessage(
|
||||
row: Record<string, any>,
|
||||
myWxid?: string | null
|
||||
): CloneMessage | null {
|
||||
const content = decodeMessageContent(getRowField(row, CONTENT_FIELDS), getRowField(row, COMPRESS_FIELDS))
|
||||
const localType = getRowInt(row, LOCAL_TYPE_FIELDS, 1)
|
||||
const createTime = getRowInt(row, CREATE_TIME_FIELDS, 0)
|
||||
const senderUsername = getRowField(row, SENDER_FIELDS)
|
||||
const isSendRaw = getRowField(row, IS_SEND_FIELDS)
|
||||
let isSend = isSendRaw === null ? null : parseInt(String(isSendRaw), 10)
|
||||
|
||||
if (senderUsername && myWxid) {
|
||||
const senderLower = String(senderUsername).toLowerCase()
|
||||
const myLower = myWxid.toLowerCase()
|
||||
if (isSend === null) {
|
||||
isSend = senderLower === myLower ? 1 : 0
|
||||
}
|
||||
}
|
||||
|
||||
const parsedContent = parseMessageContent(content, localType)
|
||||
if (!parsedContent) return null
|
||||
|
||||
const role: CloneRole = isSend === 1 ? 'me' : 'target'
|
||||
return { role, content: parsedContent, createTime }
|
||||
}
|
||||
|
||||
export function parseMessageContent(content: string, localType: number): string {
|
||||
if (!content) {
|
||||
return TYPE_LABELS[localType] || ''
|
||||
}
|
||||
if (Buffer.isBuffer(content as unknown)) {
|
||||
content = (content as unknown as Buffer).toString('utf-8')
|
||||
}
|
||||
if (localType === 1) {
|
||||
return stripSenderPrefix(content)
|
||||
}
|
||||
return TYPE_LABELS[localType] || content
|
||||
}
|
||||
|
||||
function stripSenderPrefix(content: string): string {
|
||||
const trimmed = content.trim()
|
||||
const separatorIdx = trimmed.indexOf(':\n')
|
||||
if (separatorIdx > 0 && separatorIdx < 64) {
|
||||
return trimmed.slice(separatorIdx + 2).trim()
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
function decodeMessageContent(raw: unknown, compressed: unknown): string {
|
||||
const source = raw ?? compressed
|
||||
if (source == null) return ''
|
||||
if (typeof source === 'string') return source
|
||||
if (Buffer.isBuffer(source)) return source.toString('utf-8')
|
||||
try {
|
||||
return String(source)
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
function getRowField(row: Record<string, any>, keys: string[]): any {
|
||||
for (const key of keys) {
|
||||
if (row[key] !== undefined && row[key] !== null) return row[key]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function getRowInt(row: Record<string, any>, keys: string[], fallback: number): number {
|
||||
const raw = getRowField(row, keys)
|
||||
if (raw === null || raw === undefined) return fallback
|
||||
const parsed = parseInt(String(raw), 10)
|
||||
return Number.isFinite(parsed) ? parsed : fallback
|
||||
}
|
||||
@@ -1,356 +0,0 @@
|
||||
import { Worker } from 'worker_threads'
|
||||
import { join } from 'path'
|
||||
import { app } from 'electron'
|
||||
import { existsSync, mkdirSync } from 'fs'
|
||||
import { readFile, writeFile } from 'fs/promises'
|
||||
import { ConfigService } from './config'
|
||||
import { chatService } from './chatService'
|
||||
import { wcdbService } from './wcdbService'
|
||||
import { mapRowToCloneMessage, CloneMessage, CloneRole } from './cloneMessageUtils'
|
||||
|
||||
interface IndexOptions {
|
||||
reset?: boolean
|
||||
batchSize?: number
|
||||
chunkGapSeconds?: number
|
||||
maxChunkChars?: number
|
||||
maxChunkMessages?: number
|
||||
}
|
||||
|
||||
interface QueryOptions {
|
||||
topK?: number
|
||||
roleFilter?: CloneRole
|
||||
}
|
||||
|
||||
interface ToneGuide {
|
||||
sessionId: string
|
||||
createdAt: string
|
||||
model: string
|
||||
sampleSize: number
|
||||
summary: string
|
||||
details?: Record<string, any>
|
||||
}
|
||||
|
||||
interface ChatRequest {
|
||||
sessionId: string
|
||||
message: string
|
||||
topK?: number
|
||||
}
|
||||
|
||||
type WorkerRequest =
|
||||
| { id: string; type: 'index'; payload: any }
|
||||
| { id: string; type: 'query'; payload: any }
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (value: any) => void
|
||||
reject: (err: any) => void
|
||||
onProgress?: (payload: any) => void
|
||||
}
|
||||
|
||||
class CloneService {
|
||||
private configService = new ConfigService()
|
||||
private worker: Worker | null = null
|
||||
private pending: Map<string, PendingRequest> = new Map()
|
||||
private requestId = 0
|
||||
|
||||
private resolveResourcesPath(): string {
|
||||
const candidate = app.isPackaged
|
||||
? join(process.resourcesPath, 'resources')
|
||||
: join(app.getAppPath(), 'resources')
|
||||
if (existsSync(candidate)) return candidate
|
||||
const fallback = join(process.cwd(), 'resources')
|
||||
if (existsSync(fallback)) return fallback
|
||||
return candidate
|
||||
}
|
||||
|
||||
private getBaseStoragePath(): string {
|
||||
const cachePath = this.configService.get('cachePath')
|
||||
if (cachePath && cachePath.length > 0) {
|
||||
return cachePath
|
||||
}
|
||||
const documents = app.getPath('documents')
|
||||
const defaultDir = join(documents, 'WeFlow')
|
||||
if (!existsSync(defaultDir)) {
|
||||
mkdirSync(defaultDir, { recursive: true })
|
||||
}
|
||||
return defaultDir
|
||||
}
|
||||
|
||||
private getSessionDir(sessionId: string): string {
|
||||
const safeId = sessionId.replace(/[\\/:"*?<>|]+/g, '_')
|
||||
const base = this.getBaseStoragePath()
|
||||
const dir = join(base, 'clone_memory', safeId)
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
|
||||
return dir
|
||||
}
|
||||
|
||||
private getToneGuidePath(sessionId: string): string {
|
||||
return join(this.getSessionDir(sessionId), 'tone_guide.json')
|
||||
}
|
||||
|
||||
private ensureWorker(): Worker {
|
||||
if (this.worker) return this.worker
|
||||
const workerPath = join(__dirname, 'cloneEmbeddingWorker.js')
|
||||
const worker = new Worker(workerPath, {
|
||||
workerData: {
|
||||
resourcesPath: this.resolveResourcesPath(),
|
||||
userDataPath: this.getBaseStoragePath(),
|
||||
logEnabled: this.configService.get('logEnabled'),
|
||||
embeddingModel: 'Xenova/bge-small-zh-v1.5'
|
||||
}
|
||||
})
|
||||
|
||||
worker.on('message', (msg: any) => {
|
||||
if (msg?.type === 'event' && msg.event === 'clone:indexProgress') {
|
||||
const entry = this.pending.get(msg.data?.requestId)
|
||||
if (entry?.onProgress) entry.onProgress(msg.data)
|
||||
return
|
||||
}
|
||||
if (msg?.type === 'response' && msg.id) {
|
||||
const entry = this.pending.get(msg.id)
|
||||
if (!entry) return
|
||||
this.pending.delete(msg.id)
|
||||
if (msg.ok) {
|
||||
entry.resolve(msg.data)
|
||||
} else {
|
||||
entry.reject(new Error(msg.error || 'worker error'))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
worker.on('exit', () => {
|
||||
this.worker = null
|
||||
this.pending.clear()
|
||||
})
|
||||
|
||||
this.worker = worker
|
||||
return worker
|
||||
}
|
||||
|
||||
private callWorker(type: WorkerRequest['type'], payload: any, onProgress?: (payload: any) => void) {
|
||||
const worker = this.ensureWorker()
|
||||
const id = String(++this.requestId)
|
||||
const request: WorkerRequest = { id, type, payload }
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.set(id, { resolve, reject, onProgress })
|
||||
worker.postMessage(request)
|
||||
})
|
||||
}
|
||||
|
||||
private validatePrivateSession(sessionId: string): { ok: boolean; error?: string } {
|
||||
if (!sessionId) return { ok: false, error: 'sessionId 不能为空' }
|
||||
if (sessionId.includes('@chatroom')) {
|
||||
return { ok: false, error: '当前仅支持私聊' }
|
||||
}
|
||||
return { ok: true }
|
||||
}
|
||||
|
||||
async indexSession(sessionId: string, options: IndexOptions = {}, onProgress?: (payload: any) => void) {
|
||||
const check = this.validatePrivateSession(sessionId)
|
||||
if (!check.ok) return { success: false, error: check.error }
|
||||
|
||||
const dbPath = this.configService.get('dbPath')
|
||||
const decryptKey = this.configService.get('decryptKey')
|
||||
const myWxid = this.configService.get('myWxid')
|
||||
if (!dbPath || !decryptKey || !myWxid) {
|
||||
return { success: false, error: '数据库配置不完整' }
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.callWorker(
|
||||
'index',
|
||||
{ sessionId, dbPath, decryptKey, myWxid, ...options },
|
||||
onProgress
|
||||
)
|
||||
return result
|
||||
} catch (err) {
|
||||
return { success: false, error: String(err) }
|
||||
}
|
||||
}
|
||||
|
||||
async queryMemory(sessionId: string, keyword: string, options: QueryOptions = {}) {
|
||||
const check = this.validatePrivateSession(sessionId)
|
||||
if (!check.ok) return { success: false, error: check.error }
|
||||
if (!keyword) return { success: false, error: 'keyword 不能为空' }
|
||||
try {
|
||||
const result = await this.callWorker('query', { sessionId, keyword, ...options })
|
||||
return result
|
||||
} catch (err) {
|
||||
return { success: false, error: String(err) }
|
||||
}
|
||||
}
|
||||
|
||||
async getToneGuide(sessionId: string) {
|
||||
const check = this.validatePrivateSession(sessionId)
|
||||
if (!check.ok) return { success: false, error: check.error }
|
||||
const filePath = this.getToneGuidePath(sessionId)
|
||||
if (!existsSync(filePath)) {
|
||||
return { success: false, error: '未找到性格说明书' }
|
||||
}
|
||||
const raw = await readFile(filePath, 'utf8')
|
||||
return { success: true, data: JSON.parse(raw) as ToneGuide }
|
||||
}
|
||||
|
||||
async generateToneGuide(sessionId: string, sampleSize = 500) {
|
||||
const check = this.validatePrivateSession(sessionId)
|
||||
if (!check.ok) return { success: false, error: check.error }
|
||||
const connectResult = await chatService.connect()
|
||||
if (!connectResult.success) return { success: false, error: connectResult.error || '数据库未连接' }
|
||||
|
||||
const myWxid = this.configService.get('myWxid')
|
||||
if (!myWxid) return { success: false, error: '缺少 myWxid 配置' }
|
||||
|
||||
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, 300, true, 0, 0)
|
||||
if (!cursorResult.success || !cursorResult.cursor) {
|
||||
return { success: false, error: cursorResult.error || '创建游标失败' }
|
||||
}
|
||||
|
||||
const samples: CloneMessage[] = []
|
||||
let seen = 0
|
||||
let hasMore = true
|
||||
let cursor = cursorResult.cursor
|
||||
|
||||
while (hasMore) {
|
||||
const batchResult = await wcdbService.fetchMessageBatch(cursor)
|
||||
if (!batchResult.success || !batchResult.rows) {
|
||||
await wcdbService.closeMessageCursor(cursor)
|
||||
return { success: false, error: batchResult.error || '读取消息失败' }
|
||||
}
|
||||
|
||||
for (const row of batchResult.rows) {
|
||||
const msg = mapRowToCloneMessage(row, myWxid)
|
||||
if (!msg || msg.role !== 'target') continue
|
||||
seen += 1
|
||||
if (samples.length < sampleSize) {
|
||||
samples.push(msg)
|
||||
} else {
|
||||
const idx = Math.floor(Math.random() * seen)
|
||||
if (idx < sampleSize) samples[idx] = msg
|
||||
}
|
||||
}
|
||||
|
||||
hasMore = batchResult.hasMore === true
|
||||
}
|
||||
|
||||
await wcdbService.closeMessageCursor(cursor)
|
||||
|
||||
if (samples.length === 0) {
|
||||
return { success: false, error: '样本为空,无法生成说明书' }
|
||||
}
|
||||
|
||||
const toneResult = await this.runToneGuideLlm(sessionId, samples)
|
||||
if (!toneResult.success) return toneResult
|
||||
|
||||
const filePath = this.getToneGuidePath(sessionId)
|
||||
await writeFile(filePath, JSON.stringify(toneResult.data, null, 2), 'utf8')
|
||||
return toneResult
|
||||
}
|
||||
|
||||
async chat(request: ChatRequest) {
|
||||
const { sessionId, message, topK = 5 } = request
|
||||
const check = this.validatePrivateSession(sessionId)
|
||||
if (!check.ok) return { success: false, error: check.error }
|
||||
if (!message) return { success: false, error: '消息不能为空' }
|
||||
|
||||
const toneGuide = await this.getToneGuide(sessionId)
|
||||
const toneText = toneGuide.success ? JSON.stringify(toneGuide.data) : '未找到说明书'
|
||||
|
||||
const toolPrompt = [
|
||||
'你是一个微信好友的私聊分身,只能基于已知事实回答。',
|
||||
'如果需要查询过去的对话事实,请用工具。',
|
||||
'请严格输出 JSON,不要输出多余文本。',
|
||||
'当需要工具时输出:{"tool":"query_chat_history","parameters":{"keyword":"关键词"}}',
|
||||
'当无需工具时输出:{"tool":"none","response":"直接回复"}',
|
||||
`性格说明书: ${toneText}`,
|
||||
`用户: ${message}`
|
||||
].join('\n')
|
||||
|
||||
const decision = await this.runLlm(toolPrompt)
|
||||
const parsed = parseToolJson(decision)
|
||||
if (!parsed || parsed.tool === 'none') {
|
||||
return { success: true, response: parsed?.response || decision }
|
||||
}
|
||||
|
||||
if (parsed.tool === 'query_chat_history') {
|
||||
const keyword = parsed.parameters?.keyword
|
||||
if (!keyword) return { success: true, response: decision }
|
||||
const memory = await this.queryMemory(sessionId, keyword, { topK, roleFilter: 'target' })
|
||||
const finalPrompt = [
|
||||
'你是一个微信好友的私聊分身,请根据工具返回的历史记录回答。',
|
||||
`性格说明书: ${toneText}`,
|
||||
`用户: ${message}`,
|
||||
`工具结果: ${JSON.stringify(memory)}`,
|
||||
'请直接回复用户,不要提及工具调用。'
|
||||
].join('\n')
|
||||
const finalAnswer = await this.runLlm(finalPrompt)
|
||||
return { success: true, response: finalAnswer }
|
||||
}
|
||||
|
||||
if (parsed.tool === 'get_tone_guide') {
|
||||
const finalPrompt = [
|
||||
'你是一个微信好友的私聊分身。',
|
||||
`性格说明书: ${toneText}`,
|
||||
`用户: ${message}`,
|
||||
'请直接回复用户。'
|
||||
].join('\n')
|
||||
const finalAnswer = await this.runLlm(finalPrompt)
|
||||
return { success: true, response: finalAnswer }
|
||||
}
|
||||
|
||||
return { success: true, response: decision }
|
||||
}
|
||||
|
||||
private async runToneGuideLlm(sessionId: string, samples: CloneMessage[]) {
|
||||
const prompt = [
|
||||
'你是对话风格分析助手,请根据聊天样本总结性格说明书。',
|
||||
'输出 JSON:{"summary":"一句话概括","details":{"口癖":[],"情绪价值":"","回复速度":"","表情偏好":"","风格要点":[]}}',
|
||||
'以下是聊天样本(仅该好友的发言):',
|
||||
samples.map((msg) => msg.content).join('\n')
|
||||
].join('\n')
|
||||
|
||||
const response = await this.runLlm(prompt)
|
||||
const parsed = parseToolJson(response)
|
||||
const toneGuide: ToneGuide = {
|
||||
sessionId,
|
||||
createdAt: new Date().toISOString(),
|
||||
model: this.configService.get('llmModelPath') || 'node-llama-cpp',
|
||||
sampleSize: samples.length,
|
||||
summary: parsed?.summary || response,
|
||||
details: parsed?.details || parsed?.data
|
||||
}
|
||||
return { success: true, data: toneGuide }
|
||||
}
|
||||
|
||||
private async runLlm(prompt: string): Promise<string> {
|
||||
const modelPath = this.configService.get('llmModelPath')
|
||||
if (!modelPath) {
|
||||
return 'LLM 未配置,请设置 llmModelPath'
|
||||
}
|
||||
|
||||
const llama = await import('node-llama-cpp').catch(() => null)
|
||||
if (!llama) {
|
||||
return 'node-llama-cpp 未安装'
|
||||
}
|
||||
|
||||
const { LlamaModel, LlamaContext, LlamaChatSession } = llama as any
|
||||
const model = new LlamaModel({ modelPath })
|
||||
const context = new LlamaContext({ model })
|
||||
const session = new LlamaChatSession({ context })
|
||||
const result = await session.prompt(prompt)
|
||||
return typeof result === 'string' ? result : String(result)
|
||||
}
|
||||
}
|
||||
|
||||
function parseToolJson(raw: string): any | null {
|
||||
if (!raw) return null
|
||||
const trimmed = raw.trim()
|
||||
const start = trimmed.indexOf('{')
|
||||
const end = trimmed.lastIndexOf('}')
|
||||
if (start === -1 || end === -1 || end <= start) return null
|
||||
try {
|
||||
return JSON.parse(trimmed.slice(start, end + 1))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export const cloneService = new CloneService()
|
||||
Reference in New Issue
Block a user