mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
refactor(server): delegate import data writing to shared writeParseResultToDb
Replace ~120 lines of inline SQL in server stream-import.ts and ~60 lines in importer.ts fullImport with calls to shared writeParseResultToDb from @openchatlab/node-runtime. Server imports now gain name history tracking that was previously Electron-only.
This commit is contained in:
1 parent
1796731535
commit
172d227caa
2 files changed
+12
-179
No files matched your search
@@ -10,7 +10,7 @@ import * as crypto from 'crypto'
|
||||
import type { DatabaseAdapter } from '@openchatlab/core'
|
||||
import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, generateMessageKey, buildMemberIdMap } from '@openchatlab/core'
|
||||
import type { DatabaseManager } from '@openchatlab/node-runtime'
|
||||
import { openBetterSqliteDatabase } from '@openchatlab/node-runtime'
|
||||
import { openBetterSqliteDatabase, writeParseResultToDb } from '@openchatlab/node-runtime'
|
||||
import type { ParsedData, ImportMessage } from './chatlab-reader'
|
||||
|
||||
export interface ImportResult {
|
||||
@@ -37,6 +37,7 @@ function generateSessionId(): string {
|
||||
|
||||
/**
|
||||
* 新建导入:创建新的会话数据库并导入全部数据
|
||||
* Delegates to shared writeParseResultToDb for data writing + name history.
|
||||
*/
|
||||
function fullImport(
|
||||
db: DatabaseAdapter,
|
||||
@@ -45,92 +46,12 @@ function fullImport(
|
||||
): { messageCount: number; memberCount: number; duplicateCount: number } {
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
|
||||
onProgress?.('写入元信息...')
|
||||
db.prepare(
|
||||
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id, schema_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 4)`
|
||||
).run(
|
||||
data.meta.name,
|
||||
data.meta.platform,
|
||||
data.meta.type,
|
||||
Math.floor(Date.now() / 1000),
|
||||
data.meta.groupId || null,
|
||||
data.meta.groupAvatar || null,
|
||||
data.meta.ownerId || null
|
||||
)
|
||||
|
||||
onProgress?.(`写入 ${data.members.length} 个成员...`)
|
||||
const insertMember = db.prepare(
|
||||
`INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
)
|
||||
for (const m of data.members) {
|
||||
insertMember.run(
|
||||
m.platformId,
|
||||
m.accountName || m.platformId,
|
||||
m.groupNickname || null,
|
||||
m.avatar || null,
|
||||
m.roles ? JSON.stringify(m.roles) : '[]'
|
||||
)
|
||||
}
|
||||
|
||||
const memberIdMap = buildMemberIdMap(db)
|
||||
|
||||
onProgress?.(`写入 ${data.messages.length} 条消息...`)
|
||||
const insertMsg = db.prepare(
|
||||
`INSERT INTO message (sender_id, sender_account_name, sender_group_nickname, ts, type, content, reply_to_message_id, platform_message_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
|
||||
let written = 0
|
||||
let skipped = 0
|
||||
const BATCH = 5000
|
||||
|
||||
for (let i = 0; i < data.messages.length; i += BATCH) {
|
||||
const batch = data.messages.slice(i, i + BATCH)
|
||||
db.transaction(() => {
|
||||
for (const msg of batch) {
|
||||
let senderId = memberIdMap.get(msg.senderPlatformId)
|
||||
if (!senderId) {
|
||||
insertMember.run(
|
||||
msg.senderPlatformId,
|
||||
msg.senderAccountName || msg.senderPlatformId,
|
||||
msg.senderGroupNickname || null,
|
||||
null,
|
||||
'[]'
|
||||
)
|
||||
senderId = (
|
||||
db.prepare('SELECT id FROM member WHERE platform_id = ?').get(msg.senderPlatformId) as { id: number }
|
||||
)?.id
|
||||
if (senderId) memberIdMap.set(msg.senderPlatformId, senderId)
|
||||
}
|
||||
if (!senderId) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
insertMsg.run(
|
||||
senderId,
|
||||
msg.senderAccountName || null,
|
||||
msg.senderGroupNickname || null,
|
||||
msg.timestamp,
|
||||
msg.type,
|
||||
msg.content,
|
||||
msg.replyToMessageId || null,
|
||||
msg.platformMessageId || null
|
||||
)
|
||||
written++
|
||||
}
|
||||
})
|
||||
|
||||
if (onProgress && (i + BATCH) % 50000 < BATCH) {
|
||||
onProgress(`已写入 ${Math.min(i + BATCH, data.messages.length)} / ${data.messages.length} 条消息`)
|
||||
}
|
||||
}
|
||||
onProgress?.(`写入 ${data.members.length} 个成员, ${data.messages.length} 条消息...`)
|
||||
const stats = writeParseResultToDb(db, data.meta, data.members, data.messages)
|
||||
|
||||
buildFts(db, onProgress)
|
||||
|
||||
return { messageCount: written, memberCount: data.members.length, duplicateCount: skipped }
|
||||
return { messageCount: stats.messageCount, memberCount: stats.memberCount, duplicateCount: stats.skippedCount }
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
*/
|
||||
|
||||
import type { DatabaseManager } from '@openchatlab/node-runtime'
|
||||
import { openBetterSqliteDatabase } from '@openchatlab/node-runtime'
|
||||
import { CHAT_DB_SCHEMA, FTS_TABLE_SCHEMA, buildMemberIdMap } from '@openchatlab/core'
|
||||
import { openBetterSqliteDatabase, writeParseResultToDb, buildFtsIndex } from '@openchatlab/node-runtime'
|
||||
import { CHAT_DB_SCHEMA } from '@openchatlab/core'
|
||||
import {
|
||||
streamParseFile,
|
||||
detectFormat as parserDetectFormat,
|
||||
@@ -125,99 +125,11 @@ export async function streamImport(
|
||||
db = openBetterSqliteDatabase(dbPath, { nativeBinding })
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
|
||||
db!
|
||||
.prepare(
|
||||
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id, schema_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 4)`
|
||||
)
|
||||
.run(
|
||||
parsedMeta.name,
|
||||
parsedMeta.platform,
|
||||
parsedMeta.type,
|
||||
Math.floor(Date.now() / 1000),
|
||||
parsedMeta.groupId || null,
|
||||
parsedMeta.groupAvatar || null,
|
||||
parsedMeta.ownerId || null
|
||||
)
|
||||
|
||||
const insertMember = db.prepare(
|
||||
`INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
)
|
||||
for (const m of members) {
|
||||
insertMember.run(
|
||||
m.platformId,
|
||||
m.accountName || m.platformId,
|
||||
m.groupNickname || null,
|
||||
m.avatar || null,
|
||||
m.roles ? JSON.stringify(m.roles) : '[]'
|
||||
)
|
||||
}
|
||||
|
||||
const memberIdMap = buildMemberIdMap(db)
|
||||
|
||||
const insertMsg = db.prepare(
|
||||
`INSERT INTO message (sender_id, sender_account_name, sender_group_nickname, ts, type, content, reply_to_message_id, platform_message_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
|
||||
let written = 0
|
||||
const BATCH = 5000
|
||||
|
||||
for (let i = 0; i < messages.length; i += BATCH) {
|
||||
const batch = messages.slice(i, i + BATCH)
|
||||
db.transaction(() => {
|
||||
for (const msg of batch) {
|
||||
let senderId = memberIdMap.get(msg.senderPlatformId)
|
||||
if (!senderId) {
|
||||
insertMember.run(
|
||||
msg.senderPlatformId,
|
||||
msg.senderAccountName || msg.senderPlatformId,
|
||||
msg.senderGroupNickname || null,
|
||||
null,
|
||||
'[]'
|
||||
)
|
||||
senderId = (
|
||||
db!.prepare('SELECT id FROM member WHERE platform_id = ?').get(msg.senderPlatformId) as { id: number }
|
||||
)?.id
|
||||
if (senderId) memberIdMap.set(msg.senderPlatformId, senderId)
|
||||
}
|
||||
if (!senderId) continue
|
||||
|
||||
insertMsg.run(
|
||||
senderId,
|
||||
msg.senderAccountName || null,
|
||||
msg.senderGroupNickname || null,
|
||||
msg.timestamp,
|
||||
msg.type,
|
||||
msg.content,
|
||||
msg.replyToMessageId || null,
|
||||
msg.platformMessageId || null
|
||||
)
|
||||
written++
|
||||
}
|
||||
})
|
||||
|
||||
const pct = 75 + Math.round((Math.min(i + BATCH, messages.length) / messages.length) * 15)
|
||||
onProgress?.({ stage: 'saving', progress: pct, message: '', messagesProcessed: written })
|
||||
}
|
||||
onProgress?.({ stage: 'saving', progress: 80, message: '' })
|
||||
const stats = writeParseResultToDb(db, parsedMeta, members, messages)
|
||||
|
||||
onProgress?.({ stage: 'indexing', progress: 92, message: '' })
|
||||
|
||||
db.exec(FTS_TABLE_SCHEMA)
|
||||
const textMessages = db
|
||||
.prepare("SELECT id, content FROM message WHERE type = 0 AND content IS NOT NULL AND content != ''")
|
||||
.all() as Array<{ id: number; content: string }>
|
||||
|
||||
const insertFts = db.prepare('INSERT INTO message_fts(rowid, content) VALUES (?, ?)')
|
||||
for (let i = 0; i < textMessages.length; i += BATCH) {
|
||||
const batch = textMessages.slice(i, i + BATCH)
|
||||
db.transaction(() => {
|
||||
for (const row of batch) {
|
||||
insertFts.run(row.id, row.content)
|
||||
}
|
||||
})
|
||||
}
|
||||
buildFtsIndex(db)
|
||||
|
||||
db.close()
|
||||
|
||||
@@ -226,8 +138,8 @@ export async function streamImport(
|
||||
return {
|
||||
success: true,
|
||||
sessionId,
|
||||
messageCount: written,
|
||||
memberCount: members.length,
|
||||
messageCount: stats.messageCount,
|
||||
memberCount: stats.memberCount,
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
|
||||
Reference in new issue
Block a user