feat: add web import support with adapter pattern and ImportArea refactor

- Extend QueryAdapter with importFile, detectFormat, scanMultiChatFile,
  and getSupportedFormats methods
- Implement FetchAdapter import methods with FormData upload and SSE
  progress stream parsing
- Implement ElectronAdapter import methods wrapping existing IPC calls
- Refactor ImportArea.vue to handle both Electron (native dialog + paths)
  and Web (FileDropZone + File objects) import flows
- Add 'indexing' stage to ImportProgress type
- Revert CLI-only import UI, restore drag-and-drop for web mode
This commit is contained in:
digua committed 2026-05-13 20:28:46 +08:00
1 parent 008094710c
commit 3363640e75
7 files changed
+392 -64

No files matched your search

+32 -14
View File
@@ -24,13 +24,7 @@ import {
getDatabaseSchema,
executeReadonlySql,
} from '@openchatlab/core'
import {
streamImport,
detectFormat,
detectAllFormats,
getSupportedFormats,
scanMultiChatFile,
} from '../../import'
import { streamImport, detectFormat, detectAllFormats, getSupportedFormats, scanMultiChatFile } from '../../import'
function resolveNativeBinding(): string | undefined {
if (process.versions.electron) return undefined
@@ -418,8 +412,16 @@ export async function registerWebRoutes(server: FastifyInstance, dbManager: Data
const allFormats = detectAllFormats(tmpPath)
return { format, allFormats }
} finally {
try { fs.unlinkSync(tmpPath) } catch { /* ignore */ }
try { fs.rmdirSync(tmpDir) } catch { /* ignore */ }
try {
fs.unlinkSync(tmpPath)
} catch {
/* ignore */
}
try {
fs.rmdirSync(tmpDir)
} catch {
/* ignore */
}
}
})
@@ -440,8 +442,16 @@ export async function registerWebRoutes(server: FastifyInstance, dbManager: Data
const chats = await scanMultiChatFile(tmpPath)
return { chats }
} finally {
try { fs.unlinkSync(tmpPath) } catch { /* ignore */ }
try { fs.rmdirSync(tmpDir) } catch { /* ignore */ }
try {
fs.unlinkSync(tmpPath)
} catch {
/* ignore */
}
try {
fs.rmdirSync(tmpDir)
} catch {
/* ignore */
}
}
})
@@ -465,7 +475,7 @@ export async function registerWebRoutes(server: FastifyInstance, dbManager: Data
reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
Connection: 'keep-alive',
})
function sendEvent(event: string, data: unknown) {
@@ -497,8 +507,16 @@ export async function registerWebRoutes(server: FastifyInstance, dbManager: Data
sendEvent('error', { success: false, error: err instanceof Error ? err.message : String(err) })
} finally {
reply.raw.end()
try { fs.unlinkSync(tmpPath) } catch { /* ignore */ }
try { fs.rmdirSync(tmpDir) } catch { /* ignore */ }
try {
fs.unlinkSync(tmpPath)
} catch {
/* ignore */
}
try {
fs.rmdirSync(tmpDir)
} catch {
/* ignore */
}
}
})
}
+42 -16
View File
@@ -62,7 +62,7 @@ function generateSessionId(): string {
export async function streamImport(
dbManager: DatabaseManager,
filePath: string,
options?: StreamImportOptions,
options?: StreamImportOptions
): Promise<StreamImportResult> {
const { formatId, chatIndex, nativeBinding, onProgress } = options || {}
@@ -89,12 +89,18 @@ export async function streamImport(
messagesProcessed: p.messagesProcessed,
})
},
onMeta: (m) => { meta = m },
onMembers: (m) => { members.push(...m) },
onMessageBatch: (m) => { messages.push(...m) },
onMeta: (m) => {
meta = m
},
onMembers: (m) => {
members.push(...m)
},
onMessageBatch: (m) => {
messages.push(...m)
},
formatOptions,
},
formatId,
formatId
)
} catch (err) {
const msg = err instanceof Error ? err.message : String(err)
@@ -121,7 +127,7 @@ export async function streamImport(
db.prepare(
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id, schema_version)
VALUES (?, ?, ?, ?, ?, ?, ?, 4)`,
VALUES (?, ?, ?, ?, ?, ?, ?, 4)`
).run(
parsedMeta.name,
parsedMeta.platform,
@@ -129,12 +135,12 @@ export async function streamImport(
Math.floor(Date.now() / 1000),
parsedMeta.groupId || null,
parsedMeta.groupAvatar || null,
parsedMeta.ownerId || null,
parsedMeta.ownerId || null
)
const insertMember = db.prepare(
`INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar, roles)
VALUES (?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, ?)`
)
for (const m of members) {
insertMember.run(
@@ -142,7 +148,7 @@ export async function streamImport(
m.accountName || m.platformId,
m.groupNickname || null,
m.avatar || null,
m.roles ? JSON.stringify(m.roles) : '[]',
m.roles ? JSON.stringify(m.roles) : '[]'
)
}
@@ -150,7 +156,7 @@ export async function streamImport(
const insertMsg = db.prepare(
`INSERT INTO message (sender_id, sender_account_name, sender_group_nickname, ts, type, content, reply_to_message_id, platform_message_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
)
let written = 0
@@ -162,8 +168,16 @@ export async function streamImport(
for (const msg of batch) {
let senderId = memberIdMap.get(msg.senderPlatformId)
if (!senderId) {
insertMember.run(msg.senderPlatformId, msg.senderAccountName || msg.senderPlatformId, msg.senderGroupNickname || null, null, '[]')
senderId = (db.prepare('SELECT id FROM member WHERE platform_id = ?').get(msg.senderPlatformId) as { id: number })?.id
insertMember.run(
msg.senderPlatformId,
msg.senderAccountName || msg.senderPlatformId,
msg.senderGroupNickname || null,
null,
'[]'
)
senderId = (
db.prepare('SELECT id FROM member WHERE platform_id = ?').get(msg.senderPlatformId) as { id: number }
)?.id
if (senderId) memberIdMap.set(msg.senderPlatformId, senderId)
}
if (!senderId) continue
@@ -176,7 +190,7 @@ export async function streamImport(
msg.type,
msg.content,
msg.replyToMessageId || null,
msg.platformMessageId || null,
msg.platformMessageId || null
)
written++
}
@@ -215,9 +229,21 @@ export async function streamImport(
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err)
try { fs.unlinkSync(dbPath) } catch { /* ignore */ }
try { fs.unlinkSync(dbPath + '-wal') } catch { /* ignore */ }
try { fs.unlinkSync(dbPath + '-shm') } catch { /* ignore */ }
try {
fs.unlinkSync(dbPath)
} catch {
/* ignore */
}
try {
fs.unlinkSync(dbPath + '-wal')
} catch {
/* ignore */
}
try {
fs.unlinkSync(dbPath + '-shm')
} catch {
/* ignore */
}
onProgress?.({ stage: 'error', progress: 0, message: msg })
return { success: false, error: msg }
}
+63
View File
@@ -14,6 +14,10 @@ import type {
PaginatedResult,
MentionGraphData,
MessageLengthDistribution,
ImportProgress,
ImportResult,
FormatInfo,
MultiChatEntry,
} from './types'
import type { AnalysisSession, MessageType } from '@/types/base'
import type { TimeFilter } from '@openchatlab/shared-types'
@@ -200,6 +204,65 @@ export class ElectronAdapter implements QueryAdapter {
return window.chatApi.getSchema(sessionId)
}
// ==================== 导入管线 ====================
async importFile(
file: File,
options?: { formatId?: string; chatIndex?: number },
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult> {
const filePath = (window as any).electron?.webUtils?.getPathForFile?.(file)
if (!filePath) {
return { success: false, error: 'Cannot get file path in Electron' }
}
return new Promise((resolve) => {
const unlisten = window.chatApi.onImportProgress((progress: any) => {
onProgress?.({
stage: progress.stage || 'parsing',
progress: progress.percentage || 0,
message: progress.message || '',
bytesRead: progress.bytesRead,
totalBytes: progress.totalBytes,
messagesProcessed: progress.messagesProcessed,
})
})
window.chatApi
.importWithOptions(filePath, options || {})
.then((result) => {
unlisten()
resolve({
success: result.success,
sessionId: result.sessionId,
error: result.error,
})
})
.catch((err: Error) => {
unlisten()
resolve({ success: false, error: err.message })
})
})
}
async detectFormat(file: File): Promise<FormatInfo | null> {
const filePath = (window as any).electron?.webUtils?.getPathForFile?.(file)
if (!filePath) return null
const result = await window.chatApi.detectFormat(filePath)
if (!result) return null
return { ...result, extensions: [] }
}
async scanMultiChatFile(file: File): Promise<MultiChatEntry[]> {
const filePath = (window as any).electron?.webUtils?.getPathForFile?.(file)
if (!filePath) return []
const result = await window.chatApi.scanMultiChatFile(filePath)
if (!result.success || !result.chats) return []
return result.chats
}
async getSupportedFormats(): Promise<FormatInfo[]> {
return window.chatApi.getSupportedFormats()
}
// ==================== 插件系统 ====================
pluginQuery<T = Record<string, unknown>>(sessionId: string, sql: string, params?: unknown[]): Promise<T[]> {
+81
View File
@@ -14,6 +14,10 @@ import type {
PaginatedResult,
MentionGraphData,
MessageLengthDistribution,
ImportProgress,
ImportResult,
FormatInfo,
MultiChatEntry,
} from './types'
import type { AnalysisSession, MessageType } from '@/types/base'
import type { TimeFilter } from '@openchatlab/shared-types'
@@ -276,6 +280,83 @@ export class FetchAdapter implements QueryAdapter {
return get(`/_web/sessions/${sessionId}/schema`)
}
// ==================== 导入管线 ====================
async importFile(
file: File,
options?: { formatId?: string; chatIndex?: number },
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult> {
const form = new FormData()
form.append('file', file)
if (options?.formatId) form.append('formatId', options.formatId)
if (options?.chatIndex !== undefined) form.append('chatIndex', String(options.chatIndex))
const res = await fetch('/_web/import', { method: 'POST', body: form })
if (!res.ok) {
const text = await res.text()
return { success: false, error: `HTTP ${res.status}: ${text}` }
}
const reader = res.body?.getReader()
if (!reader) return { success: false, error: 'No response body' }
const decoder = new TextDecoder()
let buffer = ''
let result: ImportResult = { success: false, error: 'Unknown error' }
while (true) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() || ''
let eventType = ''
for (const line of lines) {
if (line.startsWith('event: ')) {
eventType = line.slice(7).trim()
} else if (line.startsWith('data: ')) {
const data = JSON.parse(line.slice(6))
if (eventType === 'progress') {
onProgress?.(data as ImportProgress)
} else if (eventType === 'done') {
result = data as ImportResult
} else if (eventType === 'error') {
result = data as ImportResult
}
eventType = ''
}
}
}
return result
}
async detectFormat(file: File): Promise<FormatInfo | null> {
const form = new FormData()
form.append('file', file)
const res = await fetch('/_web/detect-format', { method: 'POST', body: form })
if (!res.ok) return null
const data = (await res.json()) as { format: FormatInfo | null }
return data.format
}
async scanMultiChatFile(file: File): Promise<MultiChatEntry[]> {
const form = new FormData()
form.append('file', file)
const res = await fetch('/_web/scan-multi-chat', { method: 'POST', body: form })
if (!res.ok) return []
const data = (await res.json()) as { chats: MultiChatEntry[] }
return data.chats
}
async getSupportedFormats(): Promise<FormatInfo[]> {
return get('/_web/supported-formats')
}
// ==================== 插件系统(不支持) ====================
pluginQuery<T = Record<string, unknown>>(_sessionId: string, _sql: string, _params?: unknown[]): Promise<T[]> {
+46
View File
@@ -106,6 +106,41 @@ export interface MessageLengthDistribution {
grouped: Array<{ range: string; count: number }>
}
// ==================== 导入相关 ====================
export interface ImportProgress {
stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'
progress: number
message: string
bytesRead?: number
totalBytes?: number
messagesProcessed?: number
}
export interface ImportResult {
success: boolean
sessionId?: string
error?: string
messageCount?: number
memberCount?: number
}
export interface FormatInfo {
id: string
name: string
platform: string
extensions: string[]
multiChat?: boolean
}
export interface MultiChatEntry {
index: number
name: string
type: string
id: number
messageCount: number
}
// ==================== 核心适配器接口 ====================
export interface QueryAdapter {
@@ -172,6 +207,17 @@ export interface QueryAdapter {
executeSQL(sessionId: string, sql: string): Promise<SQLResult>
getSchema(sessionId: string): Promise<TableSchema[]>
// ==================== 导入管线 ====================
importFile(
file: File,
options?: { formatId?: string; chatIndex?: number },
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult>
detectFormat(file: File): Promise<FormatInfo | null>
scanMultiChatFile(file: File): Promise<MultiChatEntry[]>
getSupportedFormats(): Promise<FormatInfo[]>
// ==================== 插件系统(能力依赖) ====================
pluginQuery<T = Record<string, unknown>>(sessionId: string, sql: string, params?: unknown[]): Promise<T[]>
+127 -33
View File
@@ -119,46 +119,120 @@ async function checkImportLog() {
hasImportLog.value = result.success && !!result.path
}
const dropZoneRef = ref<InstanceType<typeof FileDropZone> | null>(null)
// 处理文件选择(点击选择)- 支持多选
async function handleClickImport() {
if (!IS_ELECTRON) return
importError.value = null
hasImportLog.value = false
importDiagnostics.value = null
// 使用系统对话框选择多个文件
const result = await window.api.dialog.showOpenDialog({
title: t('home.import.selectFiles'),
properties: ['openFile', 'multiSelections'],
filters: [
{ name: t('home.import.chatRecords'), extensions: ['json', 'jsonl', 'txt'] },
{ name: t('home.import.allFiles'), extensions: ['*'] },
],
})
if (result.canceled || result.filePaths.length === 0) {
return
if (IS_ELECTRON) {
const result = await window.api.dialog.showOpenDialog({
title: t('home.import.selectFiles'),
properties: ['openFile', 'multiSelections'],
filters: [
{ name: t('home.import.chatRecords'), extensions: ['json', 'jsonl', 'txt'] },
{ name: t('home.import.allFiles'), extensions: ['*'] },
],
})
if (result.canceled || result.filePaths.length === 0) return
await processFilePaths(result.filePaths)
} else {
dropZoneRef.value?.openFileDialog()
}
await processFilePaths(result.filePaths)
}
// 处理文件拖拽 - 支持多选 - Electron only
async function handleFileDrop({ paths }: { files: File[]; paths: string[] }) {
if (!IS_ELECTRON) return
if (paths.length === 0) {
// 处理文件拖拽/选择 - 支持 Electron 路径和 Web File 对象
async function handleFileDrop({ files, paths }: { files: File[]; paths: string[] }) {
importError.value = null
hasImportLog.value = false
importDiagnostics.value = null
if (IS_ELECTRON && paths.length > 0) {
await processFilePaths(paths)
} else if (files.length > 0) {
await processWebFiles(files)
} else {
importError.value = t('home.import.cannotReadPath')
return
}
importError.value = null
hasImportLog.value = false
importDiagnostics.value = null
await processFilePaths(paths)
}
// 统一处理文件路径(单文件或多文件)
// Web 模式:使用 adapter 导入 File 对象
async function processWebFiles(files: File[]) {
const adapter = getAdapter()
if (files.length === 1) {
const file = files[0]
const format = await adapter.detectFormat(file)
if (format?.multiChat) {
pendingWebFile.value = file
const chats = await adapter.scanMultiChatFile(file)
if (chats.length > 0) {
webMultiChats.value = chats.map((c) => ({
index: c.index,
name: c.name,
type: c.type,
id: c.id,
messageCount: c.messageCount,
}))
showChatSelector.value = true
return
}
}
if (!format) {
pendingFormatFile.value = file
formatSelectorFilePath.value = file.name
}
await importSingleWebFile(file)
} else {
for (const file of files) {
await importSingleWebFile(file)
}
}
}
const pendingWebFile = ref<File | null>(null)
const webMultiChats = ref<ChatInfo[]>([])
async function importSingleWebFile(file: File, options?: { formatId?: string; chatIndex?: number }) {
const adapter = getAdapter()
isImporting.value = true
importProgress.value = { stage: 'detecting', progress: 0, message: '' }
try {
const result = await adapter.importFile(file, options, (p) => {
if (p.stage === 'done') return
importProgress.value = p
})
if (importProgress.value) {
importProgress.value.progress = 100
}
await new Promise((resolve) => setTimeout(resolve, 300))
if (result.success && result.sessionId) {
await sessionStore.loadSessions()
sessionStore.selectSession(result.sessionId)
await navigateToSession(result.sessionId)
} else {
importError.value = translateError(result.error || 'error.import_failed')
}
} catch (error) {
importError.value = String(error)
} finally {
isImporting.value = false
setTimeout(() => {
importProgress.value = null
}, 500)
}
}
// 统一处理文件路径(单文件或多文件)- Electron only
async function processFilePaths(paths: string[]) {
// 单文件 或 未启用合并导入 - 使用原有逻辑
if (paths.length === 1 || !mergeImportEnabled.value) {
@@ -207,8 +281,17 @@ async function processFilePaths(paths: string[]) {
await sessionStore.mergeImportFiles(paths)
}
const pendingFormatFile = ref<File | null>(null)
// 手动格式选择后的导入处理
async function handleFormatSelect(formatId: string) {
if (!IS_ELECTRON && pendingFormatFile.value) {
const file = pendingFormatFile.value
pendingFormatFile.value = null
await importSingleWebFile(file, { formatId })
return
}
const filePath = formatSelectorFilePath.value
if (!filePath) return
@@ -253,10 +336,25 @@ async function handleFormatSelect(formatId: string) {
async function handleChatSelect(selectedChats: ChatInfo[]) {
if (selectedChats.length === 0) return
// Web 模式:使用 pendingWebFile + adapter
if (!IS_ELECTRON && pendingWebFile.value) {
const file = pendingWebFile.value
pendingWebFile.value = null
webMultiChats.value = []
if (selectedChats.length === 1) {
await importSingleWebFile(file, { chatIndex: selectedChats[0].index })
} else {
for (const chat of selectedChats) {
await importSingleWebFile(file, { chatIndex: chat.index })
}
}
return
}
const filePath = chatSelectorFilePath.value
if (selectedChats.length === 1) {
// 单个聊天:直接导入
isImporting.value = true
importProgress.value = { stage: 'detecting', progress: 0, message: '' }
@@ -277,7 +375,6 @@ async function handleChatSelect(selectedChats: ChatInfo[]) {
if (result.success && result.sessionId) {
await sessionStore.loadSessions()
sessionStore.selectSession(result.sessionId)
// 自动生成会话索引
await autoGenerateSessionIndex(result.sessionId)
await navigateToSession(result.sessionId)
} else {
@@ -292,7 +389,6 @@ async function handleChatSelect(selectedChats: ChatInfo[]) {
}, 500)
}
} else {
// 多个聊天:逐个导入(类似批量导入)
isBatchImporting.value = true
batchFiles.value = selectedChats.map((chat) => ({
path: `${filePath}#${chat.index}`,
@@ -320,7 +416,6 @@ async function handleChatSelect(selectedChats: ChatInfo[]) {
batchFiles.value[i].status = 'success'
batchFiles.value[i].sessionId = result.sessionId
successCount++
// 自动生成会话索引
await autoGenerateSessionIndex(result.sessionId)
} else {
batchFiles.value[i].status = 'failed'
@@ -335,10 +430,8 @@ async function handleChatSelect(selectedChats: ChatInfo[]) {
}
}
// 刷新会话列表
await sessionStore.loadSessions()
// 转为结果状态
isBatchImporting.value = false
batchImportResult.value = {
total: selectedChats.length,
@@ -681,6 +774,7 @@ const getMergeFileProgressText = (file: MergeFileInfo) =>
<!-- 默认导入区域(非批量状态) -->
<FileDropZone
v-else
ref="dropZoneRef"
:accept="['.json', '.jsonl', '.txt']"
:disabled="isAnyImporting"
:multiple="true"
+1 -1
View File
@@ -138,7 +138,7 @@ export interface AnalysisSession {
* 导入进度回调
*/
export interface ImportProgress {
stage: 'detecting' | 'reading' | 'parsing' | 'saving' | 'done' | 'error'
stage: 'detecting' | 'reading' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'
progress: number // 0-100
message?: string
// 流式解析额外字段