mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
feat(import): support unified directory import for Electron and Web
Add directory import support for multi-file formats (e.g. shuakami chunked-jsonl). Users can select or drag-and-drop a directory via the "Import Folder" checkbox. - Add `findEntryFileInDirectory()` to parser layer for format-agnostic entry file discovery - Add `importDirectory` method to ImportAdapter interface - Implement FetchImportAdapter.importDirectory with multi-file FormData upload - Add `POST /_web/import-directory` CLI route with directory reconstruction - Add `chat:importDirectory` IPC handler in Electron main process - Enhance FileDropZone with `directory` prop and drag-and-drop directory support - Refactor FetchImportAdapter to extract reusable `consumeSseStream` helper Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
8bb904fb22
commit
9421662a52
16 files changed
+462
-76
No files matched your search
@@ -70,6 +70,7 @@ import {
|
||||
detectAllFormats,
|
||||
getSupportedFormats,
|
||||
scanMultiChatFile,
|
||||
findEntryFileInDirectory,
|
||||
} from '../../import'
|
||||
import { getDefaultAssistantConfig, buildPiModel } from '../../ai/llm-config'
|
||||
|
||||
@@ -880,6 +881,92 @@ export function registerWebRoutes(
|
||||
}
|
||||
})
|
||||
|
||||
// ==================== Directory Import ====================
|
||||
|
||||
server.post('/_web/import-directory', async (request, reply) => {
|
||||
const parts = (request as any).parts()
|
||||
if (!parts) return reply.code(400).send({ error: 'No files uploaded' })
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-dir-import-'))
|
||||
|
||||
// Interleaved fields: relativePaths (field) and files (file) come in order
|
||||
const relativePaths: string[] = []
|
||||
const fileBuffers: { data: Buffer; filename: string }[] = []
|
||||
|
||||
try {
|
||||
for await (const part of parts) {
|
||||
if (part.type === 'field' && part.fieldname === 'relativePaths') {
|
||||
relativePaths.push(String(part.value))
|
||||
} else if (part.type === 'file') {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
fileBuffers.push({ data: Buffer.concat(chunks), filename: part.filename || '' })
|
||||
}
|
||||
}
|
||||
|
||||
// Write files preserving directory structure
|
||||
for (let i = 0; i < fileBuffers.length; i++) {
|
||||
let relPath = relativePaths[i] || fileBuffers[i].filename || `file_${i}`
|
||||
|
||||
// Strip top-level directory from webkitRelativePath (e.g. "export_dir/chunks/..." → "chunks/...")
|
||||
const segments = relPath.split('/')
|
||||
if (segments.length > 1) {
|
||||
relPath = segments.slice(1).join('/')
|
||||
}
|
||||
|
||||
const targetPath = path.join(tmpDir, relPath)
|
||||
fs.mkdirSync(path.dirname(targetPath), { recursive: true })
|
||||
fs.writeFileSync(targetPath, fileBuffers[i].data)
|
||||
}
|
||||
|
||||
const entryPath = findEntryFileInDirectory(tmpDir)
|
||||
if (!entryPath) {
|
||||
return reply.code(400).send({ error: 'No recognizable import format found in directory' })
|
||||
}
|
||||
|
||||
reply.raw.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
})
|
||||
|
||||
function sendEvent(event: string, eventData: unknown) {
|
||||
reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(eventData)}\n\n`)
|
||||
}
|
||||
|
||||
const nativeBinding = resolveNativeBinding()
|
||||
const result = await streamImport(dbManager, entryPath, {
|
||||
nativeBinding,
|
||||
onProgress: (p) => {
|
||||
sendEvent('progress', p)
|
||||
},
|
||||
})
|
||||
|
||||
if (result.success) {
|
||||
sendEvent('done', {
|
||||
success: true,
|
||||
sessionId: result.sessionId,
|
||||
messageCount: result.diagnostics?.messagesWritten ?? 0,
|
||||
memberCount: 0,
|
||||
})
|
||||
} else {
|
||||
sendEvent('error', { success: false, error: result.error })
|
||||
}
|
||||
} catch (err) {
|
||||
if (!reply.raw.headersSent) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) })
|
||||
}
|
||||
reply.raw.write(
|
||||
`event: error\ndata: ${JSON.stringify({ success: false, error: err instanceof Error ? err.message : String(err) })}\n\n`
|
||||
)
|
||||
} finally {
|
||||
reply.raw.end()
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
// ==================== Incremental Import ====================
|
||||
|
||||
server.post<{ Params: { id: string } }>('/_web/sessions/:id/import/incremental', async (request, reply) => {
|
||||
|
||||
@@ -15,6 +15,7 @@ export {
|
||||
getFormatFeatureById,
|
||||
getSupportedFormats,
|
||||
scanMultiChatFile,
|
||||
findEntryFileInDirectory,
|
||||
} from './stream-import'
|
||||
export type {
|
||||
StreamImportProgress,
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
getFormatFeatureById,
|
||||
getSupportedFormats as parserGetSupportedFormats,
|
||||
scanMultiChatFile as parserScanMultiChatFile,
|
||||
findEntryFileInDirectory,
|
||||
type FormatFeature,
|
||||
type MultiChatInfo,
|
||||
type ParseProgress,
|
||||
@@ -216,6 +217,7 @@ export {
|
||||
getFormatFeatureById,
|
||||
parserGetSupportedFormats as getSupportedFormats,
|
||||
parserScanMultiChatFile as scanMultiChatFile,
|
||||
findEntryFileInDirectory,
|
||||
}
|
||||
export type { FormatFeature, MultiChatInfo, ParseProgress }
|
||||
export type {
|
||||
|
||||
@@ -7,7 +7,7 @@ import { getConversationCountsBySession } from '../ai/conversations'
|
||||
import * as databaseCore from '../database/core'
|
||||
import * as worker from '../worker/workerManager'
|
||||
import * as parser from '../parser'
|
||||
import { detectFormat, scanMultiChatFile, type ParseProgress } from '../parser'
|
||||
import { detectFormat, findEntryFileInDirectory, scanMultiChatFile, type ParseProgress } from '../parser'
|
||||
import type { IpcContext } from './types'
|
||||
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos } from '../database/migrations'
|
||||
import { exportSessionToTempFile, cleanupTempExportFiles } from '../merger'
|
||||
@@ -143,6 +143,45 @@ export function registerChatHandlers(ctx: IpcContext): void {
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Import from a directory path: scan for entry file and import
|
||||
*/
|
||||
ipcMain.handle('chat:importDirectory', async (_, dirPath: string) => {
|
||||
try {
|
||||
const entryPath = findEntryFileInDirectory(dirPath)
|
||||
if (!entryPath) {
|
||||
return { success: false, error: 'No recognizable import format found in directory' }
|
||||
}
|
||||
|
||||
win.webContents.send('chat:importProgress', {
|
||||
stage: 'detecting',
|
||||
progress: 5,
|
||||
message: '',
|
||||
})
|
||||
|
||||
const result = await worker.streamImport(entryPath, (progress: ParseProgress) => {
|
||||
win.webContents.send('chat:importProgress', {
|
||||
stage: progress.stage,
|
||||
progress: progress.percentage,
|
||||
message: progress.message,
|
||||
bytesRead: progress.bytesRead,
|
||||
totalBytes: progress.totalBytes,
|
||||
messagesProcessed: progress.messagesProcessed,
|
||||
})
|
||||
})
|
||||
|
||||
if (result.success) {
|
||||
return { success: true, sessionId: result.sessionId, diagnostics: result.diagnostics }
|
||||
} else {
|
||||
win.webContents.send('chat:importProgress', { stage: 'error', progress: 0, message: result.error })
|
||||
return { success: false, error: result.error, diagnostics: result.diagnostics }
|
||||
}
|
||||
} catch (error) {
|
||||
win.webContents.send('chat:importProgress', { stage: 'error', progress: 0, message: String(error) })
|
||||
return { success: false, error: String(error) }
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* 检测文件格式(轻量级,仅返回格式 ID、名称和是否多聊天)
|
||||
*/
|
||||
|
||||
@@ -16,6 +16,7 @@ export {
|
||||
parseFileSync,
|
||||
parseFileInfo,
|
||||
streamParseFile,
|
||||
findEntryFileInDirectory,
|
||||
FormatSniffer,
|
||||
createSniffer,
|
||||
getFileSize,
|
||||
|
||||
@@ -60,6 +60,13 @@ export const chatApi = {
|
||||
return ipcRenderer.invoke('chat:import', filePath)
|
||||
},
|
||||
|
||||
/**
|
||||
* Import from a directory path (scans for entry file)
|
||||
*/
|
||||
importDirectory: (dirPath: string): Promise<{ success: boolean; sessionId?: string; error?: string }> => {
|
||||
return ipcRenderer.invoke('chat:importDirectory', dirPath)
|
||||
},
|
||||
|
||||
/**
|
||||
* 检测文件格式(轻量级)
|
||||
*/
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
* 三层架构:标准层、嗅探层、解析层
|
||||
*/
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { FormatSniffer, createSniffer } from './sniffer'
|
||||
import { formats } from './formats'
|
||||
import { getFileSize } from './utils'
|
||||
@@ -238,8 +240,6 @@ export async function parseFileInfo(
|
||||
}
|
||||
}
|
||||
|
||||
// 获取文件大小
|
||||
const fs = await import('fs')
|
||||
const fileSize = fs.statSync(filePath).size
|
||||
|
||||
return {
|
||||
@@ -252,6 +252,41 @@ export async function parseFileInfo(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan a directory for a recognizable entry file (e.g. manifest.json for chunked formats).
|
||||
* Uses existing format detection; no format-specific filenames are hardcoded here.
|
||||
* @param dirPath directory path to scan
|
||||
* @param maxDepth max depth to scan (default 1, only top-level files)
|
||||
* @returns path to the entry file, or null if none found
|
||||
*/
|
||||
export function findEntryFileInDirectory(dirPath: string, maxDepth = 1): string | null {
|
||||
function walk(current: string, depth: number): string | null {
|
||||
if (depth > maxDepth) return null
|
||||
let entries: fs.Dirent[]
|
||||
try {
|
||||
entries = fs.readdirSync(current, { withFileTypes: true })
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const full = path.join(current, entry.name)
|
||||
if (entry.isFile() && entry.name.endsWith('.json')) {
|
||||
const format = detectFormat(full)
|
||||
if (format) return full
|
||||
}
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const found = walk(path.join(current, entry.name), depth + 1)
|
||||
if (found) return found
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
return walk(dirPath, 0)
|
||||
}
|
||||
|
||||
// ==================== 导出类型 ====================
|
||||
|
||||
export type {
|
||||
|
||||
@@ -13,17 +13,22 @@ interface Props {
|
||||
disabled?: boolean
|
||||
/** 接受的文件扩展名,如 ['.json', '.txt'] */
|
||||
accept?: string[]
|
||||
/** 是否为目录选择模式(webkitdirectory) */
|
||||
directory?: boolean
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
multiple: false,
|
||||
disabled: false,
|
||||
accept: () => ['*'],
|
||||
directory: false,
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
/** 选择文件后触发,返回文件列表和路径列表 */
|
||||
files: [payload: { files: File[]; paths: string[] }]
|
||||
/** 拖入文件夹时触发 (Electron: dirPath, Web: File[]) */
|
||||
'directory-drop': [payload: { files: File[]; dirPath: string | null }]
|
||||
}>()
|
||||
|
||||
// 拖拽状态
|
||||
@@ -79,7 +84,7 @@ function handleDragLeave(e: DragEvent) {
|
||||
}
|
||||
|
||||
// 处理拖拽放下
|
||||
function handleDrop(e: DragEvent) {
|
||||
async function handleDrop(e: DragEvent) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
isDragOver.value = false
|
||||
@@ -87,16 +92,26 @@ function handleDrop(e: DragEvent) {
|
||||
if (props.disabled) return
|
||||
|
||||
const dataTransfer = e.dataTransfer
|
||||
if (!dataTransfer?.files || dataTransfer.files.length === 0) return
|
||||
if (!dataTransfer) return
|
||||
|
||||
// Check if a directory was dropped (via DataTransferItem API)
|
||||
const items = dataTransfer.items
|
||||
if (items?.length === 1) {
|
||||
const entry = items[0].webkitGetAsEntry?.()
|
||||
if (entry?.isDirectory) {
|
||||
await handleDirectoryDrop(entry as FileSystemDirectoryEntry, dataTransfer.files)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (dataTransfer.files.length === 0) return
|
||||
|
||||
let files = Array.from(dataTransfer.files)
|
||||
|
||||
// 如果不支持多选,只取第一个
|
||||
if (!props.multiple) {
|
||||
files = [files[0]]
|
||||
}
|
||||
|
||||
// 过滤文件类型
|
||||
if (!props.accept.includes('*')) {
|
||||
files = files.filter((file) => {
|
||||
const ext = '.' + file.name.split('.').pop()?.toLowerCase()
|
||||
@@ -109,6 +124,59 @@ function handleDrop(e: DragEvent) {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDirectoryDrop(dirEntry: FileSystemDirectoryEntry, fileList: FileList) {
|
||||
// Electron: try to get directory path
|
||||
if (fileList.length > 0) {
|
||||
try {
|
||||
// @ts-expect-error Electron webUtils
|
||||
const dirPath = window.electron?.webUtils?.getPathForFile?.(fileList[0])
|
||||
if (dirPath) {
|
||||
emit('directory-drop', { files: [], dirPath })
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// Not Electron
|
||||
}
|
||||
}
|
||||
|
||||
// Web: recursively read directory entries into File objects
|
||||
const files = await readDirectoryEntries(dirEntry, dirEntry.name)
|
||||
if (files.length > 0) {
|
||||
emit('directory-drop', { files, dirPath: null })
|
||||
}
|
||||
}
|
||||
|
||||
async function readDirectoryEntries(dirEntry: FileSystemDirectoryEntry, basePath: string): Promise<File[]> {
|
||||
const reader = dirEntry.createReader()
|
||||
const files: File[] = []
|
||||
|
||||
const readBatch = (): Promise<FileSystemEntry[]> =>
|
||||
new Promise((resolve, reject) => reader.readEntries(resolve, reject))
|
||||
|
||||
let entries: FileSystemEntry[]
|
||||
do {
|
||||
entries = await readBatch()
|
||||
for (const entry of entries) {
|
||||
if (entry.isFile) {
|
||||
const file = await new Promise<File>((resolve, reject) => {
|
||||
;(entry as FileSystemFileEntry).file(resolve, reject)
|
||||
})
|
||||
// Attach webkitRelativePath-like info via property
|
||||
Object.defineProperty(file, 'webkitRelativePath', {
|
||||
value: `${basePath}/${entry.name}`,
|
||||
writable: false,
|
||||
})
|
||||
files.push(file)
|
||||
} else if (entry.isDirectory) {
|
||||
const subFiles = await readDirectoryEntries(entry as FileSystemDirectoryEntry, `${basePath}/${entry.name}`)
|
||||
files.push(...subFiles)
|
||||
}
|
||||
}
|
||||
} while (entries.length > 0)
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// 处理文件并发送事件
|
||||
function processFiles(files: File[]) {
|
||||
const paths: string[] = []
|
||||
@@ -142,8 +210,9 @@ defineExpose({
|
||||
ref="fileInputRef"
|
||||
type="file"
|
||||
:multiple="multiple"
|
||||
:accept="acceptAttr"
|
||||
:accept="directory ? undefined : acceptAttr"
|
||||
class="hidden"
|
||||
v-bind="directory ? { webkitdirectory: '', directory: '' } : {}"
|
||||
@change="handleFileSelect"
|
||||
/>
|
||||
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
"clickHint": "Click to select or drag chat records to import",
|
||||
"multipleHint": "Multiple files can be selected or dragged at once",
|
||||
"selectFiles": "Select chat record files",
|
||||
"selectFolder": "Select export folder",
|
||||
"importFolder": "Import Folder",
|
||||
"chatRecords": "Chat Records",
|
||||
"allFiles": "All Files",
|
||||
"progress": {
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
"clickHint": "クリックして選択、またはチャット記録をドラッグ&ドロップでインポート",
|
||||
"multipleHint": "複数ファイルの同時選択・ドラッグに対応しています",
|
||||
"selectFiles": "チャット記録ファイルを選択",
|
||||
"selectFolder": "エクスポートフォルダを選択",
|
||||
"importFolder": "フォルダをインポート",
|
||||
"chatRecords": "チャット記録",
|
||||
"allFiles": "すべてのファイル",
|
||||
"progress": {
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
"clickHint": "点击选择或拖拽聊天记录导入",
|
||||
"multipleHint": "支持同时选择或拖拽多个文件",
|
||||
"selectFiles": "选择聊天记录文件",
|
||||
"selectFolder": "选择导出文件夹",
|
||||
"importFolder": "导入文件夹",
|
||||
"chatRecords": "聊天记录",
|
||||
"allFiles": "所有文件",
|
||||
"progress": {
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
"clickHint": "點一下選擇,或直接拖曳聊天紀錄匯入",
|
||||
"multipleHint": "支援同時選擇或拖曳多個檔案",
|
||||
"selectFiles": "選擇聊天紀錄檔案",
|
||||
"selectFolder": "選擇匯出資料夾",
|
||||
"importFolder": "匯入資料夾",
|
||||
"chatRecords": "聊天紀錄",
|
||||
"allFiles": "所有檔案",
|
||||
"progress": {
|
||||
|
||||
@@ -119,12 +119,20 @@ async function checkImportLog() {
|
||||
|
||||
const dropZoneRef = ref<InstanceType<typeof FileDropZone> | null>(null)
|
||||
|
||||
// 处理文件选择(点击选择)- 支持多选
|
||||
// 文件夹导入模式
|
||||
const folderImportEnabled = ref(false)
|
||||
|
||||
// 处理文件选择(点击选择)
|
||||
async function handleClickImport() {
|
||||
importError.value = null
|
||||
hasImportLog.value = false
|
||||
importDiagnostics.value = null
|
||||
|
||||
if (folderImportEnabled.value) {
|
||||
handleClickImportDirectory()
|
||||
return
|
||||
}
|
||||
|
||||
if (IS_ELECTRON) {
|
||||
const result = await usePlatformService().showOpenDialog({
|
||||
title: t('home.import.selectFiles'),
|
||||
@@ -228,6 +236,73 @@ async function importSingleWebFile(file: File, options?: { formatId?: string; ch
|
||||
}
|
||||
}
|
||||
|
||||
// 目录导入 ref
|
||||
const dirDropZoneRef = ref<InstanceType<typeof FileDropZone> | null>(null)
|
||||
|
||||
async function handleClickImportDirectory() {
|
||||
importError.value = null
|
||||
if (IS_ELECTRON) {
|
||||
const result = await usePlatformService().showOpenDialog({
|
||||
title: t('home.import.selectFolder'),
|
||||
properties: ['openDirectory'],
|
||||
})
|
||||
if (result.canceled || result.filePaths.length === 0) return
|
||||
await importDirectory(result.filePaths[0])
|
||||
} else {
|
||||
dirDropZoneRef.value?.openFileDialog()
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDirectoryDrop({ files }: { files: File[]; paths: string[] }) {
|
||||
if (files.length === 0) return
|
||||
importError.value = null
|
||||
await importDirectory(files)
|
||||
}
|
||||
|
||||
async function handleDirectoryDropEvent({ files, dirPath }: { files: File[]; dirPath: string | null }) {
|
||||
importError.value = null
|
||||
hasImportLog.value = false
|
||||
importDiagnostics.value = null
|
||||
|
||||
if (dirPath) {
|
||||
await importDirectory(dirPath)
|
||||
} else if (files.length > 0) {
|
||||
await importDirectory(files)
|
||||
}
|
||||
}
|
||||
|
||||
async function importDirectory(source: File[] | string) {
|
||||
isImporting.value = true
|
||||
importProgress.value = { stage: 'detecting', progress: 0, message: '' }
|
||||
|
||||
try {
|
||||
const result = await useImportService().importDirectory(source, undefined, (p) => {
|
||||
if (p.stage === 'done') return
|
||||
importProgress.value = p
|
||||
})
|
||||
|
||||
if (importProgress.value) {
|
||||
importProgress.value.progress = 100
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
|
||||
if (result.success && result.sessionId) {
|
||||
await sessionStore.loadSessions()
|
||||
sessionStore.selectSession(result.sessionId)
|
||||
await navigateToSession(result.sessionId)
|
||||
} else {
|
||||
importError.value = translateError(result.error || 'error.import_failed')
|
||||
}
|
||||
} catch (error) {
|
||||
importError.value = String(error)
|
||||
} finally {
|
||||
isImporting.value = false
|
||||
setTimeout(() => {
|
||||
importProgress.value = null
|
||||
}, 500)
|
||||
}
|
||||
}
|
||||
|
||||
// 统一处理文件路径(单文件或多文件)- Electron only
|
||||
async function processFilePaths(paths: string[]) {
|
||||
// 单文件 或 未启用合并导入 - 使用原有逻辑
|
||||
@@ -769,6 +844,7 @@ const getMergeFileProgressText = (file: MergeFileInfo) =>
|
||||
:multiple="true"
|
||||
class="w-full max-w-4xl"
|
||||
@files="handleFileDrop"
|
||||
@directory-drop="handleDirectoryDropEvent"
|
||||
>
|
||||
<template #default="{ isDragOver }">
|
||||
<div
|
||||
@@ -819,8 +895,21 @@ const getMergeFileProgressText = (file: MergeFileInfo) =>
|
||||
</template>
|
||||
</FileDropZone>
|
||||
|
||||
<!-- 合并导入选项 -->
|
||||
<div v-if="!isAnyImporting && !batchImportResult" class="flex items-center justify-center">
|
||||
<!-- 隐藏的目录选择 input -->
|
||||
<FileDropZone ref="dirDropZoneRef" :directory="true" class="hidden" @files="handleDirectoryDrop" />
|
||||
|
||||
<!-- 导入选项 -->
|
||||
<div v-if="!isAnyImporting && !batchImportResult" class="flex flex-wrap items-center justify-center gap-4">
|
||||
<!-- 导入文件夹模式 -->
|
||||
<UCheckbox
|
||||
v-model="folderImportEnabled"
|
||||
:label="t('home.import.importFolder')"
|
||||
input-class="h-4 w-4"
|
||||
size="sm"
|
||||
label-class="text-sm font-medium text-gray-600 dark:text-gray-300"
|
||||
/>
|
||||
|
||||
<!-- 合并导入 -->
|
||||
<div class="flex items-center gap-2">
|
||||
<UCheckbox
|
||||
v-model="mergeImportEnabled"
|
||||
|
||||
@@ -125,4 +125,43 @@ export class ElectronImportAdapter implements ImportAdapter {
|
||||
unlisten()
|
||||
}
|
||||
}
|
||||
|
||||
async importDirectory(
|
||||
source: File[] | string,
|
||||
_options?: ImportOptions,
|
||||
onProgress?: (p: ImportProgress) => void
|
||||
): Promise<ImportResult> {
|
||||
if (typeof source !== 'string') {
|
||||
return { success: false, error: 'Expected directory path in Electron mode' }
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const unlisten = window.chatApi.onImportProgress((progress: any) => {
|
||||
onProgress?.({
|
||||
stage: progress.stage || 'parsing',
|
||||
progress: progress.percentage || progress.progress || 0,
|
||||
message: progress.message || '',
|
||||
bytesRead: progress.bytesRead,
|
||||
totalBytes: progress.totalBytes,
|
||||
messagesProcessed: progress.messagesProcessed,
|
||||
})
|
||||
})
|
||||
|
||||
window.chatApi
|
||||
.importDirectory(source)
|
||||
.then((result: any) => {
|
||||
unlisten()
|
||||
resolve({
|
||||
success: result.success,
|
||||
sessionId: result.sessionId,
|
||||
error: result.error,
|
||||
diagnostics: result.diagnostics,
|
||||
})
|
||||
})
|
||||
.catch((err: Error) => {
|
||||
unlisten()
|
||||
resolve({ success: false, error: err.message })
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,41 @@ import { get } from '../utils/http'
|
||||
|
||||
const BASE = '/_web'
|
||||
|
||||
async function consumeSseStream<T>(res: Response, fallback: T, onProgress?: (p: ImportProgress) => void): Promise<T> {
|
||||
const reader = res.body?.getReader()
|
||||
if (!reader) return fallback
|
||||
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
let result: T = fallback
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
let eventType = ''
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event: ')) {
|
||||
eventType = line.slice(7).trim()
|
||||
} else if (line.startsWith('data: ')) {
|
||||
const data = JSON.parse(line.slice(6))
|
||||
if (eventType === 'progress') {
|
||||
onProgress?.(data as ImportProgress)
|
||||
} else if (eventType === 'done' || eventType === 'error') {
|
||||
result = data as T
|
||||
}
|
||||
eventType = ''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export class FetchImportAdapter implements ImportAdapter {
|
||||
async importFile(
|
||||
file: File | string,
|
||||
@@ -42,40 +77,7 @@ export class FetchImportAdapter implements ImportAdapter {
|
||||
return { success: false, error: `HTTP ${res.status}: ${text}` }
|
||||
}
|
||||
|
||||
const reader = res.body?.getReader()
|
||||
if (!reader) return { success: false, error: 'No response body' }
|
||||
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
let result: ImportResult = { success: false, error: 'Unknown error' }
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
let eventType = ''
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event: ')) {
|
||||
eventType = line.slice(7).trim()
|
||||
} else if (line.startsWith('data: ')) {
|
||||
const data = JSON.parse(line.slice(6))
|
||||
if (eventType === 'progress') {
|
||||
onProgress?.(data as ImportProgress)
|
||||
} else if (eventType === 'done') {
|
||||
result = data as ImportResult
|
||||
} else if (eventType === 'error') {
|
||||
result = data as ImportResult
|
||||
}
|
||||
eventType = ''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
return consumeSseStream<ImportResult>(res, { success: false, error: 'Unknown error' }, onProgress)
|
||||
}
|
||||
|
||||
async detectFormat(file: File | string): Promise<FormatInfo | null> {
|
||||
@@ -194,39 +196,39 @@ export class FetchImportAdapter implements ImportAdapter {
|
||||
return { success: false, newMessageCount: 0, error: `HTTP ${res.status}: ${text}` }
|
||||
}
|
||||
|
||||
const reader = res.body?.getReader()
|
||||
if (!reader) return { success: false, newMessageCount: 0, error: 'No response body' }
|
||||
return consumeSseStream<IncrementalImportResult>(
|
||||
res,
|
||||
{ success: false, newMessageCount: 0, error: 'Unknown error' },
|
||||
onProgress
|
||||
)
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
let result: IncrementalImportResult = { success: false, newMessageCount: 0, error: 'Unknown error' }
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
let eventType = ''
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event: ')) {
|
||||
eventType = line.slice(7).trim()
|
||||
} else if (line.startsWith('data: ')) {
|
||||
const data = JSON.parse(line.slice(6))
|
||||
if (eventType === 'progress') {
|
||||
onProgress?.(data as ImportProgress)
|
||||
} else if (eventType === 'done') {
|
||||
result = data as IncrementalImportResult
|
||||
} else if (eventType === 'error') {
|
||||
result = data as IncrementalImportResult
|
||||
}
|
||||
eventType = ''
|
||||
}
|
||||
}
|
||||
async importDirectory(
|
||||
source: File[] | string,
|
||||
_options?: ImportOptions,
|
||||
onProgress?: (p: ImportProgress) => void
|
||||
): Promise<ImportResult> {
|
||||
if (typeof source === 'string') {
|
||||
return { success: false, error: 'Directory path import is not supported in Web mode' }
|
||||
}
|
||||
|
||||
return result
|
||||
if (source.length === 0) {
|
||||
return { success: false, error: 'No files in directory' }
|
||||
}
|
||||
|
||||
const form = new FormData()
|
||||
for (const file of source) {
|
||||
form.append('files', file)
|
||||
form.append('relativePaths', file.webkitRelativePath || file.name)
|
||||
}
|
||||
|
||||
const res = await fetch(`${BASE}/import-directory`, { method: 'POST', body: form })
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text()
|
||||
return { success: false, error: `HTTP ${res.status}: ${text}` }
|
||||
}
|
||||
|
||||
return consumeSseStream<ImportResult>(res, { success: false, error: 'Unknown error' }, onProgress)
|
||||
}
|
||||
}
|
||||
@@ -118,6 +118,13 @@ export interface ImportAdapter {
|
||||
file: File | string,
|
||||
onProgress?: (p: ImportProgress) => void
|
||||
): Promise<IncrementalImportResult>
|
||||
|
||||
/** 导入目录(多文件格式如 chunked-jsonl)。Electron 传目录路径,Web 传 File[](含 webkitRelativePath) */
|
||||
importDirectory(
|
||||
source: File[] | string,
|
||||
options?: ImportOptions,
|
||||
onProgress?: (p: ImportProgress) => void
|
||||
): Promise<ImportResult>
|
||||
}
|
||||
|
||||
export type { ImportProgress }
|
||||
Reference in new issue
Block a user