feat(import): support unified directory import for Electron and Web

Add directory import support for multi-file formats (e.g. shuakami chunked-jsonl).
Users can select or drag-and-drop a directory via the "Import Folder" checkbox.

- Add `findEntryFileInDirectory()` to parser layer for format-agnostic entry file discovery
- Add `importDirectory` method to ImportAdapter interface
- Implement FetchImportAdapter.importDirectory with multi-file FormData upload
- Add `POST /_web/import-directory` CLI route with directory reconstruction
- Add `chat:importDirectory` IPC handler in Electron main process
- Enhance FileDropZone with `directory` prop and drag-and-drop directory support
- Refactor FetchImportAdapter to extract reusable `consumeSseStream` helper

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
diguaandCursor authored and digua committed 2026-05-22 10:20:39 +08:00
1 parent 8bb904fb22
commit 9421662a52
16 files changed
+462 -76

No files matched your search

+87
View File
@@ -70,6 +70,7 @@ import {
detectAllFormats,
getSupportedFormats,
scanMultiChatFile,
findEntryFileInDirectory,
} from '../../import'
import { getDefaultAssistantConfig, buildPiModel } from '../../ai/llm-config'
@@ -880,6 +881,92 @@ export function registerWebRoutes(
}
})
// ==================== Directory Import ====================
server.post('/_web/import-directory', async (request, reply) => {
const parts = (request as any).parts()
if (!parts) return reply.code(400).send({ error: 'No files uploaded' })
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-dir-import-'))
// Interleaved fields: relativePaths (field) and files (file) come in order
const relativePaths: string[] = []
const fileBuffers: { data: Buffer; filename: string }[] = []
try {
for await (const part of parts) {
if (part.type === 'field' && part.fieldname === 'relativePaths') {
relativePaths.push(String(part.value))
} else if (part.type === 'file') {
const chunks: Buffer[] = []
for await (const chunk of part.file) {
chunks.push(chunk)
}
fileBuffers.push({ data: Buffer.concat(chunks), filename: part.filename || '' })
}
}
// Write files preserving directory structure
for (let i = 0; i < fileBuffers.length; i++) {
let relPath = relativePaths[i] || fileBuffers[i].filename || `file_${i}`
// Strip top-level directory from webkitRelativePath (e.g. "export_dir/chunks/..." → "chunks/...")
const segments = relPath.split('/')
if (segments.length > 1) {
relPath = segments.slice(1).join('/')
}
const targetPath = path.join(tmpDir, relPath)
fs.mkdirSync(path.dirname(targetPath), { recursive: true })
fs.writeFileSync(targetPath, fileBuffers[i].data)
}
const entryPath = findEntryFileInDirectory(tmpDir)
if (!entryPath) {
return reply.code(400).send({ error: 'No recognizable import format found in directory' })
}
reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
})
function sendEvent(event: string, eventData: unknown) {
reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(eventData)}\n\n`)
}
const nativeBinding = resolveNativeBinding()
const result = await streamImport(dbManager, entryPath, {
nativeBinding,
onProgress: (p) => {
sendEvent('progress', p)
},
})
if (result.success) {
sendEvent('done', {
success: true,
sessionId: result.sessionId,
messageCount: result.diagnostics?.messagesWritten ?? 0,
memberCount: 0,
})
} else {
sendEvent('error', { success: false, error: result.error })
}
} catch (err) {
if (!reply.raw.headersSent) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) })
}
reply.raw.write(
`event: error\ndata: ${JSON.stringify({ success: false, error: err instanceof Error ? err.message : String(err) })}\n\n`
)
} finally {
reply.raw.end()
fs.rmSync(tmpDir, { recursive: true, force: true })
}
})
// ==================== Incremental Import ====================
server.post<{ Params: { id: string } }>('/_web/sessions/:id/import/incremental', async (request, reply) => {
+1
View File
@@ -15,6 +15,7 @@ export {
getFormatFeatureById,
getSupportedFormats,
scanMultiChatFile,
findEntryFileInDirectory,
} from './stream-import'
export type {
StreamImportProgress,
+2
View File
@@ -34,6 +34,7 @@ import {
getFormatFeatureById,
getSupportedFormats as parserGetSupportedFormats,
scanMultiChatFile as parserScanMultiChatFile,
findEntryFileInDirectory,
type FormatFeature,
type MultiChatInfo,
type ParseProgress,
@@ -216,6 +217,7 @@ export {
getFormatFeatureById,
parserGetSupportedFormats as getSupportedFormats,
parserScanMultiChatFile as scanMultiChatFile,
findEntryFileInDirectory,
}
export type { FormatFeature, MultiChatInfo, ParseProgress }
export type {
+40 -1
View File
@@ -7,7 +7,7 @@ import { getConversationCountsBySession } from '../ai/conversations'
import * as databaseCore from '../database/core'
import * as worker from '../worker/workerManager'
import * as parser from '../parser'
import { detectFormat, scanMultiChatFile, type ParseProgress } from '../parser'
import { detectFormat, findEntryFileInDirectory, scanMultiChatFile, type ParseProgress } from '../parser'
import type { IpcContext } from './types'
import { CURRENT_SCHEMA_VERSION, getPendingMigrationInfos } from '../database/migrations'
import { exportSessionToTempFile, cleanupTempExportFiles } from '../merger'
@@ -143,6 +143,45 @@ export function registerChatHandlers(ctx: IpcContext): void {
}
})
/**
* Import from a directory path: scan for entry file and import
*/
ipcMain.handle('chat:importDirectory', async (_, dirPath: string) => {
try {
const entryPath = findEntryFileInDirectory(dirPath)
if (!entryPath) {
return { success: false, error: 'No recognizable import format found in directory' }
}
win.webContents.send('chat:importProgress', {
stage: 'detecting',
progress: 5,
message: '',
})
const result = await worker.streamImport(entryPath, (progress: ParseProgress) => {
win.webContents.send('chat:importProgress', {
stage: progress.stage,
progress: progress.percentage,
message: progress.message,
bytesRead: progress.bytesRead,
totalBytes: progress.totalBytes,
messagesProcessed: progress.messagesProcessed,
})
})
if (result.success) {
return { success: true, sessionId: result.sessionId, diagnostics: result.diagnostics }
} else {
win.webContents.send('chat:importProgress', { stage: 'error', progress: 0, message: result.error })
return { success: false, error: result.error, diagnostics: result.diagnostics }
}
} catch (error) {
win.webContents.send('chat:importProgress', { stage: 'error', progress: 0, message: String(error) })
return { success: false, error: String(error) }
}
})
/**
* 检测文件格式(轻量级,仅返回格式 ID、名称和是否多聊天)
*/
+1
View File
@@ -16,6 +16,7 @@ export {
parseFileSync,
parseFileInfo,
streamParseFile,
findEntryFileInDirectory,
FormatSniffer,
createSniffer,
getFileSize,
+7
View File
@@ -60,6 +60,13 @@ export const chatApi = {
return ipcRenderer.invoke('chat:import', filePath)
},
/**
* Import from a directory path (scans for entry file)
*/
importDirectory: (dirPath: string): Promise<{ success: boolean; sessionId?: string; error?: string }> => {
return ipcRenderer.invoke('chat:importDirectory', dirPath)
},
/**
* 检测文件格式(轻量级)
*/
+37 -2
View File
@@ -3,6 +3,8 @@
* 三层架构:标准层、嗅探层、解析层
*/
import * as fs from 'fs'
import * as path from 'path'
import { FormatSniffer, createSniffer } from './sniffer'
import { formats } from './formats'
import { getFileSize } from './utils'
@@ -238,8 +240,6 @@ export async function parseFileInfo(
}
}
// 获取文件大小
const fs = await import('fs')
const fileSize = fs.statSync(filePath).size
return {
@@ -252,6 +252,41 @@ export async function parseFileInfo(
}
}
/**
* Scan a directory for a recognizable entry file (e.g. manifest.json for chunked formats).
* Uses existing format detection; no format-specific filenames are hardcoded here.
* @param dirPath directory path to scan
* @param maxDepth max depth to scan (default 1, only top-level files)
* @returns path to the entry file, or null if none found
*/
export function findEntryFileInDirectory(dirPath: string, maxDepth = 1): string | null {
function walk(current: string, depth: number): string | null {
if (depth > maxDepth) return null
let entries: fs.Dirent[]
try {
entries = fs.readdirSync(current, { withFileTypes: true })
} catch {
return null
}
for (const entry of entries) {
const full = path.join(current, entry.name)
if (entry.isFile() && entry.name.endsWith('.json')) {
const format = detectFormat(full)
if (format) return full
}
}
for (const entry of entries) {
if (entry.isDirectory()) {
const found = walk(path.join(current, entry.name), depth + 1)
if (found) return found
}
}
return null
}
return walk(dirPath, 0)
}
// ==================== 导出类型 ====================
export type {
+74 -5
View File
@@ -13,17 +13,22 @@ interface Props {
disabled?: boolean
/** 接受的文件扩展名,如 ['.json', '.txt'] */
accept?: string[]
/** 是否为目录选择模式(webkitdirectory) */
directory?: boolean
}
const props = withDefaults(defineProps<Props>(), {
multiple: false,
disabled: false,
accept: () => ['*'],
directory: false,
})
const emit = defineEmits<{
/** 选择文件后触发,返回文件列表和路径列表 */
files: [payload: { files: File[]; paths: string[] }]
/** 拖入文件夹时触发 (Electron: dirPath, Web: File[]) */
'directory-drop': [payload: { files: File[]; dirPath: string | null }]
}>()
// 拖拽状态
@@ -79,7 +84,7 @@ function handleDragLeave(e: DragEvent) {
}
// 处理拖拽放下
function handleDrop(e: DragEvent) {
async function handleDrop(e: DragEvent) {
e.preventDefault()
e.stopPropagation()
isDragOver.value = false
@@ -87,16 +92,26 @@ function handleDrop(e: DragEvent) {
if (props.disabled) return
const dataTransfer = e.dataTransfer
if (!dataTransfer?.files || dataTransfer.files.length === 0) return
if (!dataTransfer) return
// Check if a directory was dropped (via DataTransferItem API)
const items = dataTransfer.items
if (items?.length === 1) {
const entry = items[0].webkitGetAsEntry?.()
if (entry?.isDirectory) {
await handleDirectoryDrop(entry as FileSystemDirectoryEntry, dataTransfer.files)
return
}
}
if (dataTransfer.files.length === 0) return
let files = Array.from(dataTransfer.files)
// 如果不支持多选,只取第一个
if (!props.multiple) {
files = [files[0]]
}
// 过滤文件类型
if (!props.accept.includes('*')) {
files = files.filter((file) => {
const ext = '.' + file.name.split('.').pop()?.toLowerCase()
@@ -109,6 +124,59 @@ function handleDrop(e: DragEvent) {
}
}
async function handleDirectoryDrop(dirEntry: FileSystemDirectoryEntry, fileList: FileList) {
// Electron: try to get directory path
if (fileList.length > 0) {
try {
// @ts-expect-error Electron webUtils
const dirPath = window.electron?.webUtils?.getPathForFile?.(fileList[0])
if (dirPath) {
emit('directory-drop', { files: [], dirPath })
return
}
} catch {
// Not Electron
}
}
// Web: recursively read directory entries into File objects
const files = await readDirectoryEntries(dirEntry, dirEntry.name)
if (files.length > 0) {
emit('directory-drop', { files, dirPath: null })
}
}
async function readDirectoryEntries(dirEntry: FileSystemDirectoryEntry, basePath: string): Promise<File[]> {
const reader = dirEntry.createReader()
const files: File[] = []
const readBatch = (): Promise<FileSystemEntry[]> =>
new Promise((resolve, reject) => reader.readEntries(resolve, reject))
let entries: FileSystemEntry[]
do {
entries = await readBatch()
for (const entry of entries) {
if (entry.isFile) {
const file = await new Promise<File>((resolve, reject) => {
;(entry as FileSystemFileEntry).file(resolve, reject)
})
// Attach webkitRelativePath-like info via property
Object.defineProperty(file, 'webkitRelativePath', {
value: `${basePath}/${entry.name}`,
writable: false,
})
files.push(file)
} else if (entry.isDirectory) {
const subFiles = await readDirectoryEntries(entry as FileSystemDirectoryEntry, `${basePath}/${entry.name}`)
files.push(...subFiles)
}
}
} while (entries.length > 0)
return files
}
// 处理文件并发送事件
function processFiles(files: File[]) {
const paths: string[] = []
@@ -142,8 +210,9 @@ defineExpose({
ref="fileInputRef"
type="file"
:multiple="multiple"
:accept="acceptAttr"
:accept="directory ? undefined : acceptAttr"
class="hidden"
v-bind="directory ? { webkitdirectory: '', directory: '' } : {}"
@change="handleFileSelect"
/>
+2
View File
@@ -20,6 +20,8 @@
"clickHint": "Click to select or drag chat records to import",
"multipleHint": "Multiple files can be selected or dragged at once",
"selectFiles": "Select chat record files",
"selectFolder": "Select export folder",
"importFolder": "Import Folder",
"chatRecords": "Chat Records",
"allFiles": "All Files",
"progress": {
+2
View File
@@ -20,6 +20,8 @@
"clickHint": "クリックして選択、またはチャット記録をドラッグ&ドロップでインポート",
"multipleHint": "複数ファイルの同時選択・ドラッグに対応しています",
"selectFiles": "チャット記録ファイルを選択",
"selectFolder": "エクスポートフォルダを選択",
"importFolder": "フォルダをインポート",
"chatRecords": "チャット記録",
"allFiles": "すべてのファイル",
"progress": {
+2
View File
@@ -20,6 +20,8 @@
"clickHint": "点击选择或拖拽聊天记录导入",
"multipleHint": "支持同时选择或拖拽多个文件",
"selectFiles": "选择聊天记录文件",
"selectFolder": "选择导出文件夹",
"importFolder": "导入文件夹",
"chatRecords": "聊天记录",
"allFiles": "所有文件",
"progress": {
+2
View File
@@ -20,6 +20,8 @@
"clickHint": "點一下選擇,或直接拖曳聊天紀錄匯入",
"multipleHint": "支援同時選擇或拖曳多個檔案",
"selectFiles": "選擇聊天紀錄檔案",
"selectFolder": "選擇匯出資料夾",
"importFolder": "匯入資料夾",
"chatRecords": "聊天紀錄",
"allFiles": "所有檔案",
"progress": {
+92 -3
View File
@@ -119,12 +119,20 @@ async function checkImportLog() {
const dropZoneRef = ref<InstanceType<typeof FileDropZone> | null>(null)
// 处理文件选择(点击选择)- 支持多选
// 文件夹导入模式
const folderImportEnabled = ref(false)
// 处理文件选择(点击选择)
async function handleClickImport() {
importError.value = null
hasImportLog.value = false
importDiagnostics.value = null
if (folderImportEnabled.value) {
handleClickImportDirectory()
return
}
if (IS_ELECTRON) {
const result = await usePlatformService().showOpenDialog({
title: t('home.import.selectFiles'),
@@ -228,6 +236,73 @@ async function importSingleWebFile(file: File, options?: { formatId?: string; ch
}
}
// 目录导入 ref
const dirDropZoneRef = ref<InstanceType<typeof FileDropZone> | null>(null)
async function handleClickImportDirectory() {
importError.value = null
if (IS_ELECTRON) {
const result = await usePlatformService().showOpenDialog({
title: t('home.import.selectFolder'),
properties: ['openDirectory'],
})
if (result.canceled || result.filePaths.length === 0) return
await importDirectory(result.filePaths[0])
} else {
dirDropZoneRef.value?.openFileDialog()
}
}
async function handleDirectoryDrop({ files }: { files: File[]; paths: string[] }) {
if (files.length === 0) return
importError.value = null
await importDirectory(files)
}
async function handleDirectoryDropEvent({ files, dirPath }: { files: File[]; dirPath: string | null }) {
importError.value = null
hasImportLog.value = false
importDiagnostics.value = null
if (dirPath) {
await importDirectory(dirPath)
} else if (files.length > 0) {
await importDirectory(files)
}
}
async function importDirectory(source: File[] | string) {
isImporting.value = true
importProgress.value = { stage: 'detecting', progress: 0, message: '' }
try {
const result = await useImportService().importDirectory(source, undefined, (p) => {
if (p.stage === 'done') return
importProgress.value = p
})
if (importProgress.value) {
importProgress.value.progress = 100
}
await new Promise((resolve) => setTimeout(resolve, 300))
if (result.success && result.sessionId) {
await sessionStore.loadSessions()
sessionStore.selectSession(result.sessionId)
await navigateToSession(result.sessionId)
} else {
importError.value = translateError(result.error || 'error.import_failed')
}
} catch (error) {
importError.value = String(error)
} finally {
isImporting.value = false
setTimeout(() => {
importProgress.value = null
}, 500)
}
}
// 统一处理文件路径(单文件或多文件)- Electron only
async function processFilePaths(paths: string[]) {
// 单文件 或 未启用合并导入 - 使用原有逻辑
@@ -769,6 +844,7 @@ const getMergeFileProgressText = (file: MergeFileInfo) =>
:multiple="true"
class="w-full max-w-4xl"
@files="handleFileDrop"
@directory-drop="handleDirectoryDropEvent"
>
<template #default="{ isDragOver }">
<div
@@ -819,8 +895,21 @@ const getMergeFileProgressText = (file: MergeFileInfo) =>
</template>
</FileDropZone>
<!-- 合并导入选项 -->
<div v-if="!isAnyImporting && !batchImportResult" class="flex items-center justify-center">
<!-- 隐藏的目录选择 input -->
<FileDropZone ref="dirDropZoneRef" :directory="true" class="hidden" @files="handleDirectoryDrop" />
<!-- 导入选项 -->
<div v-if="!isAnyImporting && !batchImportResult" class="flex flex-wrap items-center justify-center gap-4">
<!-- 导入文件夹模式 -->
<UCheckbox
v-model="folderImportEnabled"
:label="t('home.import.importFolder')"
input-class="h-4 w-4"
size="sm"
label-class="text-sm font-medium text-gray-600 dark:text-gray-300"
/>
<!-- 合并导入 -->
<div class="flex items-center gap-2">
<UCheckbox
v-model="mergeImportEnabled"
+39
View File
@@ -125,4 +125,43 @@ export class ElectronImportAdapter implements ImportAdapter {
unlisten()
}
}
async importDirectory(
source: File[] | string,
_options?: ImportOptions,
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult> {
if (typeof source !== 'string') {
return { success: false, error: 'Expected directory path in Electron mode' }
}
return new Promise((resolve) => {
const unlisten = window.chatApi.onImportProgress((progress: any) => {
onProgress?.({
stage: progress.stage || 'parsing',
progress: progress.percentage || progress.progress || 0,
message: progress.message || '',
bytesRead: progress.bytesRead,
totalBytes: progress.totalBytes,
messagesProcessed: progress.messagesProcessed,
})
})
window.chatApi
.importDirectory(source)
.then((result: any) => {
unlisten()
resolve({
success: result.success,
sessionId: result.sessionId,
error: result.error,
diagnostics: result.diagnostics,
})
})
.catch((err: Error) => {
unlisten()
resolve({ success: false, error: err.message })
})
})
}
}
+67 -65
View File
@@ -20,6 +20,41 @@ import { get } from '../utils/http'
const BASE = '/_web'
async function consumeSseStream<T>(res: Response, fallback: T, onProgress?: (p: ImportProgress) => void): Promise<T> {
const reader = res.body?.getReader()
if (!reader) return fallback
const decoder = new TextDecoder()
let buffer = ''
let result: T = fallback
while (true) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() || ''
let eventType = ''
for (const line of lines) {
if (line.startsWith('event: ')) {
eventType = line.slice(7).trim()
} else if (line.startsWith('data: ')) {
const data = JSON.parse(line.slice(6))
if (eventType === 'progress') {
onProgress?.(data as ImportProgress)
} else if (eventType === 'done' || eventType === 'error') {
result = data as T
}
eventType = ''
}
}
}
return result
}
export class FetchImportAdapter implements ImportAdapter {
async importFile(
file: File | string,
@@ -42,40 +77,7 @@ export class FetchImportAdapter implements ImportAdapter {
return { success: false, error: `HTTP ${res.status}: ${text}` }
}
const reader = res.body?.getReader()
if (!reader) return { success: false, error: 'No response body' }
const decoder = new TextDecoder()
let buffer = ''
let result: ImportResult = { success: false, error: 'Unknown error' }
while (true) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() || ''
let eventType = ''
for (const line of lines) {
if (line.startsWith('event: ')) {
eventType = line.slice(7).trim()
} else if (line.startsWith('data: ')) {
const data = JSON.parse(line.slice(6))
if (eventType === 'progress') {
onProgress?.(data as ImportProgress)
} else if (eventType === 'done') {
result = data as ImportResult
} else if (eventType === 'error') {
result = data as ImportResult
}
eventType = ''
}
}
}
return result
return consumeSseStream<ImportResult>(res, { success: false, error: 'Unknown error' }, onProgress)
}
async detectFormat(file: File | string): Promise<FormatInfo | null> {
@@ -194,39 +196,39 @@ export class FetchImportAdapter implements ImportAdapter {
return { success: false, newMessageCount: 0, error: `HTTP ${res.status}: ${text}` }
}
const reader = res.body?.getReader()
if (!reader) return { success: false, newMessageCount: 0, error: 'No response body' }
return consumeSseStream<IncrementalImportResult>(
res,
{ success: false, newMessageCount: 0, error: 'Unknown error' },
onProgress
)
}
const decoder = new TextDecoder()
let buffer = ''
let result: IncrementalImportResult = { success: false, newMessageCount: 0, error: 'Unknown error' }
while (true) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
const lines = buffer.split('\n')
buffer = lines.pop() || ''
let eventType = ''
for (const line of lines) {
if (line.startsWith('event: ')) {
eventType = line.slice(7).trim()
} else if (line.startsWith('data: ')) {
const data = JSON.parse(line.slice(6))
if (eventType === 'progress') {
onProgress?.(data as ImportProgress)
} else if (eventType === 'done') {
result = data as IncrementalImportResult
} else if (eventType === 'error') {
result = data as IncrementalImportResult
}
eventType = ''
}
}
async importDirectory(
source: File[] | string,
_options?: ImportOptions,
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult> {
if (typeof source === 'string') {
return { success: false, error: 'Directory path import is not supported in Web mode' }
}
return result
if (source.length === 0) {
return { success: false, error: 'No files in directory' }
}
const form = new FormData()
for (const file of source) {
form.append('files', file)
form.append('relativePaths', file.webkitRelativePath || file.name)
}
const res = await fetch(`${BASE}/import-directory`, { method: 'POST', body: form })
if (!res.ok) {
const text = await res.text()
return { success: false, error: `HTTP ${res.status}: ${text}` }
}
return consumeSseStream<ImportResult>(res, { success: false, error: 'Unknown error' }, onProgress)
}
}
+7
View File
@@ -118,6 +118,13 @@ export interface ImportAdapter {
file: File | string,
onProgress?: (p: ImportProgress) => void
): Promise<IncrementalImportResult>
/** 导入目录(多文件格式如 chunked-jsonl)。Electron 传目录路径,Web 传 File[](含 webkitRelativePath) */
importDirectory(
source: File[] | string,
options?: ImportOptions,
onProgress?: (p: ImportProgress) => void
): Promise<ImportResult>
}
export type { ImportProgress }