mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-26 02:04:37 +08:00
fix(import): stabilize large and multi-file imports
This commit is contained in:
1 parent
4518a3aaa9
commit
c0942a72f0
6 files changed
+201
-17
No files matched your search
@@ -300,9 +300,7 @@ async function processWebFiles(files: File[]) {
|
||||
importError.value = translateError('error.archive_single_file_required')
|
||||
return
|
||||
}
|
||||
for (const file of files) {
|
||||
await importSingleWebFile(file)
|
||||
}
|
||||
await sessionStore.importFiles(files)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -465,7 +463,7 @@ async function processFilePaths(paths: string[]) {
|
||||
}
|
||||
} else {
|
||||
// 多文件批量导入(未启用合并)
|
||||
await sessionStore.importFilesFromPaths(paths)
|
||||
await sessionStore.importFiles(paths)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { setImmediate as waitForImmediate } from 'node:timers/promises'
|
||||
import test from 'node:test'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { registerAdapter } from '@/services/registry'
|
||||
import type { DataAdapter, ImportAdapter, SessionIndexAdapter } from '@/services'
|
||||
|
||||
function createMemoryStorage(): Storage {
|
||||
const values = new Map<string, string>()
|
||||
return {
|
||||
get length() {
|
||||
return values.size
|
||||
},
|
||||
clear: () => values.clear(),
|
||||
getItem: (key) => values.get(key) ?? null,
|
||||
key: (index) => [...values.keys()][index] ?? null,
|
||||
removeItem: (key) => values.delete(key),
|
||||
setItem: (key, value) => values.set(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
test('batch import accepts browser File objects and processes them serially', async () => {
|
||||
Object.defineProperties(globalThis, {
|
||||
localStorage: { configurable: true, value: createMemoryStorage() },
|
||||
sessionStorage: { configurable: true, value: createMemoryStorage() },
|
||||
})
|
||||
|
||||
const importCalls: File[] = []
|
||||
let activeImports = 0
|
||||
let maxActiveImports = 0
|
||||
|
||||
registerAdapter('import', {
|
||||
async importFile(source: File | string) {
|
||||
assert.ok(source instanceof File)
|
||||
importCalls.push(source)
|
||||
activeImports++
|
||||
maxActiveImports = Math.max(maxActiveImports, activeImports)
|
||||
await waitForImmediate()
|
||||
activeImports--
|
||||
return {
|
||||
success: true,
|
||||
sessionId: `session-${importCalls.length}`,
|
||||
importMode: 'incremental' as const,
|
||||
newMessageCount: 1,
|
||||
duplicateCount: 0,
|
||||
}
|
||||
},
|
||||
} as ImportAdapter)
|
||||
registerAdapter('data', {
|
||||
getSessions: async () => [],
|
||||
tryApplyOwnerProfile: async () => ({ applied: false }),
|
||||
} as unknown as DataAdapter)
|
||||
registerAdapter('session-index', {
|
||||
generate: async () => undefined,
|
||||
} as unknown as SessionIndexAdapter)
|
||||
|
||||
setActivePinia(createPinia())
|
||||
const { useSessionStore } = await import('./session')
|
||||
const store = useSessionStore()
|
||||
const files = [new File(['a'], 'first.json'), new File(['b'], 'second.jsonl')]
|
||||
|
||||
const result = await store.importFiles(files)
|
||||
|
||||
assert.deepEqual(
|
||||
importCalls.map((file) => file.name),
|
||||
['first.json', 'second.jsonl']
|
||||
)
|
||||
assert.equal(maxActiveImports, 1)
|
||||
assert.equal(result.success, 2)
|
||||
assert.equal(result.failed, 0)
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => ({ name: file.name, status: file.status })),
|
||||
[
|
||||
{ name: 'first.json', status: 'success' },
|
||||
{ name: 'second.jsonl', status: 'success' },
|
||||
]
|
||||
)
|
||||
})
|
||||
+23
-7
@@ -340,20 +340,36 @@ export const useSessionStore = defineStore(
|
||||
/**
|
||||
* 批量导入多个文件(串行执行)
|
||||
*/
|
||||
async function importFilesFromPaths(filePaths: string[]): Promise<BatchImportResult> {
|
||||
if (filePaths.length === 0) {
|
||||
async function importFiles(sources: Array<File | string>): Promise<BatchImportResult> {
|
||||
if (sources.length === 0) {
|
||||
return { total: 0, success: 0, failed: 0, cancelled: 0, files: [] }
|
||||
}
|
||||
|
||||
const importSources = sources.map((source) => {
|
||||
if (typeof source === 'string') {
|
||||
return {
|
||||
source,
|
||||
path: source,
|
||||
name: source.split('/').pop() || source.split('\\').pop() || source,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source,
|
||||
path: source.webkitRelativePath || source.name,
|
||||
name: source.name,
|
||||
}
|
||||
})
|
||||
|
||||
// 初始化批量导入状态
|
||||
isBatchImporting.value = true
|
||||
batchImportCancelled.value = false
|
||||
batchImportResult.value = null
|
||||
|
||||
// 初始化文件列表
|
||||
batchFiles.value = filePaths.map((path) => ({
|
||||
batchFiles.value = importSources.map(({ path, name }) => ({
|
||||
path,
|
||||
name: path.split('/').pop() || path.split('\\').pop() || path,
|
||||
name,
|
||||
status: 'pending' as BatchFileStatus,
|
||||
}))
|
||||
|
||||
@@ -422,7 +438,7 @@ export const useSessionStore = defineStore(
|
||||
isProcessing = false
|
||||
}
|
||||
|
||||
const importResult = await useImportService().importFile(file.path, undefined, (progress) => {
|
||||
const importResult = await useImportService().importFile(importSources[i].source, undefined, (progress) => {
|
||||
if (progress.stage === 'done') return
|
||||
queue.push(progress)
|
||||
processQueue()
|
||||
@@ -503,7 +519,7 @@ export const useSessionStore = defineStore(
|
||||
|
||||
// 生成结果
|
||||
const result: BatchImportResult = {
|
||||
total: filePaths.length,
|
||||
total: sources.length,
|
||||
success: successCount,
|
||||
failed: failedCount,
|
||||
cancelled: cancelledCount,
|
||||
@@ -880,7 +896,7 @@ export const useSessionStore = defineStore(
|
||||
batchFiles,
|
||||
batchImportCancelled,
|
||||
batchImportResult,
|
||||
importFilesFromPaths,
|
||||
importFiles,
|
||||
cancelBatchImport,
|
||||
clearBatchImportResult,
|
||||
// 合并导入
|
||||
|
||||
Reference in new issue
Block a user