mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
fix(import): report accurate post-parse progress stages in real time
The UI stayed on "parsing messages" during the whole post-parse phase: the importer emitted an 'importing' stage the frontend had no text for, and on CLI Web the synchronous DB writes / index / FTS build starved the event loop so buffered SSE progress events only arrived after the import finished. - Replace 'importing' with 'saving' (message writes) and a new 'indexing' stage (indexes / FTS / session index) in ParseProgress, with UI texts for all four locales. - Yield a macrotask between native parser batches and before heavy synchronous steps so SSE progress events flush while importing. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
f7150e2cff
commit
c265925e3e
10 files changed
+36
-8
No files matched your search
@@ -108,11 +108,14 @@ export async function streamImport(
|
||||
stage = 'parsing'
|
||||
pct = Math.min(Math.round(progress.percentage * 0.7), 70)
|
||||
break
|
||||
case 'importing':
|
||||
case 'saving':
|
||||
stage = 'saving'
|
||||
pct = 80
|
||||
break
|
||||
case 'indexing':
|
||||
stage = 'indexing'
|
||||
pct = 90
|
||||
break
|
||||
case 'done':
|
||||
stage = 'done'
|
||||
pct = 100
|
||||
|
||||
@@ -84,6 +84,13 @@ export interface StreamImportDeps {
|
||||
const BATCH_COMMIT_SIZE = 50000
|
||||
const CHECKPOINT_INTERVAL = 200000
|
||||
|
||||
/**
|
||||
* Let the event loop process pending I/O before a long synchronous step.
|
||||
* Without this, progress events written to an SSE socket stay buffered
|
||||
* until the blocking work (index/FTS build) finishes.
|
||||
*/
|
||||
const yieldToEventLoop = () => new Promise<void>((resolve) => setImmediate(resolve))
|
||||
|
||||
function defaultGenerateSessionId(): string {
|
||||
const ts = Date.now()
|
||||
const rand = Math.random().toString(36).substring(2, 8)
|
||||
@@ -268,7 +275,7 @@ async function streamImportSingle(
|
||||
}
|
||||
|
||||
onProgress({
|
||||
stage: 'importing',
|
||||
stage: 'saving',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: totalMessageCount,
|
||||
@@ -466,13 +473,14 @@ async function streamImportSingle(
|
||||
|
||||
// Flush nickname history in batch
|
||||
onProgress({
|
||||
stage: 'importing',
|
||||
stage: 'saving',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: totalMessageCount,
|
||||
percentage: 100,
|
||||
message: '',
|
||||
})
|
||||
await yieldToEventLoop()
|
||||
logger?.perf('Writing nickname history', totalMessageCount)
|
||||
|
||||
db.exec('BEGIN TRANSACTION')
|
||||
@@ -493,13 +501,14 @@ async function streamImportSingle(
|
||||
|
||||
// Create indexes (deferred for performance)
|
||||
onProgress({
|
||||
stage: 'importing',
|
||||
stage: 'indexing',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: totalMessageCount,
|
||||
percentage: 100,
|
||||
message: '',
|
||||
})
|
||||
await yieldToEventLoop()
|
||||
logger?.perf('Creating indexes', totalMessageCount)
|
||||
db.exec(CHAT_DB_INDEXES)
|
||||
logger?.perf('Indexes created', totalMessageCount)
|
||||
@@ -512,15 +521,16 @@ async function streamImportSingle(
|
||||
logger?.error('FTS index build failed (non-fatal)', ftsError instanceof Error ? ftsError : undefined)
|
||||
}
|
||||
|
||||
// Final WAL checkpoint
|
||||
// Final WAL checkpoint + session index + post-import hook
|
||||
onProgress({
|
||||
stage: 'importing',
|
||||
stage: 'indexing',
|
||||
bytesRead: 0,
|
||||
totalBytes: 0,
|
||||
messagesProcessed: totalMessageCount,
|
||||
percentage: 100,
|
||||
message: '',
|
||||
})
|
||||
await yieldToEventLoop()
|
||||
doCheckpoint()
|
||||
logger?.perf('WAL checkpoint done', totalMessageCount)
|
||||
|
||||
|
||||
@@ -117,6 +117,10 @@ async function* parseChatLabNative(
|
||||
if (!batch) break
|
||||
messagesProcessed += batch.length
|
||||
yield { type: 'messages', data: batch.map(toParsedMessage) }
|
||||
// The consumer typically does synchronous DB writes per batch; yield a
|
||||
// macrotask so the event loop can flush pending I/O (e.g. SSE progress
|
||||
// events) instead of being starved until all batches are delivered.
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
}
|
||||
|
||||
const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '')
|
||||
|
||||
@@ -99,6 +99,10 @@ async function* parseWeFlowNative(
|
||||
if (!batch) break
|
||||
messagesProcessed += batch.length
|
||||
yield { type: 'messages', data: batch.map(toParsedMessage) }
|
||||
// The consumer typically does synchronous DB writes per batch; yield a
|
||||
// macrotask so the event loop can flush pending I/O (e.g. SSE progress
|
||||
// events) instead of being starved until all batches are delivered.
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
}
|
||||
|
||||
const doneProgress = createProgress('done', totalBytes, totalBytes, messagesProcessed, '')
|
||||
|
||||
@@ -24,8 +24,9 @@ export interface ParsedMeta {
|
||||
*/
|
||||
export interface ParseProgress {
|
||||
/** 阶段 */
|
||||
// 导入流程会复用解析进度结构,因此这里补充导入阶段枚举。
|
||||
stage: 'detecting' | 'parsing' | 'importing' | 'saving' | 'done' | 'error'
|
||||
// 导入流程会复用解析进度结构,因此这里补充导入阶段枚举:
|
||||
// saving = 消息批量写库,indexing = 建索引/FTS/会话分段等后处理。
|
||||
stage: 'detecting' | 'parsing' | 'saving' | 'indexing' | 'done' | 'error'
|
||||
/** 已读取字节数 */
|
||||
bytesRead: number
|
||||
/** 文件总字节数 */
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"reading": "Reading file...",
|
||||
"parsing": "Parsing messages...",
|
||||
"saving": "Saving to database...",
|
||||
"indexing": "Building search index...",
|
||||
"done": "Import complete",
|
||||
"error": "Import interrupted"
|
||||
},
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"reading": "ファイルを読み込み中...",
|
||||
"parsing": "メッセージを解析中...",
|
||||
"saving": "データベースに書き込み中...",
|
||||
"indexing": "検索インデックスを構築中...",
|
||||
"done": "インポート完了",
|
||||
"error": "インポート中断"
|
||||
},
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"reading": "正在读取文件...",
|
||||
"parsing": "正在解析消息...",
|
||||
"saving": "正在写入数据库...",
|
||||
"indexing": "正在构建搜索索引...",
|
||||
"done": "导入完成",
|
||||
"error": "导入中断"
|
||||
},
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"reading": "正在讀取檔案...",
|
||||
"parsing": "正在解析訊息...",
|
||||
"saving": "正在寫入資料庫...",
|
||||
"indexing": "正在建立搜尋索引...",
|
||||
"done": "匯入完成",
|
||||
"error": "匯入中斷"
|
||||
},
|
||||
|
||||
@@ -741,6 +741,8 @@ function getProgressText(): string {
|
||||
return t('home.import.progress.parsing')
|
||||
case 'saving':
|
||||
return t('home.import.progress.saving')
|
||||
case 'indexing':
|
||||
return t('home.import.progress.indexing')
|
||||
case 'done':
|
||||
return t('home.import.progress.done')
|
||||
case 'error':
|
||||
|
||||
Reference in new issue
Block a user