mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
fix(import): preserve ChatLab owner and aliases
Thread ownerId and member aliases through the TS and native parsers. Persist aliases across full and incremental imports so exported ChatLab JSON retains user identity metadata.
This commit is contained in:
1 parent
37c446c751
commit
a9a1557867
13 files changed
+161
-15
No files matched your search
@@ -39,6 +39,27 @@ function writeChatLabJsonl(filePath: string): void {
|
||||
fs.writeFileSync(filePath, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`, 'utf8')
|
||||
}
|
||||
|
||||
function writeChatLabJson(filePath: string): void {
|
||||
fs.writeFileSync(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
chatlab: { version: '0.0.2', exportedAt: 1780330900 },
|
||||
meta: { name: 'ChatLab Export', platform: 'wechat', type: 'private' },
|
||||
members: [{ platformId: 'wxid_alice', accountName: 'Alice', aliases: ['Ally'] }],
|
||||
messages: [
|
||||
{
|
||||
sender: 'wxid_alice',
|
||||
accountName: 'Alice',
|
||||
timestamp: 1780330832,
|
||||
type: 0,
|
||||
content: 'hello from ChatLab',
|
||||
},
|
||||
],
|
||||
}),
|
||||
'utf8'
|
||||
)
|
||||
}
|
||||
|
||||
function seedSessionDb(dbPath: string): void {
|
||||
const db = openBetterSqliteDatabase(dbPath, { nativeBinding })
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
@@ -89,3 +110,24 @@ test('imports ChatLab JSONL messages with numeric string timestamps consistently
|
||||
content: 'hello from CipherTalk',
|
||||
})
|
||||
})
|
||||
|
||||
test('preserves ChatLab JSON member aliases during incremental import', async (t) => {
|
||||
const tempDir = makeTempDir()
|
||||
t.after(() => fs.rmSync(tempDir, { recursive: true, force: true }))
|
||||
|
||||
const dbPath = path.join(tempDir, 'session.db')
|
||||
const filePath = path.join(tempDir, 'chatlab.json')
|
||||
seedSessionDb(dbPath)
|
||||
writeChatLabJson(filePath)
|
||||
|
||||
const result = await incrementalImport('session', filePath, createDeps(dbPath))
|
||||
assert.equal(result.success, true)
|
||||
|
||||
const db = openBetterSqliteDatabase(dbPath, { readonly: true, nativeBinding })
|
||||
const row = db.prepare("SELECT aliases FROM member WHERE platform_id = 'wxid_alice'").get() as
|
||||
| { aliases: string }
|
||||
| undefined
|
||||
db.close()
|
||||
|
||||
assert.deepEqual(JSON.parse(row?.aliases ?? '[]'), ['Ally'])
|
||||
})
|
||||
@@ -204,11 +204,12 @@ export async function incrementalImport(
|
||||
}
|
||||
|
||||
const upsertMember = db.prepare(`
|
||||
INSERT INTO member (platform_id, account_name, group_nickname, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
INSERT INTO member (platform_id, account_name, group_nickname, aliases, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(platform_id) DO UPDATE SET
|
||||
account_name = COALESCE(NULLIF(excluded.account_name, ''), account_name),
|
||||
group_nickname = COALESCE(NULLIF(excluded.group_nickname, ''), group_nickname),
|
||||
aliases = CASE WHEN excluded.aliases != '[]' THEN excluded.aliases ELSE aliases END,
|
||||
avatar = COALESCE(NULLIF(excluded.avatar, ''), avatar),
|
||||
roles = CASE WHEN excluded.roles != '[]' THEN excluded.roles ELSE roles END
|
||||
`)
|
||||
@@ -282,6 +283,7 @@ export async function incrementalImport(
|
||||
m.platformId,
|
||||
m.accountName || null,
|
||||
m.groupNickname || null,
|
||||
m.aliases ? JSON.stringify(m.aliases) : '[]',
|
||||
m.avatar || null,
|
||||
m.roles ? JSON.stringify(m.roles) : '[]'
|
||||
)
|
||||
|
||||
@@ -212,11 +212,12 @@ async function streamImportSingle(
|
||||
`INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
const insertMember = db.prepare(
|
||||
`INSERT INTO member (platform_id, account_name, group_nickname, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`INSERT INTO member (platform_id, account_name, group_nickname, aliases, avatar, roles)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(platform_id) DO UPDATE SET
|
||||
account_name = COALESCE(NULLIF(excluded.account_name, ''), account_name),
|
||||
group_nickname = COALESCE(NULLIF(excluded.group_nickname, ''), group_nickname),
|
||||
aliases = CASE WHEN excluded.aliases != '[]' THEN excluded.aliases ELSE aliases END,
|
||||
avatar = COALESCE(NULLIF(excluded.avatar, ''), avatar),
|
||||
roles = CASE WHEN excluded.roles != '[]' THEN excluded.roles ELSE roles END`
|
||||
)
|
||||
@@ -354,6 +355,7 @@ async function streamImportSingle(
|
||||
member.platformId,
|
||||
member.accountName || null,
|
||||
member.groupNickname || null,
|
||||
member.aliases ? JSON.stringify(member.aliases) : '[]',
|
||||
member.avatar || null,
|
||||
member.roles ? JSON.stringify(member.roles) : '[]'
|
||||
)
|
||||
@@ -401,6 +403,7 @@ async function streamImportSingle(
|
||||
msg.senderPlatformId,
|
||||
msg.senderAccountName || null,
|
||||
msg.senderGroupNickname || null,
|
||||
'[]',
|
||||
null,
|
||||
'[]'
|
||||
)
|
||||
|
||||
@@ -62,7 +62,7 @@ export function writeParseResultToDb(
|
||||
)
|
||||
|
||||
const insertMember = db.prepare(
|
||||
`INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar, roles) VALUES (?, ?, ?, ?, ?)`
|
||||
`INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, aliases, avatar, roles) VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
const getMemberId = db.prepare(`SELECT id FROM member WHERE platform_id = ?`)
|
||||
|
||||
@@ -73,6 +73,7 @@ export function writeParseResultToDb(
|
||||
member.platformId,
|
||||
member.accountName || null,
|
||||
member.groupNickname || null,
|
||||
member.aliases ? JSON.stringify(member.aliases) : '[]',
|
||||
member.avatar || null,
|
||||
member.roles ? JSON.stringify(member.roles) : '[]'
|
||||
)
|
||||
|
||||
Vendored
+1
@@ -29,6 +29,7 @@ export interface NativeMember {
|
||||
platformId: string
|
||||
accountName: string
|
||||
groupNickname?: string
|
||||
aliases?: Array<string>
|
||||
avatar?: string
|
||||
roles?: Array<NativeMemberRole>
|
||||
}
|
||||
|
||||
@@ -4,10 +4,8 @@
|
||||
//! Contract: this kernel is strict — any field that deviates from the ChatLab
|
||||
//! format spec (wrong type, unexpected null) returns an error, and the TS
|
||||
//! wrapper falls back to the pure-TS parser which replicates all passthrough
|
||||
//! quirks. Known intentional improvements over the TS reference (only reachable
|
||||
//! on degenerate inputs): meta/members are parsed structurally instead of via
|
||||
//! the TS 200KB-head regex heuristic, so files whose meta exceeds the head
|
||||
//! window or whose members contain nested arrays keep their real data here.
|
||||
//! quirks. Meta and members are parsed structurally while preserving the same
|
||||
//! output contract as the TS parser.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -30,6 +28,7 @@ struct ChatlabMeta {
|
||||
platform: String,
|
||||
group_id: Option<String>,
|
||||
group_avatar: Option<String>,
|
||||
owner_id: Option<String>,
|
||||
/// true: members came from the top-level `members` array (full shape);
|
||||
/// false: collected from messages (id/name/nickname only), as in TS.
|
||||
members_from_head: bool,
|
||||
@@ -87,6 +86,7 @@ struct MetaOut {
|
||||
chat_type: String,
|
||||
group_id: Option<String>,
|
||||
group_avatar: Option<String>,
|
||||
owner_id: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_meta(meta_raw: Option<&[u8]>, file_path: &str) -> ScanResult<MetaOut> {
|
||||
@@ -138,6 +138,7 @@ fn parse_meta(meta_raw: Option<&[u8]>, file_path: &str) -> ScanResult<MetaOut> {
|
||||
}
|
||||
let group_id = optional_str(&meta, "groupId", "meta")?;
|
||||
let group_avatar = optional_str(&meta, "groupAvatar", "meta")?;
|
||||
let owner_id = optional_str(&meta, "ownerId", "meta")?;
|
||||
|
||||
// Filename fallback fires whenever the name resolved to the default,
|
||||
// mirroring `if (meta.name === '未知群聊')` in the TS parser.
|
||||
@@ -151,6 +152,7 @@ fn parse_meta(meta_raw: Option<&[u8]>, file_path: &str) -> ScanResult<MetaOut> {
|
||||
chat_type,
|
||||
group_id,
|
||||
group_avatar,
|
||||
owner_id,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -161,6 +163,26 @@ fn parse_member(element: &[u8]) -> ScanResult<NativeMember> {
|
||||
.as_object()
|
||||
.ok_or_else(|| strict("member must be an object"))?;
|
||||
|
||||
let aliases = match obj.get("aliases") {
|
||||
None => None,
|
||||
Some(Value::Array(items)) => Some(
|
||||
items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
item.as_str()
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| strict("member alias must be a string"))
|
||||
})
|
||||
.collect::<ScanResult<Vec<_>>>()?,
|
||||
),
|
||||
other => {
|
||||
return Err(strict(format!(
|
||||
"member.aliases must be an array when present, got {}",
|
||||
type_name(other)
|
||||
)))
|
||||
}
|
||||
};
|
||||
|
||||
let roles = match obj.get("roles") {
|
||||
None => None,
|
||||
Some(Value::Array(items)) => {
|
||||
@@ -193,6 +215,7 @@ fn parse_member(element: &[u8]) -> ScanResult<NativeMember> {
|
||||
platform_id: require_str(obj, "platformId", "member")?,
|
||||
account_name: require_str(obj, "accountName", "member")?,
|
||||
group_nickname: optional_str(obj, "groupNickname", "member")?,
|
||||
aliases,
|
||||
avatar: optional_str(obj, "avatar", "member")?,
|
||||
roles,
|
||||
})
|
||||
@@ -342,6 +365,7 @@ pub fn parse_chatlab(
|
||||
platform_id,
|
||||
account_name,
|
||||
group_nickname,
|
||||
aliases: None,
|
||||
avatar: None,
|
||||
roles: None,
|
||||
})
|
||||
@@ -354,6 +378,7 @@ pub fn parse_chatlab(
|
||||
platform: meta.platform,
|
||||
group_id: meta.group_id,
|
||||
group_avatar: meta.group_avatar,
|
||||
owner_id: meta.owner_id,
|
||||
members_from_head,
|
||||
})
|
||||
.map_err(|err| strict(format!("meta serialization failed: {err}")))?;
|
||||
@@ -385,9 +410,9 @@ mod tests {
|
||||
fn parses_full_document() {
|
||||
let doc = r#"{
|
||||
"chatlab": {"version": "0.0.2", "exportedAt": 1700000000},
|
||||
"meta": {"name": "测试群", "platform": "weixin", "type": "group", "groupId": "g1@chatroom", "groupAvatar": "https://x/y.jpg"},
|
||||
"meta": {"name": "测试群", "platform": "weixin", "type": "group", "groupId": "g1@chatroom", "groupAvatar": "https://x/y.jpg", "ownerId": "u1"},
|
||||
"members": [
|
||||
{"platformId": "u1", "accountName": "Alice", "groupNickname": "小A", "avatar": "https://a", "roles": [{"id": "owner"}, {"id": "admin", "name": "管理"}]},
|
||||
{"platformId": "u1", "accountName": "Alice", "groupNickname": "小A", "aliases": ["Ally"], "avatar": "https://a", "roles": [{"id": "owner"}, {"id": "admin", "name": "管理"}]},
|
||||
{"platformId": "u2", "accountName": "Bob"}
|
||||
],
|
||||
"messages": [
|
||||
@@ -402,8 +427,13 @@ mod tests {
|
||||
assert_eq!(m["platform"], "weixin");
|
||||
assert_eq!(m["chatType"], "group");
|
||||
assert_eq!(m["groupId"], "g1@chatroom");
|
||||
assert_eq!(m["ownerId"], "u1");
|
||||
assert_eq!(m["membersFromHead"], true);
|
||||
assert_eq!(out.members.len(), 2);
|
||||
assert_eq!(
|
||||
out.members[0].aliases.as_ref().unwrap(),
|
||||
&["Ally".to_string()]
|
||||
);
|
||||
let roles = out.members[0].roles.as_ref().unwrap();
|
||||
assert_eq!(roles[0].id, "owner");
|
||||
assert_eq!(roles[0].name, None);
|
||||
|
||||
@@ -25,6 +25,7 @@ pub struct NativeMember {
|
||||
pub platform_id: String,
|
||||
pub account_name: String,
|
||||
pub group_nickname: Option<String>,
|
||||
pub aliases: Option<Vec<String>>,
|
||||
pub avatar: Option<String>,
|
||||
pub roles: Option<Vec<NativeMemberRole>>,
|
||||
}
|
||||
|
||||
@@ -102,6 +102,7 @@ impl MemberTracker {
|
||||
platform_id: platform_id.to_string(),
|
||||
account_name: account_name.to_string(),
|
||||
group_nickname: None,
|
||||
aliases: None,
|
||||
avatar: avatar.cloned(),
|
||||
roles: None,
|
||||
});
|
||||
|
||||
@@ -99,7 +99,7 @@ async function* parseChatLab(options: ParseOptions): AsyncGenerator<ParseEvent,
|
||||
// 记录解析开始
|
||||
onLog?.('info', `开始解析 ChatLab 格式文件,大小: ${(totalBytes / 1024 / 1024).toFixed(2)} MB`)
|
||||
|
||||
// 读取文件头获取 meta 和 members 信息
|
||||
// 读取文件头获取 meta 信息
|
||||
const headContent = readFileHeadBytes(filePath, 200000)
|
||||
|
||||
// 解析 meta
|
||||
@@ -140,6 +140,7 @@ async function* parseChatLab(options: ParseOptions): AsyncGenerator<ParseEvent,
|
||||
type: (metaObj.type as ChatType) || ChatType.GROUP,
|
||||
groupId: metaObj.groupId,
|
||||
groupAvatar: metaObj.groupAvatar,
|
||||
ownerId: metaObj.ownerId,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -179,6 +180,7 @@ async function* parseChatLab(options: ParseOptions): AsyncGenerator<ParseEvent,
|
||||
platformId: value.platformId,
|
||||
accountName: value.accountName,
|
||||
groupNickname: value.groupNickname,
|
||||
aliases: value.aliases,
|
||||
avatar: value.avatar,
|
||||
roles: value.roles,
|
||||
})
|
||||
|
||||
@@ -74,8 +74,7 @@ function assertParity(nativeResult: ParseResult, tsResult: ParseResult) {
|
||||
}
|
||||
}
|
||||
|
||||
// Canonical fixture without member roles (the TS head-regex parses this
|
||||
// members array successfully, so both implementations use the head members).
|
||||
// Canonical fixture used to verify native and TS parser parity.
|
||||
const CANONICAL_FIXTURE = {
|
||||
chatlab: { version: '0.0.2', exportedAt: 1700000000, generator: 'ChatLab' },
|
||||
meta: {
|
||||
@@ -84,9 +83,16 @@ const CANONICAL_FIXTURE = {
|
||||
type: 'group',
|
||||
groupId: 'g1@chatroom',
|
||||
groupAvatar: 'https://example.com/g.jpg',
|
||||
ownerId: 'u1',
|
||||
},
|
||||
members: [
|
||||
{ platformId: 'u1', accountName: 'Alice', groupNickname: '小A', avatar: 'https://example.com/a.jpg' },
|
||||
{
|
||||
platformId: 'u1',
|
||||
accountName: 'Alice',
|
||||
groupNickname: '小A',
|
||||
aliases: ['Ally'],
|
||||
avatar: 'https://example.com/a.jpg',
|
||||
},
|
||||
{ platformId: 'u2', accountName: 'Bob' },
|
||||
],
|
||||
messages: [
|
||||
@@ -123,7 +129,9 @@ describe('chatlab native parser parity', { skip: !nativeAvailable() && 'native m
|
||||
assert.equal(nativeResult.meta.name, '测试群')
|
||||
assert.equal(nativeResult.meta.platform, 'weixin')
|
||||
assert.equal(nativeResult.meta.groupId, 'g1@chatroom')
|
||||
assert.equal(nativeResult.meta.ownerId, 'u1')
|
||||
assert.equal(nativeResult.members.length, 2)
|
||||
assert.deepEqual(nativeResult.members[0].aliases, ['Ally'])
|
||||
assert.equal(nativeResult.members[0].avatar, 'https://example.com/a.jpg')
|
||||
assert.equal(nativeResult.messages.length, 4)
|
||||
assert.equal(nativeResult.messages[1].replyToMessageId, 'm1')
|
||||
|
||||
@@ -20,6 +20,7 @@ interface ChatlabMetaJson {
|
||||
platform: string
|
||||
groupId: string | null
|
||||
groupAvatar: string | null
|
||||
ownerId: string | null
|
||||
/**
|
||||
* true when members came from the top-level `members` array;
|
||||
* false when collected from messages (TS emits different object shapes).
|
||||
@@ -41,6 +42,7 @@ function toParsedMember(member: NativeMember, fromHead: boolean): ParsedMember {
|
||||
platformId: member.platformId,
|
||||
accountName: member.accountName,
|
||||
groupNickname: member.groupNickname,
|
||||
aliases: member.aliases,
|
||||
avatar: member.avatar,
|
||||
// Role objects pass through with their original key set ({id} or {id, name}).
|
||||
roles: member.roles?.map((role) => (role.name !== undefined ? { id: role.id, name: role.name } : { id: role.id })),
|
||||
@@ -59,6 +61,7 @@ const chatlabAdapter: NativeFormatAdapter = {
|
||||
type: meta.chatType as ChatType,
|
||||
groupId: meta.groupId ?? undefined,
|
||||
groupAvatar: meta.groupAvatar ?? undefined,
|
||||
ownerId: meta.ownerId ?? undefined,
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -96,6 +96,7 @@ export interface ParsedMember {
|
||||
platformId: string
|
||||
accountName: string
|
||||
groupNickname?: string
|
||||
aliases?: string[]
|
||||
avatar?: string
|
||||
roles?: MemberRole[]
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import { detectFormat, parseFileSync } from '@openchatlab/parser'
|
||||
import { CHAT_DB_SCHEMA } from '../packages/core/src/schema/tables'
|
||||
import { BetterSqliteAdapter } from '../packages/node-runtime/src/better-sqlite3-adapter'
|
||||
import { exportWithFormat } from '../packages/node-runtime/src/export/format-exporter'
|
||||
import { streamingImport } from '../packages/node-runtime/src/import/streaming-importer'
|
||||
import { writeParseResultToDb } from '../packages/node-runtime/src/import/write-parse-result'
|
||||
|
||||
test('exports JSON as ChatLab format that can be parsed for re-import', async () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'chatlab-json-export-'))
|
||||
@@ -84,12 +86,14 @@ test('exports JSON as ChatLab format that can be parsed for re-import', async ()
|
||||
type: 'group',
|
||||
groupId: 'group-1',
|
||||
groupAvatar: 'group-avatar',
|
||||
ownerId: 'alice',
|
||||
})
|
||||
assert.deepEqual(
|
||||
parsed.members.map((member) => ({
|
||||
platformId: member.platformId,
|
||||
accountName: member.accountName,
|
||||
groupNickname: member.groupNickname,
|
||||
aliases: member.aliases,
|
||||
roles: member.roles,
|
||||
})),
|
||||
[
|
||||
@@ -97,12 +101,14 @@ test('exports JSON as ChatLab format that can be parsed for re-import', async ()
|
||||
platformId: 'alice',
|
||||
accountName: 'Alice Account',
|
||||
groupNickname: '爱丽丝',
|
||||
aliases: ['Alice'],
|
||||
roles: [{ id: 'owner' }],
|
||||
},
|
||||
{
|
||||
platformId: 'bob',
|
||||
accountName: 'Bob Account',
|
||||
groupNickname: '鲍勃',
|
||||
aliases: undefined,
|
||||
roles: undefined,
|
||||
},
|
||||
]
|
||||
@@ -129,6 +135,51 @@ test('exports JSON as ChatLab format that can be parsed for re-import', async ()
|
||||
replyToMessageId: 'msg-2',
|
||||
},
|
||||
])
|
||||
|
||||
const sharedWriterDb = new Database(':memory:')
|
||||
try {
|
||||
sharedWriterDb.exec(CHAT_DB_SCHEMA)
|
||||
writeParseResultToDb(new BetterSqliteAdapter(sharedWriterDb), parsed.meta, parsed.members, parsed.messages)
|
||||
const sharedWriterAlice = sharedWriterDb
|
||||
.prepare("SELECT aliases FROM member WHERE platform_id = 'alice'")
|
||||
.get() as { aliases: string }
|
||||
assert.deepEqual(JSON.parse(sharedWriterAlice.aliases), ['Alice'])
|
||||
} finally {
|
||||
sharedWriterDb.close()
|
||||
}
|
||||
|
||||
const importedDbPath = join(tempDir, 'imported.db')
|
||||
const importResult = await streamingImport(
|
||||
filePath,
|
||||
{
|
||||
openDatabase() {
|
||||
const db = new Database(importedDbPath)
|
||||
db.exec(CHAT_DB_SCHEMA)
|
||||
return new BetterSqliteAdapter(db)
|
||||
},
|
||||
deleteDatabase() {
|
||||
/* temp directory cleanup is handled by the outer finally */
|
||||
},
|
||||
onProgress() {
|
||||
/* progress is outside this round-trip assertion */
|
||||
},
|
||||
},
|
||||
undefined,
|
||||
'imported-session'
|
||||
)
|
||||
assert.equal(importResult.success, true)
|
||||
|
||||
const importedDb = new Database(importedDbPath)
|
||||
try {
|
||||
const importedMeta = importedDb.prepare('SELECT owner_id FROM meta').get() as { owner_id: string | null }
|
||||
const importedAlice = importedDb.prepare("SELECT aliases FROM member WHERE platform_id = 'alice'").get() as {
|
||||
aliases: string
|
||||
}
|
||||
assert.equal(importedMeta.owner_id, 'alice')
|
||||
assert.deepEqual(JSON.parse(importedAlice.aliases), ['Alice'])
|
||||
} finally {
|
||||
importedDb.close()
|
||||
}
|
||||
} finally {
|
||||
rawDb.close()
|
||||
rmSync(tempDir, { recursive: true, force: true })
|
||||
|
||||
Reference in new issue
Block a user