mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
test(parser,config): cover representative fixtures and migrations
This commit is contained in:
digua committed
1 parent
2d88215d63
commit
2887f1cb73
2 files changed
+444
No files matched your search
@@ -0,0 +1,138 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { createCipheriv, createHash, randomBytes } from 'node:crypto'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { mkdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { after, beforeEach, describe, it } from 'node:test'
|
||||
|
||||
const tempHome = mkdtempSync(join(tmpdir(), 'chatlab-config-migration-'))
|
||||
const originalHome = process.env.HOME
|
||||
process.env.HOME = tempHome
|
||||
|
||||
const [{ m004EncryptedKeysToAuthProfiles }, { MigrationRunner }, { getApiKeyByProfile, loadAuthProfiles }] =
|
||||
await Promise.all([import('./m004-encrypted-keys-to-auth-profiles'), import('./runner'), import('../auth-profiles')])
|
||||
|
||||
const chatlabDir = join(tempHome, '.chatlab')
|
||||
const aiDataDir = join(chatlabDir, 'ai')
|
||||
|
||||
function createLogger() {
|
||||
const infos: string[] = []
|
||||
const warnings: string[] = []
|
||||
const errors: string[] = []
|
||||
|
||||
return {
|
||||
infos,
|
||||
warnings,
|
||||
errors,
|
||||
logger: {
|
||||
info(_category: string, message: string) {
|
||||
infos.push(message)
|
||||
},
|
||||
warn(_category: string, message: string) {
|
||||
warnings.push(message)
|
||||
},
|
||||
error(_category: string, message: string) {
|
||||
errors.push(message)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function writeLlmConfig(configs: Array<Record<string, unknown>>): Promise<void> {
|
||||
await mkdir(aiDataDir, { recursive: true })
|
||||
await writeFile(join(aiDataDir, 'llm-config.json'), JSON.stringify({ version: 3, configs }, null, 2), 'utf-8')
|
||||
}
|
||||
|
||||
function readLlmConfig(): { configs: Array<Record<string, unknown>> } {
|
||||
return JSON.parse(readFileSync(join(aiDataDir, 'llm-config.json'), 'utf-8')) as {
|
||||
configs: Array<Record<string, unknown>>
|
||||
}
|
||||
}
|
||||
|
||||
function encryptWithDeviceKey(plainText: string, deviceKey: string): string {
|
||||
const key = createHash('sha256')
|
||||
.update(deviceKey + 'chatlab-api-key-encryption-v1')
|
||||
.digest()
|
||||
const iv = randomBytes(12)
|
||||
const cipher = createCipheriv('aes-256-gcm', key, iv)
|
||||
let encrypted = cipher.update(plainText, 'utf8', 'base64')
|
||||
encrypted += cipher.final('base64')
|
||||
const authTag = cipher.getAuthTag()
|
||||
return `enc:${iv.toString('base64')}:${authTag.toString('base64')}:${encrypted}`
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await rm(chatlabDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
after(() => {
|
||||
if (originalHome === undefined) {
|
||||
delete process.env.HOME
|
||||
} else {
|
||||
process.env.HOME = originalHome
|
||||
}
|
||||
rmSync(tempHome, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('m004 encrypted keys to auth profiles migration', () => {
|
||||
it('migrates plaintext keys through the runner, deduplicates profile names, and clears old apiKey fields', async () => {
|
||||
await writeLlmConfig([
|
||||
{ name: 'DeepSeek Main', provider: 'deepseek', apiKey: 'plain-one' },
|
||||
{ name: 'DeepSeek Backup', provider: 'deepseek', apiKey: 'plain-two' },
|
||||
{
|
||||
name: 'OpenAI Compatible',
|
||||
provider: 'openai-compatible',
|
||||
baseUrl: 'https://api.example.com/v1',
|
||||
apiKey: 'plain-three',
|
||||
},
|
||||
])
|
||||
|
||||
const { logger } = createLogger()
|
||||
const runner = new MigrationRunner([m004EncryptedKeysToAuthProfiles], {
|
||||
dataDir: chatlabDir,
|
||||
aiDataDir,
|
||||
logger,
|
||||
})
|
||||
|
||||
assert.deepEqual(await runner.run(), { executed: 1, currentVersion: 4 })
|
||||
|
||||
const profiles = loadAuthProfiles().profiles
|
||||
assert.equal(profiles.deepseek.key, 'plain-one')
|
||||
assert.equal(profiles['deepseek-2'].key, 'plain-two')
|
||||
assert.equal(profiles['api.example.com'].key, 'plain-three')
|
||||
assert.equal(getApiKeyByProfile('deepseek'), 'plain-one')
|
||||
assert.deepEqual(
|
||||
readLlmConfig().configs.map((config) => config.apiKey),
|
||||
['', '', '']
|
||||
)
|
||||
assert.equal(readFileSync(join(chatlabDir, '.migration-version'), 'utf-8'), '4')
|
||||
})
|
||||
|
||||
it('decrypts legacy device-key encrypted api keys into auth profiles', async () => {
|
||||
const deviceKey = '0123456789abcdef0123456789abcdef'
|
||||
await mkdir(chatlabDir, { recursive: true })
|
||||
await writeFile(join(chatlabDir, '.device-key'), deviceKey, 'utf-8')
|
||||
await writeLlmConfig([
|
||||
{ name: 'Anthropic', provider: 'anthropic', apiKey: encryptWithDeviceKey('secret-key', deviceKey) },
|
||||
])
|
||||
|
||||
const { logger } = createLogger()
|
||||
await m004EncryptedKeysToAuthProfiles.up({ dataDir: chatlabDir, aiDataDir, logger })
|
||||
|
||||
assert.equal(getApiKeyByProfile('anthropic'), 'secret-key')
|
||||
assert.equal(readLlmConfig().configs[0].apiKey, '')
|
||||
})
|
||||
|
||||
it('does not create an auth profile or clear apiKey when encrypted key decryption fails', async () => {
|
||||
await writeLlmConfig([{ name: 'Google', provider: 'google', apiKey: 'enc:not:a:valid-key' }])
|
||||
|
||||
const { logger, warnings } = createLogger()
|
||||
await m004EncryptedKeysToAuthProfiles.up({ dataDir: chatlabDir, aiDataDir, logger })
|
||||
|
||||
assert.equal(existsSync(join(chatlabDir, 'auth-profiles.json')), false)
|
||||
assert.equal(loadAuthProfiles().profiles.google, undefined)
|
||||
assert.equal(readLlmConfig().configs[0].apiKey, 'enc:not:a:valid-key')
|
||||
assert.equal(warnings.length, 1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,306 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { describe, it } from 'node:test'
|
||||
import { ChatType, KNOWN_PLATFORMS, MessageType } from '@openchatlab/shared-types'
|
||||
|
||||
import { detectFormat, parseFileSync } from '../index'
|
||||
import type { ParseResult } from '../types'
|
||||
|
||||
interface ParserFixture {
|
||||
filename: string
|
||||
content: string
|
||||
formatId: string
|
||||
expected: {
|
||||
meta: {
|
||||
name: string
|
||||
platform: string
|
||||
type: ChatType
|
||||
}
|
||||
memberIds: string[]
|
||||
messages: Array<{
|
||||
senderPlatformId: string
|
||||
timestamp: number
|
||||
type: MessageType
|
||||
content: string | null
|
||||
}>
|
||||
}
|
||||
}
|
||||
|
||||
function localTs(isoLocal: string): number {
|
||||
return Math.floor(new Date(isoLocal).getTime() / 1000)
|
||||
}
|
||||
|
||||
function json(value: unknown): string {
|
||||
return JSON.stringify(value, null, 2)
|
||||
}
|
||||
|
||||
function jsonLine(value: unknown): string {
|
||||
return JSON.stringify(value)
|
||||
}
|
||||
|
||||
async function parseFixture(fixture: ParserFixture): Promise<ParseResult> {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'chatlab-parser-fixture-'))
|
||||
try {
|
||||
const filePath = join(dir, fixture.filename)
|
||||
writeFileSync(filePath, fixture.content, 'utf-8')
|
||||
|
||||
assert.equal(detectFormat(filePath)?.id, fixture.formatId)
|
||||
return await parseFileSync(filePath)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
const fixtures: ParserFixture[] = [
|
||||
{
|
||||
filename: 'qq-group.txt',
|
||||
content: [
|
||||
'消息记录（此消息记录为文本格式，不支持重新导入）',
|
||||
'消息对象:测试 QQ 群',
|
||||
'2024-01-02 03:04:05 Alice(10001)',
|
||||
'hello qq',
|
||||
'2024-01-02 03:05:06 Bob(10002)',
|
||||
'[图片]',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'qq-native-txt',
|
||||
expected: {
|
||||
meta: { name: '测试 QQ 群', platform: KNOWN_PLATFORMS.QQ, type: ChatType.GROUP },
|
||||
memberIds: ['10001', '10002'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: '10001',
|
||||
timestamp: localTs('2024-01-02T03:04:05'),
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello qq',
|
||||
},
|
||||
{
|
||||
senderPlatformId: '10002',
|
||||
timestamp: localTs('2024-01-02T03:05:06'),
|
||||
type: MessageType.IMAGE,
|
||||
content: '[图片]',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
filename: 'weflow.json',
|
||||
content: json({
|
||||
weflow: { version: '1.0.0', exportedAt: 1704164645 },
|
||||
session: {
|
||||
wxid: 'room@chatroom',
|
||||
nickname: '微信测试群',
|
||||
remark: '',
|
||||
displayName: '微信测试群',
|
||||
type: '群聊',
|
||||
lastTimestamp: 1704164706,
|
||||
messageCount: 2,
|
||||
},
|
||||
avatars: {},
|
||||
messages: [
|
||||
{
|
||||
localId: 1,
|
||||
createTime: 1704164645,
|
||||
formattedTime: '2024-01-02 03:04:05',
|
||||
type: '文本消息',
|
||||
localType: 1,
|
||||
content: 'hello wechat',
|
||||
isSend: 0,
|
||||
senderUsername: 'wxid_alice',
|
||||
senderDisplayName: 'Alice',
|
||||
senderAvatarKey: 'wxid_alice',
|
||||
source: '',
|
||||
},
|
||||
{
|
||||
localId: 2,
|
||||
createTime: 1704164706,
|
||||
formattedTime: '2024-01-02 03:05:06',
|
||||
type: '图片消息',
|
||||
localType: 3,
|
||||
content: '[图片]',
|
||||
isSend: 1,
|
||||
senderUsername: 'wxid_bob',
|
||||
senderDisplayName: 'Bob',
|
||||
senderAvatarKey: 'wxid_bob',
|
||||
source: '',
|
||||
},
|
||||
],
|
||||
}),
|
||||
formatId: 'weflow',
|
||||
expected: {
|
||||
meta: { name: '微信测试群', platform: KNOWN_PLATFORMS.WECHAT, type: ChatType.GROUP },
|
||||
memberIds: ['wxid_alice', 'wxid_bob'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: 'wxid_alice',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello wechat',
|
||||
},
|
||||
{
|
||||
senderPlatformId: 'wxid_bob',
|
||||
timestamp: 1704164706,
|
||||
type: MessageType.IMAGE,
|
||||
content: '[图片]',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
filename: 'telegram.json',
|
||||
content: json({
|
||||
name: 'Telegram Test Chat',
|
||||
type: 'private_group',
|
||||
id: 4242,
|
||||
messages: [
|
||||
{
|
||||
id: 1,
|
||||
type: 'message',
|
||||
date: '2024-01-02T03:04:05',
|
||||
date_unixtime: '1704164645',
|
||||
from: 'Alice',
|
||||
from_id: 'user10001',
|
||||
text: 'hello telegram',
|
||||
text_entities: [{ type: 'plain', text: 'hello telegram' }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
formatId: 'telegram-native-single',
|
||||
expected: {
|
||||
meta: { name: 'Telegram Test Chat', platform: KNOWN_PLATFORMS.TELEGRAM, type: ChatType.GROUP },
|
||||
memberIds: ['10001'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: '10001',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello telegram',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
filename: '与Alice的 WhatsApp 聊天.txt',
|
||||
content: [
|
||||
'Messages and calls are end-to-end encrypted.',
|
||||
'2024/01/02 03:04 - Alice: hello whatsapp',
|
||||
'2024/01/02 03:05 - Bob: image omitted',
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'whatsapp-native-txt',
|
||||
expected: {
|
||||
meta: { name: 'Alice', platform: KNOWN_PLATFORMS.WHATSAPP, type: ChatType.PRIVATE },
|
||||
memberIds: ['Alice', 'Bob'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: 'Alice',
|
||||
timestamp: localTs('2024-01-02T03:04:00'),
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello whatsapp',
|
||||
},
|
||||
{
|
||||
senderPlatformId: 'Bob',
|
||||
timestamp: localTs('2024-01-02T03:05:00'),
|
||||
type: MessageType.IMAGE,
|
||||
content: 'image omitted',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
filename: 'chatlab.json',
|
||||
content: json({
|
||||
chatlab: { version: '1.0.0', exportedAt: 1704164645 },
|
||||
meta: { name: 'ChatLab JSON 群', platform: 'qq', type: 'group' },
|
||||
members: [
|
||||
{ platformId: 'u1', accountName: 'Alice', groupNickname: 'A' },
|
||||
{ platformId: 'u2', accountName: 'Bob', groupNickname: 'B' },
|
||||
],
|
||||
messages: [
|
||||
{
|
||||
sender: 'u1',
|
||||
accountName: 'Alice',
|
||||
groupNickname: 'A',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello chatlab json',
|
||||
},
|
||||
],
|
||||
}),
|
||||
formatId: 'chatlab',
|
||||
expected: {
|
||||
meta: { name: 'ChatLab JSON 群', platform: KNOWN_PLATFORMS.QQ, type: ChatType.GROUP },
|
||||
memberIds: ['u1', 'u2'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: 'u1',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello chatlab json',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
filename: 'chatlab.jsonl',
|
||||
content: [
|
||||
jsonLine({
|
||||
_type: 'header',
|
||||
chatlab: { version: '1.0.0', exportedAt: 1704164645 },
|
||||
meta: { name: 'ChatLab JSONL 群', platform: 'telegram', type: 'group' },
|
||||
}),
|
||||
jsonLine({ _type: 'member', platformId: 'tg1', accountName: 'Alice' }),
|
||||
jsonLine({
|
||||
_type: 'message',
|
||||
sender: 'tg1',
|
||||
accountName: 'Alice',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello chatlab jsonl',
|
||||
}),
|
||||
'',
|
||||
].join('\n'),
|
||||
formatId: 'chatlab-jsonl',
|
||||
expected: {
|
||||
meta: { name: 'ChatLab JSONL 群', platform: KNOWN_PLATFORMS.TELEGRAM, type: ChatType.GROUP },
|
||||
memberIds: ['tg1'],
|
||||
messages: [
|
||||
{
|
||||
senderPlatformId: 'tg1',
|
||||
timestamp: 1704164645,
|
||||
type: MessageType.TEXT,
|
||||
content: 'hello chatlab jsonl',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
describe('parser representative format fixtures', () => {
|
||||
for (const fixture of fixtures) {
|
||||
it(`detects and parses ${fixture.formatId}`, async () => {
|
||||
const result = await parseFixture(fixture)
|
||||
|
||||
assert.deepEqual(
|
||||
{
|
||||
name: result.meta.name,
|
||||
platform: result.meta.platform,
|
||||
type: result.meta.type,
|
||||
},
|
||||
fixture.expected.meta
|
||||
)
|
||||
assert.deepEqual(result.members.map((member) => member.platformId).sort(), [...fixture.expected.memberIds].sort())
|
||||
assert.deepEqual(
|
||||
result.messages.map((message) => ({
|
||||
senderPlatformId: message.senderPlatformId,
|
||||
timestamp: message.timestamp,
|
||||
type: message.type,
|
||||
content: message.content,
|
||||
})),
|
||||
fixture.expected.messages
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
Reference in new issue
Block a user