mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
feat(docker): preinstall default NLP dictionary
This commit is contained in:
1 parent
e2723cf751
commit
758b0667fc
9 files changed
+69
-2
No files matched your search
@@ -4,6 +4,10 @@ FROM node:24-bookworm-slim
|
||||
|
||||
ARG CHATLAB_VERSION
|
||||
ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding
|
||||
ENV CHATLAB_NLP_DICT_DIR=/opt/chatlab/nlp
|
||||
|
||||
ADD --checksum=sha256:139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f --chmod=0644 \
|
||||
https://chatlab.fun/assets/nlp/zh-CN.dict /opt/chatlab/nlp/zh-CN.dict
|
||||
|
||||
RUN test -n "$CHATLAB_VERSION" \
|
||||
&& apt-get update \
|
||||
|
||||
@@ -139,6 +139,7 @@ export function registerWebRoutes(
|
||||
runtimeIdentity: options?.runtimeIdentity,
|
||||
getVersion,
|
||||
nativeBinding: options?.nativeBinding,
|
||||
bundledNlpDictDir: process.env.CHATLAB_NLP_DICT_DIR?.trim() || undefined,
|
||||
semanticIndexService,
|
||||
contactsService: options?.contactsService,
|
||||
globalInsightService: options?.globalInsightService,
|
||||
|
||||
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
|
||||
|
||||
官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时,只需按界面提示下载所选模型文件,不会在容器启动后再安装约 370 MB 的 Node 依赖。
|
||||
|
||||
镜像还将默认的简体中文分词词典存放在 `/opt/chatlab/nlp`,并通过 `CHATLAB_NLP_DICT_DIR` 指向该路径,因此首次启动时无需下载。挂载的 ChatLab 目录中已有的词典会被保留。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 与 Desktop / 本地 CLI 共用数据(推荐)
|
||||
|
||||
@@ -13,6 +13,8 @@ ghcr.io/chatlab/chatlab-cli
|
||||
|
||||
The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts.
|
||||
|
||||
The image also stores the default Simplified Chinese segmentation dictionary under `/opt/chatlab/nlp` and points `CHATLAB_NLP_DICT_DIR` there, so first startup does not need to download it. An existing dictionary in the mounted ChatLab directory is preserved.
|
||||
|
||||
## Quick start
|
||||
|
||||
### Share data with Desktop and a local CLI (recommended)
|
||||
|
||||
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
|
||||
|
||||
官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時,只需依照介面提示下載所選模型檔案,不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。
|
||||
|
||||
映像也將預設的簡體中文斷詞詞典存放在 `/opt/chatlab/nlp`,並透過 `CHATLAB_NLP_DICT_DIR` 指向該路徑,因此首次啟動時不需要下載。掛載的 ChatLab 目錄中既有的詞典會保留。
|
||||
|
||||
## 快速開始
|
||||
|
||||
### 與 Desktop / 本機 CLI 共用資料(建議)
|
||||
|
||||
@@ -14,4 +14,6 @@ export interface RuntimeRouteContext {
|
||||
runtimeIdentity?: RuntimeIdentity
|
||||
/** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */
|
||||
nativeBinding?: string
|
||||
/** Read-only directory containing dictionaries bundled by the current runtime. */
|
||||
bundledNlpDictDir?: string
|
||||
}
|
||||
@@ -27,12 +27,15 @@ import {
|
||||
type NlpRouteContext = AnalyticsCacheContext & {
|
||||
pathProvider: Pick<PathProvider, 'getCacheDir' | 'getSystemDir'>
|
||||
dbManager: Pick<DatabaseManager, 'open'>
|
||||
bundledNlpDictDir?: string
|
||||
}
|
||||
|
||||
export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void {
|
||||
const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp')
|
||||
initNlpDir(nlpDir)
|
||||
ensureDefaultDict(nlpDir).catch((err) => console.warn('[NLP] Auto-download zh-CN dict failed:', err))
|
||||
ensureDefaultDict(nlpDir, ctx.bundledNlpDictDir).catch((err) =>
|
||||
console.warn('[NLP] Initialize zh-CN dict failed:', err)
|
||||
)
|
||||
|
||||
server.get('/_web/nlp/pos-tags', async () => {
|
||||
return POS_TAG_DEFINITIONS
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { ensureDefaultDict } from './dict-manager'
|
||||
|
||||
test('ensureDefaultDict installs a bundled dictionary when the writable directory is empty', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
|
||||
const bundledDir = path.join(root, 'bundled')
|
||||
const nlpDir = path.join(root, 'writable')
|
||||
fs.mkdirSync(bundledDir)
|
||||
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
|
||||
|
||||
try {
|
||||
await ensureDefaultDict(nlpDir, bundledDir)
|
||||
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'bundled dictionary')
|
||||
assert.equal(fs.existsSync(path.join(nlpDir, 'zh-CN.dict.tmp')), false)
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('ensureDefaultDict preserves an existing writable dictionary', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
|
||||
const bundledDir = path.join(root, 'bundled')
|
||||
const nlpDir = path.join(root, 'writable')
|
||||
fs.mkdirSync(bundledDir)
|
||||
fs.mkdirSync(nlpDir)
|
||||
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
|
||||
fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), 'existing dictionary')
|
||||
|
||||
try {
|
||||
await ensureDefaultDict(nlpDir, bundledDir)
|
||||
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'existing dictionary')
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -152,8 +152,20 @@ export function deleteDict(nlpDir: string, dictId: string): { success: boolean;
|
||||
/**
|
||||
* 自动下载默认词库(应用启动时调用)
|
||||
*/
|
||||
export async function ensureDefaultDict(nlpDir: string): Promise<void> {
|
||||
export async function ensureDefaultDict(nlpDir: string, bundledDictDir?: string): Promise<void> {
|
||||
if (isDictDownloaded(nlpDir, 'zh-CN')) return
|
||||
if (bundledDictDir) {
|
||||
const bundledPath = getDictFilePath(bundledDictDir, 'zh-CN')
|
||||
if (fs.existsSync(bundledPath)) {
|
||||
fs.mkdirSync(nlpDir, { recursive: true })
|
||||
const filePath = getDictFilePath(nlpDir, 'zh-CN')
|
||||
const tmpPath = filePath + '.tmp'
|
||||
fs.copyFileSync(bundledPath, tmpPath)
|
||||
fs.renameSync(tmpPath, filePath)
|
||||
console.log('[NLP DictManager] Installed bundled zh-CN dictionary')
|
||||
return
|
||||
}
|
||||
}
|
||||
console.log('[NLP DictManager] zh-CN dict not found, starting background download...')
|
||||
const result = await downloadDict(nlpDir, 'zh-CN')
|
||||
if (result.success) {
|
||||
|
||||
Reference in new issue
Block a user