From 758b0667fc683016cd74f3808f5e490141f78ccf Mon Sep 17 00:00:00 2001 From: KokerZhou <111279477+KokerZhou@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:54:55 +0800 Subject: [PATCH] feat(docker): preinstall default NLP dictionary --- .github/docker/chatlab-cli.Dockerfile | 4 ++ apps/cli/src/http/routes/web/index.ts | 1 + docs/cn/usage/docker.md | 2 + docs/en/usage/docker.md | 2 + docs/tw/usage/docker.md | 2 + packages/http-routes/src/context/runtime.ts | 2 + packages/http-routes/src/routes/web/nlp.ts | 5 ++- .../node-runtime/src/nlp/dict-manager.test.ts | 39 +++++++++++++++++++ packages/node-runtime/src/nlp/dict-manager.ts | 14 ++++++- 9 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 packages/node-runtime/src/nlp/dict-manager.test.ts diff --git a/.github/docker/chatlab-cli.Dockerfile b/.github/docker/chatlab-cli.Dockerfile index e614983a..3b3d9411 100644 --- a/.github/docker/chatlab-cli.Dockerfile +++ b/.github/docker/chatlab-cli.Dockerfile @@ -4,6 +4,10 @@ FROM node:24-bookworm-slim ARG CHATLAB_VERSION ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding +ENV CHATLAB_NLP_DICT_DIR=/opt/chatlab/nlp + +ADD --checksum=sha256:139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f --chmod=0644 \ + https://chatlab.fun/assets/nlp/zh-CN.dict /opt/chatlab/nlp/zh-CN.dict RUN test -n "$CHATLAB_VERSION" \ && apt-get update \ diff --git a/apps/cli/src/http/routes/web/index.ts b/apps/cli/src/http/routes/web/index.ts index 83c6c79a..b16f23e8 100644 --- a/apps/cli/src/http/routes/web/index.ts +++ b/apps/cli/src/http/routes/web/index.ts @@ -139,6 +139,7 @@ export function registerWebRoutes( runtimeIdentity: options?.runtimeIdentity, getVersion, nativeBinding: options?.nativeBinding, + bundledNlpDictDir: process.env.CHATLAB_NLP_DICT_DIR?.trim() || undefined, semanticIndexService, contactsService: options?.contactsService, globalInsightService: options?.globalInsightService, diff --git a/docs/cn/usage/docker.md b/docs/cn/usage/docker.md index 8716caab..f277b72a 100644 --- a/docs/cn/usage/docker.md +++ b/docs/cn/usage/docker.md @@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli 官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时，只需按界面提示下载所选模型文件，不会在容器启动后再安装约 370 MB 的 Node 依赖。 +镜像还将默认的简体中文分词词典存放在 `/opt/chatlab/nlp`，并通过 `CHATLAB_NLP_DICT_DIR` 指向该路径，因此首次启动时无需下载。挂载的 ChatLab 目录中已有的词典会被保留。 + ## 快速开始 ### 与 Desktop / 本地 CLI 共用数据（推荐） diff --git a/docs/en/usage/docker.md b/docs/en/usage/docker.md index 1cf5fa84..755f4070 100644 --- a/docs/en/usage/docker.md +++ b/docs/en/usage/docker.md @@ -13,6 +13,8 @@ ghcr.io/chatlab/chatlab-cli The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts. +The image also stores the default Simplified Chinese segmentation dictionary under `/opt/chatlab/nlp` and points `CHATLAB_NLP_DICT_DIR` there, so first startup does not need to download it. An existing dictionary in the mounted ChatLab directory is preserved. + ## Quick start ### Share data with Desktop and a local CLI (recommended) diff --git a/docs/tw/usage/docker.md b/docs/tw/usage/docker.md index 26054917..1ffe6601 100644 --- a/docs/tw/usage/docker.md +++ b/docs/tw/usage/docker.md @@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli 官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時，只需依照介面提示下載所選模型檔案，不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。 +映像也將預設的簡體中文斷詞詞典存放在 `/opt/chatlab/nlp`，並透過 `CHATLAB_NLP_DICT_DIR` 指向該路徑，因此首次啟動時不需要下載。掛載的 ChatLab 目錄中既有的詞典會保留。 + ## 快速開始 ### 與 Desktop / 本機 CLI 共用資料（建議） diff --git a/packages/http-routes/src/context/runtime.ts b/packages/http-routes/src/context/runtime.ts index 5c4b0e3d..71963650 100644 --- a/packages/http-routes/src/context/runtime.ts +++ b/packages/http-routes/src/context/runtime.ts @@ -14,4 +14,6 @@ export interface RuntimeRouteContext { runtimeIdentity?: RuntimeIdentity /** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */ nativeBinding?: string + /** Read-only directory containing dictionaries bundled by the current runtime. */ + bundledNlpDictDir?: string } diff --git a/packages/http-routes/src/routes/web/nlp.ts b/packages/http-routes/src/routes/web/nlp.ts index c85fa810..c6e36252 100644 --- a/packages/http-routes/src/routes/web/nlp.ts +++ b/packages/http-routes/src/routes/web/nlp.ts @@ -27,12 +27,15 @@ import { type NlpRouteContext = AnalyticsCacheContext & { pathProvider: Pick dbManager: Pick + bundledNlpDictDir?: string } export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void { const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp') initNlpDir(nlpDir) - ensureDefaultDict(nlpDir).catch((err) => console.warn('[NLP] Auto-download zh-CN dict failed:', err)) + ensureDefaultDict(nlpDir, ctx.bundledNlpDictDir).catch((err) => + console.warn('[NLP] Initialize zh-CN dict failed:', err) + ) server.get('/_web/nlp/pos-tags', async () => { return POS_TAG_DEFINITIONS diff --git a/packages/node-runtime/src/nlp/dict-manager.test.ts b/packages/node-runtime/src/nlp/dict-manager.test.ts new file mode 100644 index 00000000..0cc29b01 --- /dev/null +++ b/packages/node-runtime/src/nlp/dict-manager.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { ensureDefaultDict } from './dict-manager' + +test('ensureDefaultDict installs a bundled dictionary when the writable directory is empty', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-')) + const bundledDir = path.join(root, 'bundled') + const nlpDir = path.join(root, 'writable') + fs.mkdirSync(bundledDir) + fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary') + + try { + await ensureDefaultDict(nlpDir, bundledDir) + assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'bundled dictionary') + assert.equal(fs.existsSync(path.join(nlpDir, 'zh-CN.dict.tmp')), false) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('ensureDefaultDict preserves an existing writable dictionary', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-')) + const bundledDir = path.join(root, 'bundled') + const nlpDir = path.join(root, 'writable') + fs.mkdirSync(bundledDir) + fs.mkdirSync(nlpDir) + fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary') + fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), 'existing dictionary') + + try { + await ensureDefaultDict(nlpDir, bundledDir) + assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'existing dictionary') + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/packages/node-runtime/src/nlp/dict-manager.ts b/packages/node-runtime/src/nlp/dict-manager.ts index 2fb27f89..a013bfb7 100644 --- a/packages/node-runtime/src/nlp/dict-manager.ts +++ b/packages/node-runtime/src/nlp/dict-manager.ts @@ -152,8 +152,20 @@ export function deleteDict(nlpDir: string, dictId: string): { success: boolean; /** * 自动下载默认词库（应用启动时调用） */ -export async function ensureDefaultDict(nlpDir: string): Promise { +export async function ensureDefaultDict(nlpDir: string, bundledDictDir?: string): Promise { if (isDictDownloaded(nlpDir, 'zh-CN')) return + if (bundledDictDir) { + const bundledPath = getDictFilePath(bundledDictDir, 'zh-CN') + if (fs.existsSync(bundledPath)) { + fs.mkdirSync(nlpDir, { recursive: true }) + const filePath = getDictFilePath(nlpDir, 'zh-CN') + const tmpPath = filePath + '.tmp' + fs.copyFileSync(bundledPath, tmpPath) + fs.renameSync(tmpPath, filePath) + console.log('[NLP DictManager] Installed bundled zh-CN dictionary') + return + } + } console.log('[NLP DictManager] zh-CN dict not found, starting background download...') const result = await downloadDict(nlpDir, 'zh-CN') if (result.success) {