feat(docker): preinstall default NLP dictionary

This commit is contained in:
KokerZhou authored and digua committed 2026-08-28 22:24:13 +08:00
1 parent e2723cf751
commit 758b0667fc
9 files changed
+69 -2

No files matched your search

+4
View File
@@ -4,6 +4,10 @@ FROM node:24-bookworm-slim
ARG CHATLAB_VERSION ARG CHATLAB_VERSION
ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding
ENV CHATLAB_NLP_DICT_DIR=/opt/chatlab/nlp
ADD --checksum=sha256:139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f --chmod=0644 \
https://chatlab.fun/assets/nlp/zh-CN.dict /opt/chatlab/nlp/zh-CN.dict
RUN test -n "$CHATLAB_VERSION" \ RUN test -n "$CHATLAB_VERSION" \
&& apt-get update \ && apt-get update \
+1
View File
@@ -139,6 +139,7 @@ export function registerWebRoutes(
runtimeIdentity: options?.runtimeIdentity, runtimeIdentity: options?.runtimeIdentity,
getVersion, getVersion,
nativeBinding: options?.nativeBinding, nativeBinding: options?.nativeBinding,
bundledNlpDictDir: process.env.CHATLAB_NLP_DICT_DIR?.trim() || undefined,
semanticIndexService, semanticIndexService,
contactsService: options?.contactsService, contactsService: options?.contactsService,
globalInsightService: options?.globalInsightService, globalInsightService: options?.globalInsightService,
+2
View File
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时,只需按界面提示下载所选模型文件,不会在容器启动后再安装约 370 MB 的 Node 依赖。 官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时,只需按界面提示下载所选模型文件,不会在容器启动后再安装约 370 MB 的 Node 依赖。
镜像还将默认的简体中文分词词典存放在 `/opt/chatlab/nlp`,并通过 `CHATLAB_NLP_DICT_DIR` 指向该路径,因此首次启动时无需下载。挂载的 ChatLab 目录中已有的词典会被保留。
## 快速开始 ## 快速开始
### 与 Desktop / 本地 CLI 共用数据(推荐) ### 与 Desktop / 本地 CLI 共用数据(推荐)
+2
View File
@@ -13,6 +13,8 @@ ghcr.io/chatlab/chatlab-cli
The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts. The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts.
The image also stores the default Simplified Chinese segmentation dictionary under `/opt/chatlab/nlp` and points `CHATLAB_NLP_DICT_DIR` there, so first startup does not need to download it. An existing dictionary in the mounted ChatLab directory is preserved.
## Quick start ## Quick start
### Share data with Desktop and a local CLI (recommended) ### Share data with Desktop and a local CLI (recommended)
+2
View File
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時,只需依照介面提示下載所選模型檔案,不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。 官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時,只需依照介面提示下載所選模型檔案,不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。
映像也將預設的簡體中文斷詞詞典存放在 `/opt/chatlab/nlp`,並透過 `CHATLAB_NLP_DICT_DIR` 指向該路徑,因此首次啟動時不需要下載。掛載的 ChatLab 目錄中既有的詞典會保留。
## 快速開始 ## 快速開始
### 與 Desktop / 本機 CLI 共用資料(建議) ### 與 Desktop / 本機 CLI 共用資料(建議)
@@ -14,4 +14,6 @@ export interface RuntimeRouteContext {
runtimeIdentity?: RuntimeIdentity runtimeIdentity?: RuntimeIdentity
/** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */ /** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */
nativeBinding?: string nativeBinding?: string
/** Read-only directory containing dictionaries bundled by the current runtime. */
bundledNlpDictDir?: string
} }
+4 -1
View File
@@ -27,12 +27,15 @@ import {
type NlpRouteContext = AnalyticsCacheContext & { type NlpRouteContext = AnalyticsCacheContext & {
pathProvider: Pick<PathProvider, 'getCacheDir' | 'getSystemDir'> pathProvider: Pick<PathProvider, 'getCacheDir' | 'getSystemDir'>
dbManager: Pick<DatabaseManager, 'open'> dbManager: Pick<DatabaseManager, 'open'>
bundledNlpDictDir?: string
} }
export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void { export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void {
const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp') const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp')
initNlpDir(nlpDir) initNlpDir(nlpDir)
ensureDefaultDict(nlpDir).catch((err) => console.warn('[NLP] Auto-download zh-CN dict failed:', err)) ensureDefaultDict(nlpDir, ctx.bundledNlpDictDir).catch((err) =>
console.warn('[NLP] Initialize zh-CN dict failed:', err)
)
server.get('/_web/nlp/pos-tags', async () => { server.get('/_web/nlp/pos-tags', async () => {
return POS_TAG_DEFINITIONS return POS_TAG_DEFINITIONS
@@ -0,0 +1,39 @@
import assert from 'node:assert/strict'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import test from 'node:test'
import { ensureDefaultDict } from './dict-manager'
test('ensureDefaultDict installs a bundled dictionary when the writable directory is empty', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
const bundledDir = path.join(root, 'bundled')
const nlpDir = path.join(root, 'writable')
fs.mkdirSync(bundledDir)
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
try {
await ensureDefaultDict(nlpDir, bundledDir)
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'bundled dictionary')
assert.equal(fs.existsSync(path.join(nlpDir, 'zh-CN.dict.tmp')), false)
} finally {
fs.rmSync(root, { recursive: true, force: true })
}
})
test('ensureDefaultDict preserves an existing writable dictionary', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
const bundledDir = path.join(root, 'bundled')
const nlpDir = path.join(root, 'writable')
fs.mkdirSync(bundledDir)
fs.mkdirSync(nlpDir)
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), 'existing dictionary')
try {
await ensureDefaultDict(nlpDir, bundledDir)
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'existing dictionary')
} finally {
fs.rmSync(root, { recursive: true, force: true })
}
})
+13 -1
View File
@@ -152,8 +152,20 @@ export function deleteDict(nlpDir: string, dictId: string): { success: boolean;
/** /**
* 自动下载默认词库(应用启动时调用) * 自动下载默认词库(应用启动时调用)
*/ */
export async function ensureDefaultDict(nlpDir: string): Promise<void> { export async function ensureDefaultDict(nlpDir: string, bundledDictDir?: string): Promise<void> {
if (isDictDownloaded(nlpDir, 'zh-CN')) return if (isDictDownloaded(nlpDir, 'zh-CN')) return
if (bundledDictDir) {
const bundledPath = getDictFilePath(bundledDictDir, 'zh-CN')
if (fs.existsSync(bundledPath)) {
fs.mkdirSync(nlpDir, { recursive: true })
const filePath = getDictFilePath(nlpDir, 'zh-CN')
const tmpPath = filePath + '.tmp'
fs.copyFileSync(bundledPath, tmpPath)
fs.renameSync(tmpPath, filePath)
console.log('[NLP DictManager] Installed bundled zh-CN dictionary')
return
}
}
console.log('[NLP DictManager] zh-CN dict not found, starting background download...') console.log('[NLP DictManager] zh-CN dict not found, starting background download...')
const result = await downloadDict(nlpDir, 'zh-CN') const result = await downloadDict(nlpDir, 'zh-CN')
if (result.success) { if (result.success) {