mirror of
https://github.com/hellodigua/ChatLab.git
synced 2026-10-25 17:54:34 +08:00
feat(docker): preinstall default NLP dictionary
This commit is contained in:
1 parent
e2723cf751
commit
758b0667fc
9 files changed
+69
-2
No files matched your search
@@ -4,6 +4,10 @@ FROM node:24-bookworm-slim
|
|||||||
|
|
||||||
ARG CHATLAB_VERSION
|
ARG CHATLAB_VERSION
|
||||||
ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding
|
ENV CHATLAB_LOCAL_EMBEDDING_RUNTIME_DIR=/opt/chatlab/local-embedding
|
||||||
|
ENV CHATLAB_NLP_DICT_DIR=/opt/chatlab/nlp
|
||||||
|
|
||||||
|
ADD --checksum=sha256:139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f --chmod=0644 \
|
||||||
|
https://chatlab.fun/assets/nlp/zh-CN.dict /opt/chatlab/nlp/zh-CN.dict
|
||||||
|
|
||||||
RUN test -n "$CHATLAB_VERSION" \
|
RUN test -n "$CHATLAB_VERSION" \
|
||||||
&& apt-get update \
|
&& apt-get update \
|
||||||
|
|||||||
@@ -139,6 +139,7 @@ export function registerWebRoutes(
|
|||||||
runtimeIdentity: options?.runtimeIdentity,
|
runtimeIdentity: options?.runtimeIdentity,
|
||||||
getVersion,
|
getVersion,
|
||||||
nativeBinding: options?.nativeBinding,
|
nativeBinding: options?.nativeBinding,
|
||||||
|
bundledNlpDictDir: process.env.CHATLAB_NLP_DICT_DIR?.trim() || undefined,
|
||||||
semanticIndexService,
|
semanticIndexService,
|
||||||
contactsService: options?.contactsService,
|
contactsService: options?.contactsService,
|
||||||
globalInsightService: options?.globalInsightService,
|
globalInsightService: options?.globalInsightService,
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
|
|||||||
|
|
||||||
官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时,只需按界面提示下载所选模型文件,不会在容器启动后再安装约 370 MB 的 Node 依赖。
|
官方镜像已内置本地向量模型所需的运行组件。启用本地语义索引时,只需按界面提示下载所选模型文件,不会在容器启动后再安装约 370 MB 的 Node 依赖。
|
||||||
|
|
||||||
|
镜像还将默认的简体中文分词词典存放在 `/opt/chatlab/nlp`,并通过 `CHATLAB_NLP_DICT_DIR` 指向该路径,因此首次启动时无需下载。挂载的 ChatLab 目录中已有的词典会被保留。
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
|
|
||||||
### 与 Desktop / 本地 CLI 共用数据(推荐)
|
### 与 Desktop / 本地 CLI 共用数据(推荐)
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ ghcr.io/chatlab/chatlab-cli
|
|||||||
|
|
||||||
The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts.
|
The official image already includes the runtime required by local embedding models. Enabling a local semantic index only downloads the selected model files; it does not install another ~370 MB of Node dependencies after the container starts.
|
||||||
|
|
||||||
|
The image also stores the default Simplified Chinese segmentation dictionary under `/opt/chatlab/nlp` and points `CHATLAB_NLP_DICT_DIR` there, so first startup does not need to download it. An existing dictionary in the mounted ChatLab directory is preserved.
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
### Share data with Desktop and a local CLI (recommended)
|
### Share data with Desktop and a local CLI (recommended)
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ ghcr.io/chatlab/chatlab-cli
|
|||||||
|
|
||||||
官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時,只需依照介面提示下載所選模型檔案,不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。
|
官方映像已內建本地向量模型所需的執行元件。啟用本地語意索引時,只需依照介面提示下載所選模型檔案,不會在容器啟動後再安裝約 370 MB 的 Node 相依套件。
|
||||||
|
|
||||||
|
映像也將預設的簡體中文斷詞詞典存放在 `/opt/chatlab/nlp`,並透過 `CHATLAB_NLP_DICT_DIR` 指向該路徑,因此首次啟動時不需要下載。掛載的 ChatLab 目錄中既有的詞典會保留。
|
||||||
|
|
||||||
## 快速開始
|
## 快速開始
|
||||||
|
|
||||||
### 與 Desktop / 本機 CLI 共用資料(建議)
|
### 與 Desktop / 本機 CLI 共用資料(建議)
|
||||||
|
|||||||
@@ -14,4 +14,6 @@ export interface RuntimeRouteContext {
|
|||||||
runtimeIdentity?: RuntimeIdentity
|
runtimeIdentity?: RuntimeIdentity
|
||||||
/** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */
|
/** Native binding path for better-sqlite3 (CLI native copy / Electron-ABI desktop copy). */
|
||||||
nativeBinding?: string
|
nativeBinding?: string
|
||||||
|
/** Read-only directory containing dictionaries bundled by the current runtime. */
|
||||||
|
bundledNlpDictDir?: string
|
||||||
}
|
}
|
||||||
@@ -27,12 +27,15 @@ import {
|
|||||||
type NlpRouteContext = AnalyticsCacheContext & {
|
type NlpRouteContext = AnalyticsCacheContext & {
|
||||||
pathProvider: Pick<PathProvider, 'getCacheDir' | 'getSystemDir'>
|
pathProvider: Pick<PathProvider, 'getCacheDir' | 'getSystemDir'>
|
||||||
dbManager: Pick<DatabaseManager, 'open'>
|
dbManager: Pick<DatabaseManager, 'open'>
|
||||||
|
bundledNlpDictDir?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void {
|
export function registerNlpRoutes(server: FastifyInstance, ctx: NlpRouteContext): void {
|
||||||
const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp')
|
const nlpDir = path.join(ctx.pathProvider.getSystemDir(), 'nlp')
|
||||||
initNlpDir(nlpDir)
|
initNlpDir(nlpDir)
|
||||||
ensureDefaultDict(nlpDir).catch((err) => console.warn('[NLP] Auto-download zh-CN dict failed:', err))
|
ensureDefaultDict(nlpDir, ctx.bundledNlpDictDir).catch((err) =>
|
||||||
|
console.warn('[NLP] Initialize zh-CN dict failed:', err)
|
||||||
|
)
|
||||||
|
|
||||||
server.get('/_web/nlp/pos-tags', async () => {
|
server.get('/_web/nlp/pos-tags', async () => {
|
||||||
return POS_TAG_DEFINITIONS
|
return POS_TAG_DEFINITIONS
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
import os from 'node:os'
|
||||||
|
import path from 'node:path'
|
||||||
|
import test from 'node:test'
|
||||||
|
import { ensureDefaultDict } from './dict-manager'
|
||||||
|
|
||||||
|
test('ensureDefaultDict installs a bundled dictionary when the writable directory is empty', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
|
||||||
|
const bundledDir = path.join(root, 'bundled')
|
||||||
|
const nlpDir = path.join(root, 'writable')
|
||||||
|
fs.mkdirSync(bundledDir)
|
||||||
|
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
|
||||||
|
|
||||||
|
try {
|
||||||
|
await ensureDefaultDict(nlpDir, bundledDir)
|
||||||
|
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'bundled dictionary')
|
||||||
|
assert.equal(fs.existsSync(path.join(nlpDir, 'zh-CN.dict.tmp')), false)
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(root, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('ensureDefaultDict preserves an existing writable dictionary', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chatlab-nlp-bundled-'))
|
||||||
|
const bundledDir = path.join(root, 'bundled')
|
||||||
|
const nlpDir = path.join(root, 'writable')
|
||||||
|
fs.mkdirSync(bundledDir)
|
||||||
|
fs.mkdirSync(nlpDir)
|
||||||
|
fs.writeFileSync(path.join(bundledDir, 'zh-CN.dict'), 'bundled dictionary')
|
||||||
|
fs.writeFileSync(path.join(nlpDir, 'zh-CN.dict'), 'existing dictionary')
|
||||||
|
|
||||||
|
try {
|
||||||
|
await ensureDefaultDict(nlpDir, bundledDir)
|
||||||
|
assert.equal(fs.readFileSync(path.join(nlpDir, 'zh-CN.dict'), 'utf8'), 'existing dictionary')
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(root, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -152,8 +152,20 @@ export function deleteDict(nlpDir: string, dictId: string): { success: boolean;
|
|||||||
/**
|
/**
|
||||||
* 自动下载默认词库(应用启动时调用)
|
* 自动下载默认词库(应用启动时调用)
|
||||||
*/
|
*/
|
||||||
export async function ensureDefaultDict(nlpDir: string): Promise<void> {
|
export async function ensureDefaultDict(nlpDir: string, bundledDictDir?: string): Promise<void> {
|
||||||
if (isDictDownloaded(nlpDir, 'zh-CN')) return
|
if (isDictDownloaded(nlpDir, 'zh-CN')) return
|
||||||
|
if (bundledDictDir) {
|
||||||
|
const bundledPath = getDictFilePath(bundledDictDir, 'zh-CN')
|
||||||
|
if (fs.existsSync(bundledPath)) {
|
||||||
|
fs.mkdirSync(nlpDir, { recursive: true })
|
||||||
|
const filePath = getDictFilePath(nlpDir, 'zh-CN')
|
||||||
|
const tmpPath = filePath + '.tmp'
|
||||||
|
fs.copyFileSync(bundledPath, tmpPath)
|
||||||
|
fs.renameSync(tmpPath, filePath)
|
||||||
|
console.log('[NLP DictManager] Installed bundled zh-CN dictionary')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
console.log('[NLP DictManager] zh-CN dict not found, starting background download...')
|
console.log('[NLP DictManager] zh-CN dict not found, starting background download...')
|
||||||
const result = await downloadDict(nlpDir, 'zh-CN')
|
const result = await downloadDict(nlpDir, 'zh-CN')
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
|
|||||||
Reference in new issue
Block a user