diff --git a/helm-chart/.gitignore b/helm-chart/.gitignore index b5ec6e00..8c2aec04 100644 --- a/helm-chart/.gitignore +++ b/helm-chart/.gitignore @@ -1 +1,2 @@ !napcat +!.env \ No newline at end of file diff --git a/helm-chart/.gitlab-ci.yml b/helm-chart/.gitlab-ci.yml index ce6f25e5..d00ae558 100644 --- a/helm-chart/.gitlab-ci.yml +++ b/helm-chart/.gitlab-ci.yml @@ -8,35 +8,18 @@ workflow: - if: '$CI_COMMIT_BRANCH == "helm-chart"' - when: never -# 构建并推送adapter-cm-generator镜像 -build-adapter-cm-generator: +# 构建并推送processor镜像 +build-preprocessor: stage: build-image image: reg.mikumikumi.xyz/base/kaniko-builder:latest variables: BUILD_NO_CACHE: true rules: - changes: - - helm-chart/adapter-cm-generator/** + - helm-chart/preprocessor/** script: - - export BUILD_CONTEXT=helm-chart/adapter-cm-generator - - export TMP_DST=reg.mikumikumi.xyz/maibot/adapter-cm-generator - - export CHART_VERSION=$(cat helm-chart/Chart.yaml | grep '^version:' | cut -d' ' -f2) - - export BUILD_DESTINATION="${TMP_DST}:${CHART_VERSION}" - - export BUILD_ARGS="--destination ${TMP_DST}:latest" - - build - -# 构建并推送core-webui-cm-sync镜像 -build-core-webui-cm-sync: - stage: build-image - image: reg.mikumikumi.xyz/base/kaniko-builder:latest - variables: - BUILD_NO_CACHE: true - rules: - - changes: - - helm-chart/core-webui-cm-sync/** - script: - - export BUILD_CONTEXT=helm-chart/core-webui-cm-sync - - export TMP_DST=reg.mikumikumi.xyz/maibot/core-webui-cm-sync + - export BUILD_CONTEXT=helm-chart/preprocessor + - export TMP_DST=reg.mikumikumi.xyz/maibot/preprocessor - export CHART_VERSION=$(cat helm-chart/Chart.yaml | grep '^version:' | cut -d' ' -f2) - export BUILD_DESTINATION="${TMP_DST}:${CHART_VERSION}" - export BUILD_ARGS="--destination ${TMP_DST}:latest" diff --git a/helm-chart/.helmignore b/helm-chart/.helmignore index d4fe42ef..84ca3f29 100644 --- a/helm-chart/.helmignore +++ b/helm-chart/.helmignore @@ -1,3 +1,2 @@ -adapter-cm-generator -core-webui-cm-sync +preprocessor .gitlab-ci.yml \ No newline at end of file diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index ea75ef79..74cb37cb 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: maibot description: "Maimai Bot, a cyber friend dedicated to group chats" type: application -version: 0.11.5-beta -appVersion: 0.11.5-beta +version: 0.11.6-beta +appVersion: 0.11.6-beta diff --git a/helm-chart/README.md b/helm-chart/README.md index edbb417c..63a835b5 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -34,35 +34,39 @@ helm install maimai \ 1. `EULA` & `PRIVACY`: 用户必须同意这里的协议才能成功部署麦麦。 -2. `adapter`: 麦麦的Adapter的部署配置。 +2. `pre_processor`: 部署之前的预处理Job的配置。 -3. `core`: 麦麦本体的部署配置。 +3. `adapter`: 麦麦的Adapter的部署配置。 -4. `statistics_dashboard`: 麦麦的运行统计看板部署配置。 +4. `core`: 麦麦本体的部署配置。 + +5. `statistics_dashboard`: 麦麦的运行统计看板部署配置。 麦麦每隔一段时间会自动输出html格式的运行统计报告,此统计报告可以部署为看板。 出于隐私考虑,默认禁用。 -5. `napcat`: Napcat的部署配置。 +6. `napcat`: Napcat的部署配置。 考虑到复用外部Napcat实例的情况,Napcat部署已被解耦。用户可选是否要部署Napcat。 默认会捆绑部署Napcat。 -6. `sqlite_web`: sqlite-web的部署配置。 +7. `sqlite_web`: sqlite-web的部署配置。 通过sqlite-web可以在网页上操作麦麦的数据库,方便调试。不部署对麦麦的运行无影响。 此服务如果暴露在公网会十分危险,默认不会部署。 -7. `config`: 这里填写麦麦各部分组件的运行配置文件。 +8. `config`: 这里填写麦麦各部分组件的运行配置。 - 这里填写的配置文件需要严格遵守yaml文件的缩进格式。 + 这里填写的配置仅会在初次部署时或用户指定时覆盖实际配置文件,且需要严格遵守yaml文件的缩进格式。 + + - `override_*_config`: 指定本次部署/升级是否用以下配置覆盖实际配置文件。默认不覆盖。 - `adapter_config`: 对应adapter的`config.toml`。 - 此配置文件中对于`host`和`port`的配置会被上面`adapter.service`中的配置覆盖,因此不需要改动。 + 此配置文件中对于`napcat_server`和`maibot_server`的`host`和`port`字段的配置会被上面`adapter.service`中的配置覆盖,因此不需要改动。 - `core_model_config`: 对应core的`model_config.toml`。 @@ -72,22 +76,22 @@ helm install maimai \ 使用此Helm Chart的一些注意事项。 -### 修改麦麦配置 +### 麦麦的配置 -麦麦的配置文件会通过ConfigMap资源注入各个组件内。 +要修改麦麦的配置,最好的方法是通过WebUI来操作。此处的配置只会在初次部署时或者指定覆盖时注入到MaiBot中。 -对于通过Helm Chart部署的麦麦,如果需要修改配置,不应该直接修改这些ConfigMap,否则下次Helm更新可能会覆盖掉所有配置。 +`0.11.6-beta`之前的版本将配置存储于k8s的ConfigMap资源中。随着版本迭代,MaiBot对配置文件的操作复杂性增加,k8s的适配复杂度也同步增加,且WebUI可以直接修改配置文件,因此自`0.11.6-beta`版本开始,各组件的配置不再存储于k8s的ConfigMap中,而是直接存储于存储卷的实际文件中。 -最佳实践是重新配置Helm Chart的values,然后通过`helm upgrade`更新实例。 +从旧版本升级的用户,旧的ConfigMap的配置会自动迁移到新的存储卷的配置文件中。 ### 动态生成的ConfigMap -adapter的ConfigMap是每次部署/更新Helm安装实例时动态生成的。 +adapter的配置中的`napcat_server`和`maibot_server`的`host`和`port`字段,会在每次部署/更新Helm安装实例时被自动重置。 -动态生成的原因: +自动重置的原因: -- core服务的DNS名称是动态的,无法在adapter服务的配置文件中提前确定。 -- 一些与k8s现有资源冲突的配置需要被重置。 +- core的Service的DNS名称是动态的(由安装实例名拼接),无法在adapter的配置文件中提前确定。 +- 为了使adapter监听所有地址以及保持Helm Chart中配置的端口号,需要在adapter的配置文件中覆盖这些配置。 因此,首次部署时,ConfigMap的生成会需要一些时间,部分Pod会无法启动,等待几分钟即可。 @@ -101,31 +105,4 @@ adapter的ConfigMap是每次部署/更新Helm安装实例时动态生成的。 如果你的存储底层无法支持`ReadWriteMany`访问模式,你可以通过`nodeSelector`配置将statistics_dashboard与core调度到同一节点来避免问题。 -*如果启用了`sqlite-web`,那么上述问题也同样适用于`sqlite-web`与`core`,需要注意。* - -### 麦麦的默认插件 - -麦麦的`core`容器提供了一些默认插件,以提升使用体验。但是插件目录存储在存储卷中,容器启动时挂载的存储卷会完全覆盖掉容器的默认插件目录,导致默认插件无法加载,也难以被用户感知。 - -为了解决这一问题,此Helm Chart中为`core`容器引入了初始化容器。此初始化容器用于为用户自动安装默认插件到存储卷中。可以选择启用(默认启用)。 - -*初始化容器使用与`core`主容器相同的镜像,且用后即销毁,因此不会消耗额外的带宽和存储成本。* - -#### 触发插件安装的条件 - -- 首次部署时(此时没有任何插件处于安装状态) -- 默认插件更新(即默认插件内容发生变化) - -#### 安装状态识别能力 - -初始化容器会记录安装过的默认插件,不会重复安装。为了实现这一点,初始化容器会将安装状态写入`/MaiMBot/data/plugins/.installed-setup-plugins`文件中。 - -基于上述状态识别能力,如果用户不需要某个插件,可以将其删除。由于此插件已自动安装过(记录在状态文件中),即使插件本体不存在也不会再次安装(除非插件更新)。 - -#### 插件更新 - -一旦在镜像中检测到新版本插件(即插件内容不同),初始化容器即会用新插件覆盖旧插件。 - -考虑到旧插件中可能存在用户自定义配置,因此旧插件在被覆盖前会备份到`/MaiMBot/data/plugins-backup`目录中,并以时间归档。 - -因此在升级麦麦后,请注意观察初始容器的日志并重新配置插件。 +*对于预处理任务和`sqlite-web`,上述问题也同样会出现,需要注意。* diff --git a/helm-chart/adapter-cm-generator/adapter-cm-generator.py b/helm-chart/adapter-cm-generator/adapter-cm-generator.py deleted file mode 100644 index 95d7b65b..00000000 --- a/helm-chart/adapter-cm-generator/adapter-cm-generator.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/python3 -# 这个脚本的作用是在部署helm chart时动态生成adapter的配置文件,保存在configmap中 -# 需要动态生成的原因是core服务的DNS名称是动态的,无法在adapter服务的配置文件中提前确定 -# 一些与k8s现有资源冲突的配置也会在这里重置 - -import os -import toml -import base64 -from kubernetes import client, config -from datetime import datetime, timezone - -config.load_incluster_config() -core_api = client.CoreV1Api() -apps_api = client.AppsV1Api() - -# 读取部署的关键信息 -namespace = os.getenv("NAMESPACE") -release_name = os.getenv("RELEASE_NAME") -data_b64 = os.getenv("DATA_B64") - -# 解析并覆盖关键配置 -# 这里被覆盖的配置应当在helm chart中针对对应的k8s资源来灵活修改 -data = toml.loads(base64.b64decode(data_b64).decode("utf-8")) -data.setdefault('napcat_server', {}) -data['napcat_server']['host'] = '0.0.0.0' -data['napcat_server']['port'] = 8095 -data.setdefault('maibot_server', {}) -data['maibot_server']['host'] = f'{release_name}-maibot-core' # 根据release名称动态拼接core服务的DNS名称 -data['maibot_server']['port'] = 8000 - -# 创建/修改configmap -cm_name = f'{release_name}-maibot-adapter-config' -cm = client.V1ConfigMap( - metadata=client.V1ObjectMeta(name=cm_name), - data={'config.toml': toml.dumps(data)} -) -try: - core_api.create_namespaced_config_map(namespace, cm) - print(f"ConfigMap `{cm_name}` created successfully") -except client.exceptions.ApiException as e: - if e.status == 409: # 已存在,更新 - core_api.replace_namespaced_config_map(cm_name, namespace, cm) - print(f"ConfigMap `{cm_name}` replaced successfully") - else: - raise - -# 重启adapter和core的statefulset -now = datetime.now(timezone.utc).isoformat() -body = { - "spec": { - "template": { - "metadata": { - "annotations": { - "kubectl.kubernetes.io/restartedAt": now - } - } - } - } -} -apps_api.patch_namespaced_stateful_set( - name=f'{release_name}-maibot-adapter', - namespace=namespace, - body=body, -) -print(f"StatefulSet `{release_name}-maibot-adapter` restarted successfully") -apps_api.patch_namespaced_stateful_set( - name=f'{release_name}-maibot-core', - namespace=namespace, - body=body, -) -print(f"StatefulSet `{release_name}-maibot-core` restarted successfully") - -print('Job succeed.') diff --git a/helm-chart/core-webui-cm-sync/Dockerfile b/helm-chart/core-webui-cm-sync/Dockerfile deleted file mode 100644 index 55a8c25b..00000000 --- a/helm-chart/core-webui-cm-sync/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -# 此镜像用于辅助麦麦的WebUI更新配置文件,随core容器持续运行 -FROM python:3.13-slim - -WORKDIR /MaiMBot - -COPY . /MaiMBot - -RUN pip3 install --no-cache-dir -r requirements.txt - -ENTRYPOINT ["python3", "core-webui-cm-sync.py"] diff --git a/helm-chart/core-webui-cm-sync/core-webui-cm-sync.py b/helm-chart/core-webui-cm-sync/core-webui-cm-sync.py deleted file mode 100644 index e179d6f6..00000000 --- a/helm-chart/core-webui-cm-sync/core-webui-cm-sync.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/python3 -# 这个程序的作用是辅助麦麦的WebUI更新配置文件,随core容器持续运行。 -# 麦麦的配置文件存储于ConfigMap中,挂载进core容器后属于只读文件,无法直接修改。 -# 此程序将core容器内的配置文件替换为可读写的中间层临时文件。启动时将实际配置文件写入,并在后台持续检测文件变化,实时同步到k8s apiServer,反向修改ConfigMap。 -# 工作目录:/MaiMBot/webui-cm-sync - -import os -import time -from datetime import datetime -from kubernetes import client, config -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -work_dir = '/MaiMBot/webui-cm-sync' -os.chdir(work_dir) - -config.load_incluster_config() -core_api = client.CoreV1Api() -with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r") as f: - namespace = f.read().strip() -release_name = os.getenv("RELEASE_NAME") -model_configmap_name = f'{release_name}-maibot-core-model-config' -bot_configmap_name = f'{release_name}-maibot-core-bot-config' - -# 过滤列表,只监控指定文件 -target_files = { - os.path.abspath("model_config.toml"): (model_configmap_name, "model_config.toml"), - os.path.abspath("bot_config.toml"): (bot_configmap_name, "bot_config.toml") -} - - -def get_configmap(configmap_name: str): - """获取core的ConfigMap内容""" - cm = core_api.read_namespaced_config_map(name=configmap_name, namespace=namespace) - return cm.data - - -def set_configmap(configmap_name: str, configmap_data: dict[str, str]): - """设置core的ConfigMap内容""" - core_api.patch_namespaced_config_map(configmap_name, namespace, {'data': configmap_data}) - - -class ConfigObserverHandler(FileSystemEventHandler): - """配置文件变化的事件处理器""" - def on_modified(self, event): - if os.path.abspath(event.src_path) in target_files: - print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] File `{event.src_path}` was modified. ' - f'Start to sync...') - with open(event.src_path, "r", encoding="utf-8") as _f: - current_data = _f.read() - _path = str(os.path.abspath(event.src_path)) - new_cm = { - target_files[_path][1]: current_data - } - try: - set_configmap(target_files[_path][0], new_cm) - print(f'\tSync done.') - except client.exceptions.ApiException as _e: - print(f'\tError while setting configmap:\n' - f'\t\tStatus Code: {_e.status}\n' - f'\t\tReason: {_e.reason}') - - -if __name__ == '__main__': - # 初始化配置文件 - print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] Initializing config files...') - try: - __initial_model_config = get_configmap(model_configmap_name)['model_config.toml'] - __initial_bot_config = get_configmap(bot_configmap_name)['bot_config.toml'] - except client.exceptions.ApiException as e: - print(f'\tError while getting configmap:\n' - f'\t\tStatus Code: {e.status}\n' - f'\t\tReason: {e.reason}') - exit(1) - with open('model_config.toml', 'w') as f: - f.write(__initial_model_config) - with open('bot_config.toml', 'w') as f: - f.write(__initial_bot_config) - with open('ready', 'w') as f: - f.write('true') - print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] Initializing done. Ready to sync.') - - # 持续检测变化并同步 - observer = Observer() - observer.schedule(ConfigObserverHandler(), work_dir, recursive=False) - observer.start() - try: - while True: - time.sleep(1) - except KeyboardInterrupt: - observer.stop() - observer.join() diff --git a/helm-chart/core-webui-cm-sync/requirements.txt b/helm-chart/core-webui-cm-sync/requirements.txt deleted file mode 100644 index 58c73d28..00000000 --- a/helm-chart/core-webui-cm-sync/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -toml~=0.10.2 -kubernetes~=34.1.0 -watchdog~=6.0.0 diff --git a/helm-chart/files/.env b/helm-chart/files/.env new file mode 100644 index 00000000..a5d1eedf --- /dev/null +++ b/helm-chart/files/.env @@ -0,0 +1,6 @@ +HOST=0.0.0.0 +PORT=8000 +WEBUI_ENABLED={{ if .Values.core.webui.enabled }}true{{ else }}false{{ end }} +WEBUI_MODE=production +WEBUI_HOST=0.0.0.0 +WEBUI_PORT=8001 diff --git a/helm-chart/files/k8s-init.sh b/helm-chart/files/k8s-init.sh index 3729a68f..3fd805f4 100644 --- a/helm-chart/files/k8s-init.sh +++ b/helm-chart/files/k8s-init.sh @@ -1,12 +1,8 @@ #!/bin/sh # 此脚本用于覆盖core容器的默认启动命令,进行一些初始化 -# 1 # 由于k8s与docker-compose的卷挂载方式有所不同,需要利用此脚本为一些文件和目录提前创建好软链接 # /MaiMBot/data是麦麦数据的实际挂载路径 # /MaiMBot/statistics是统计数据的实际挂载路径 -# 2 -# 此脚本等待辅助容器webui-cm-sync就绪后再启动麦麦 -# 通过检测/MaiMBot/webui-cm-sync/ready文件来判断 set -e echo "[K8s Init] Preparing volume..." @@ -34,22 +30,6 @@ fi echo "[K8s Init] Volume ready." -# 如果启用了WebUI,则等待辅助容器webui-cm-sync就绪,然后创建中间层配置文件软链接 -if [ "$MAIBOT_WEBUI_ENABLED" = "true" ] -then - echo "[K8s Init] WebUI enabled. Waiting for container 'webui-cm-sync' ready..." - while [ ! -f /MaiMBot/webui-cm-sync/ready ]; do - sleep 1 - done - echo "[K8s Init] Container 'webui-cm-sync' ready." - mkdir -p /MaiMBot/config - ln -s /MaiMBot/webui-cm-sync/model_config.toml /MaiMBot/config/model_config.toml - ln -s /MaiMBot/webui-cm-sync/bot_config.toml /MaiMBot/config/bot_config.toml - echo "[K8s Init] Config files middle layer for WebUI created." -else - echo "[K8s Init] WebUI is disabled." -fi - # 启动麦麦 echo "[K8s Init] Waking up MaiBot..." echo diff --git a/helm-chart/files/setup-plugins.py b/helm-chart/files/setup-plugins.py deleted file mode 100644 index 9fde6804..00000000 --- a/helm-chart/files/setup-plugins.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/local/bin/python3 -# 用户插件目录存储在存储卷中,会在启动时覆盖掉容器的默认插件目录。此脚本用于默认插件更新后或麦麦首次启动时为用户自动安装默认插件到存储卷中 -# 如果用户主动删除插件且插件无更新,则不会再次安装。插件状态保存在/MaiMBot/data/plugins/.installed-setup-plugins文件中 -# 此脚本应当挂载进初始化容器中,从/MaiMBot工作路径开始运行。初始化容器的镜像同core容器,初始化容器中应挂载core存储卷的数据到/MaiMBot/data -import os -import shutil -import hashlib -from datetime import datetime - -SRC_DIR = '/MaiMBot/plugins' -DST_DIR = '/MaiMBot/data/plugins' -STATUS_FILE = f'{DST_DIR}/.installed-setup-plugins' -BAK_DIR = '/MaiMBot/data/plugins-backup' -CURRENT_TIME = datetime.now().strftime('%Y%m%d%H%M%S') - -def hash_dir_file(path: str): - """计算目录/文件的SHA256,用于判断是否发生变化""" - def hash_file(_file_path: str): - _h = hashlib.sha256() - with open(_file_path, 'rb') as _f: - for _chunk in iter(lambda: _f.read(8192), b''): - _h.update(_chunk) - return _h.hexdigest() - - if os.path.isfile(path): - return hash_file(path) - - h = hashlib.sha256() - for root, dirs, files in os.walk(path): - for filename in sorted(files): - filepath = os.path.join(root, filename) - relpath = os.path.relpath(filepath, path) - file_hash = hash_file(filepath) - h.update(relpath.encode('utf-8')) - h.update(file_hash.encode('utf-8')) - return h.hexdigest() - -def copy_plugin(plugin: str): - """复制插件,如果插件已存在则备份旧的插件然后用新的插件覆盖""" - src = os.path.join(SRC_DIR, plugin) - if not os.path.exists(src): - raise FileNotFoundError(f"File not found: {src}") - - dst = os.path.join(DST_DIR, plugin) - if os.path.exists(dst): - print(f"\t\tWarning: Old version of plugin '{plugin}' already exists. " - f"Old plugin will be moved to '{BAK_DIR}/{CURRENT_TIME}/{plugin}'. " - f"Remember to re-edit config of this plugin.") - if not os.path.exists(os.path.join(BAK_DIR, CURRENT_TIME)): - os.makedirs(os.path.join(BAK_DIR, CURRENT_TIME)) - if os.path.isdir(dst): - shutil.copytree(dst, os.path.join(BAK_DIR, CURRENT_TIME, plugin)) - shutil.rmtree(dst) - else: - shutil.copy2(dst, os.path.join(BAK_DIR, CURRENT_TIME)) - os.remove(dst) - - if os.path.isdir(src): - shutil.copytree(src, dst) - else: - shutil.copy2(src, DST_DIR) - -setup_plugins = {plugin: hash_dir_file(plugin) for plugin in os.listdir(SRC_DIR)} -installed_plugins = {} -to_install_plugins = {} - -print(f"[SetupPlugins] Default plugin, which has been updated or never been installed, " - f"will be installed in this init container.") -if os.path.exists(STATUS_FILE) and os.path.isfile(STATUS_FILE): - print(f"[SetupPlugins] Reading status file: '{STATUS_FILE}'...") - with open(STATUS_FILE, 'r', encoding='utf-8') as f: - lines = f.readlines() - for line in lines: - if line == '': - continue - plugin = line.strip().split(':') - installed_plugins[plugin[0]] = plugin[1] - print(f"[SetupPlugins] Found {len(installed_plugins)} default plugins which used to be installed:") - for plugin in installed_plugins.keys(): - print(f'\t{plugin}') -else: - print(f"[SetupPlugins] No status file found. Status file '{STATUS_FILE}' will be created. " - f"All default plugins will be installed now.") - -print(f"[SetupPlugins] Checking plugins...") -for plugin, sha256 in setup_plugins.items(): - if (plugin not in installed_plugins) or (sha256 != installed_plugins[plugin]): - print(f"\tFound default plugin to install: '{plugin}'. Installing...") - copy_plugin(plugin) - installed_plugins[plugin] = sha256 - -with open(STATUS_FILE, 'w', encoding='utf-8') as f: - f.write('\n'.join(sorted([f'{plugin}:{sha256}' for plugin, sha256 in installed_plugins.items()]))) - -print(f"[SetupPlugins] Default plugin checking done. Status saved to '{STATUS_FILE}'.") diff --git a/helm-chart/adapter-cm-generator/Dockerfile b/helm-chart/preprocessor/Dockerfile similarity index 72% rename from helm-chart/adapter-cm-generator/Dockerfile rename to helm-chart/preprocessor/Dockerfile index 3149f0f5..58452dd5 100644 --- a/helm-chart/adapter-cm-generator/Dockerfile +++ b/helm-chart/preprocessor/Dockerfile @@ -3,8 +3,10 @@ FROM python:3.13-slim WORKDIR /app +ENV PYTHONUNBUFFERED=1 + COPY . /app RUN pip3 install --no-cache-dir -r requirements.txt -ENTRYPOINT ["python3", "adapter-cm-generator.py"] +ENTRYPOINT ["python3", "preprocessor.py"] diff --git a/helm-chart/preprocessor/preprocessor.py b/helm-chart/preprocessor/preprocessor.py new file mode 100644 index 00000000..0edef8bd --- /dev/null +++ b/helm-chart/preprocessor/preprocessor.py @@ -0,0 +1,241 @@ +#!/bin/python3 +# 此脚本会被helm chart的post-install hook触发,在正式部署后通过k8s的job自动运行一次。 +# 这个脚本的作用是在部署helm chart时迁移旧版ConfigMap到配置文件,并调整adapter的配置文件中的服务监听和服务连接字段。 +# - 迁移旧版ConfigMap到配置文件是因为0.11.6-beta之前版本的helm chart将各个配置文件存储在k8s的ConfigMap中, +# 由于功能复杂度提升,自0.11.6-beta版本开始配置文件采用文件形式存储到存储卷中。 +# 从旧版升级来的用户会通过这个脚本自动执行配置的迁移。 +# - 需要调整adapter的配置文件的原因是: +# 1. core的Service的DNS名称是动态的(由安装实例名拼接),无法在adapter的配置文件中提前确定。 +# 用于对外连接的maibot_server.host和maibot_server.port字段,会被替换为core的Service对应的DNS名称和8000端口(硬编码,用户无需配置)。 +# 2. 为了使adapter监听所有地址以及保持chart中配置的端口号,需要在adapter的配置文件中覆盖这些配置。 +# 用于监听的napcat_server.host和napcat_server.port字段,会被替换为0.0.0.0和8095端口(实际映射到的Service端口会在Service中配置)。 + +import os +import toml +import time +import base64 +from kubernetes import client, config +from kubernetes.client.exceptions import ApiException +from datetime import datetime, timezone + +config.load_incluster_config() +core_api = client.CoreV1Api() +apps_api = client.AppsV1Api() + +# 读取部署的关键信息 +with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", 'r') as f: + namespace = f.read().strip() +release_name = os.getenv("RELEASE_NAME").strip() +config_adapter_b64 = os.getenv("CONFIG_ADAPTER_B64") +config_core_env_b64 = os.getenv("CONFIG_CORE_ENV_B64") +config_core_bot_b64 = os.getenv("CONFIG_CORE_BOT_B64") +config_core_model_b64 = os.getenv("CONFIG_CORE_MODEL_B64") + + +def log(func: str, msg: str, level: str = 'INFO'): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}][{func}][{level}] {msg}') + + +def migrate_old_config(): + """迁移旧版配置""" + func_name = 'migrate_old_config' + log(func_name, 'Checking whether there are old configmaps to migrate...') + old_configmap_version = None + status_migrating = { # 存储adapter的config.toml、core的bot_config.toml和model_config.toml三个文件的迁移状态 + 'adapter_config.toml': False, + 'core_bot_config.toml': False, + 'core_model_config.toml': False + } + + # 如果存储卷中已存在配置文件,则警告并跳过迁移 + if os.path.isfile('/app/config/core/bot_config.toml') or os.path.isfile('/app/config/core/model_config.toml') or \ + os.path.isfile('/app/config/adapter/config.toml'): + log(func_name, 'Found existing config file(s) in PV. Migration will be skipped. ' + 'Please check the configs MANUALLY.', level='WARNING') + log(func_name, '\tIf configs were ALREADY MIGRATED, you can delete old configmaps safely.', level='WARNING') + log(func_name, '\tIf configs were NOT MIGRATED, you should MIGRATE them MANUALLY, then delete the configmaps.', + level='WARNING') + return + + def migrate_cm_to_file(cm_name: str, key_name: str, file_path: str) -> bool: + """检测是否有指定名称的configmap,如果有的话备份到指定的配置文件里并删除configmap,返回是否已备份""" + try: + cm = core_api.read_namespaced_config_map( + name=cm_name, + namespace=namespace + ) + log(func_name, f'\tMigrating `{key_name}` of `{cm_name}`...') + with open(file_path, 'w', encoding='utf-8') as _f: + _f.write(cm.data[key_name]) + core_api.delete_namespaced_config_map( + name=cm_name, + namespace=namespace + ) + log(func_name, f'\tSuccessfully migrated `{key_name}` of `{cm_name}`.') + except ApiException as e: + if e.status == 404: + return False + return True + + # 对于0.11.5-beta版本,adapter的config.toml、core的bot_config.toml和model_config.toml均存储于不同的ConfigMap,需要依次迁移 + if True not in status_migrating.values(): + status_migrating['adapter_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-adapter-config', + 'config.toml', + '/app/config/adapter/config.toml') + status_migrating['core_bot_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-core-bot-config', + 'bot_config.toml', + '/app/config/core/bot_config.toml') + status_migrating['core_model_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-core-model-config', + 'model_config.toml', + '/app/config/core/model_config.toml') + if True in status_migrating.values(): + old_configmap_version = '0.11.5-beta' + + # 对于低于0.11.5-beta的版本,adapter的1个配置和core的3个配置位于各自的configmap中 + if True not in status_migrating.values(): + status_migrating['adapter_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-adapter', + 'config.toml', + '/app/config/adapter/config.toml') + status_migrating['core_bot_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-core', + 'bot_config.toml', + '/app/config/core/bot_config.toml') + status_migrating['core_model_config.toml'] = migrate_cm_to_file(f'{release_name}-maibot-core', + 'model_config.toml', + '/app/config/core/model_config.toml') + if True in status_migrating.values(): + old_configmap_version = 'before 0.11.5-beta' + + if old_configmap_version: + log(func_name, f'Migrating status for version `{old_configmap_version}`:') + for k, v in status_migrating.items(): + log(func_name, f'\t{k}: {v}') + if False in status_migrating.values(): + log(func_name, 'There is/are config(s) that not been migrated. Please check the config manually.', + level='WARNING') + else: + log(func_name, 'Successfully migrated old configs. Done.') + else: + log(func_name, 'Old config not found. Ignoring migration. Done.') + + +def write_config_files(): + """当注入了配置文件时(一般是首次安装或者用户指定覆盖),将helm chart注入的配置写入存储卷中的实际文件""" + func_name = 'write_config_files' + log(func_name, 'Detecting config files...') + if config_adapter_b64.strip(): + log(func_name, '\tWriting `config.toml` of adapter...') + config_str = base64.b64decode(config_adapter_b64).decode("utf-8") + with open('/app/config/adapter/config.toml', 'w', encoding='utf-8') as _f: + _f.write(config_str) + log(func_name, '\t`config.toml` of adapter wrote.') + if True: # .env直接覆盖 + log(func_name, '\tWriting .env file of core...') + config_str = base64.b64decode(config_core_env_b64).decode("utf-8") + with open('/app/config/core/.env', 'w', encoding='utf-8') as _f: + _f.write(config_str) + log(func_name, '\t`.env` of core wrote.') + if config_core_bot_b64.strip(): + log(func_name, '\tWriting `bot_config.toml` of core not found. Creating...') + config_str = base64.b64decode(config_core_bot_b64).decode("utf-8") + with open('/app/config/core/bot_config.toml', 'w', encoding='utf-8') as _f: + _f.write(config_str) + log(func_name, '\t`bot_config.toml` of core wrote.') + if config_core_model_b64.strip(): + log(func_name, '\tWriting `model_config.toml` of core...') + config_str = base64.b64decode(config_core_model_b64).decode("utf-8") + with open('/app/config/core/model_config.toml', 'w', encoding='utf-8') as _f: + _f.write(config_str) + log(func_name, '\t`model_config.toml` of core wrote.') + log(func_name, 'Detection done.') + + +def reconfigure_adapter(): + """调整adapter的配置文件的napcat_server和maibot_server字段,使其Service能被napcat连接以及连接到core的Service""" + func_name = 'reconfigure_adapter' + log(func_name, 'Reconfiguring `config.toml` of adapter...') + with open('/app/config/adapter/config.toml', 'r', encoding='utf-8') as _f: + config_adapter = toml.load(_f) + config_adapter.setdefault('napcat_server', {}) + config_adapter['napcat_server']['host'] = '0.0.0.0' + config_adapter['napcat_server']['port'] = 8095 + config_adapter.setdefault('maibot_server', {}) + config_adapter['maibot_server']['host'] = f'{release_name}-maibot-core' # 根据release名称动态拼接core服务的DNS名称 + config_adapter['maibot_server']['port'] = 8000 + with open('/app/config/adapter/config.toml', 'w', encoding='utf-8') as _f: + _f.write(toml.dumps(config_adapter)) + log(func_name, 'Reconfiguration done.') + + +def _scale_statefulsets(statefulsets: list[str], replicas: int, wait: bool = False, timeout: int = 300): + """调整指定几个statefulset的副本数,wait参数控制是否等待调整完成再返回""" + statefulsets = set(statefulsets) + for name in statefulsets: + apps_api.patch_namespaced_stateful_set_scale( + name=name, + namespace=namespace, + body={"spec": {"replicas": replicas}} + ) + if not wait: + return + + start_time = time.time() + while True: + remaining_pods = [] + + pods = core_api.list_namespaced_pod(namespace).items + + for pod in pods: + owners = pod.metadata.owner_references or [] + for owner in owners: + if owner.kind == "StatefulSet" and owner.name in statefulsets: + remaining_pods.append(pod.metadata.name) + + if not remaining_pods: + return + + elapsed = time.time() - start_time + if elapsed > timeout: + raise TimeoutError( + f"Timeout waiting for Pods to be deleted. " + f"Remaining Pods: {remaining_pods}" + ) + time.sleep(5) + + +def _restart_statefulset(name: str, ignore_error: bool = False): + """重启指定的statefulset""" + now = datetime.now(timezone.utc).isoformat() + body = { + "spec": { + "template": { + "metadata": { + "annotations": { + "kubectl.kubernetes.io/restartedAt": now + } + } + } + } + } + try: + apps_api.patch_namespaced_stateful_set( + name=name, + namespace=namespace, + body=body + ) + except ApiException as e: + if ignore_error: + pass + else: + raise e + + +if __name__ == '__main__': + log('main', 'Start to process data before install/upgrade...') + log('main', 'Scaling adapter and core to 0...') + _scale_statefulsets([f'{release_name}-maibot-adapter', f'{release_name}-maibot-core'], 0, wait=True) + migrate_old_config() + write_config_files() + reconfigure_adapter() + log('main', 'Scaling adapter and core to 1...') + _scale_statefulsets([f'{release_name}-maibot-adapter', f'{release_name}-maibot-core'], 1) + log('main', 'Process done.') diff --git a/helm-chart/adapter-cm-generator/requirements.txt b/helm-chart/preprocessor/requirements.txt similarity index 100% rename from helm-chart/adapter-cm-generator/requirements.txt rename to helm-chart/preprocessor/requirements.txt diff --git a/helm-chart/templates/adapter/pvc.yaml b/helm-chart/templates/adapter/pvc.yaml index 61778630..4bc24506 100644 --- a/helm-chart/templates/adapter/pvc.yaml +++ b/helm-chart/templates/adapter/pvc.yaml @@ -4,13 +4,30 @@ metadata: name: {{ .Release.Name }}-maibot-adapter namespace: {{ .Release.Namespace }} spec: - {{- if .Values.adapter.persistence.accessModes }} + {{- if .Values.adapter.persistence.data.accessModes }} accessModes: - {{ toYaml .Values.adapter.persistence.accessModes | nindent 4 }} + {{ toYaml .Values.adapter.persistence.data.accessModes | nindent 4 }} {{- end }} resources: requests: - storage: {{ .Values.adapter.persistence.size }} - {{- if .Values.adapter.persistence.storageClass }} - storageClassName: {{ .Values.adapter.persistence.storageClass | default nil }} + storage: {{ .Values.adapter.persistence.data.size }} + {{- if .Values.adapter.persistence.data.storageClass }} + storageClassName: {{ .Values.adapter.persistence.data.storageClass | default nil }} + {{- end }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-maibot-adapter-config + namespace: {{ .Release.Namespace }} +spec: + {{- if .Values.adapter.persistence.config.accessModes }} + accessModes: + {{ toYaml .Values.adapter.persistence.config.accessModes | nindent 4 }} + {{- end }} + resources: + requests: + storage: {{ .Values.adapter.persistence.config.size }} + {{- if .Values.adapter.persistence.config.storageClass }} + storageClassName: {{ .Values.adapter.persistence.config.storageClass | default nil }} {{- end }} diff --git a/helm-chart/templates/adapter/statefulset.yaml b/helm-chart/templates/adapter/statefulset.yaml index 0d9b76da..d5981b8e 100644 --- a/helm-chart/templates/adapter/statefulset.yaml +++ b/helm-chart/templates/adapter/statefulset.yaml @@ -7,7 +7,7 @@ metadata: app: {{ .Release.Name }}-maibot-adapter spec: serviceName: {{ .Release.Name }}-maibot-adapter - replicas: 1 + replicas: 0 # post-install任务初始化完毕后自动扩容至1 selector: matchLabels: app: {{ .Release.Name }}-maibot-adapter @@ -36,7 +36,6 @@ spec: name: data - mountPath: /adapters/config.toml name: config - readOnly: true subPath: config.toml {{- if .Values.adapter.image.pullSecrets }} imagePullSecrets: @@ -54,9 +53,6 @@ spec: - name: data persistentVolumeClaim: claimName: {{ .Release.Name }}-maibot-adapter - - configMap: - items: - - key: config.toml - path: config.toml - name: {{ .Release.Name }}-maibot-adapter-config - name: config + - name: config + persistentVolumeClaim: + claimName: {{ .Release.Name }}-maibot-adapter-config diff --git a/helm-chart/templates/core/configmap-bot-config.yaml b/helm-chart/templates/core/configmap-bot-config.yaml deleted file mode 100644 index d1395296..00000000 --- a/helm-chart/templates/core/configmap-bot-config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if or .Release.IsInstall (and .Values.core.webui.enabled .Values.config.enable_config_override_with_webui) (and (not .Values.core.webui.enabled) .Values.config.enable_config_override_without_webui) }} -# 渲染规则: -# 初次安装,或配置了覆盖规则 -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-maibot-core-bot-config - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/resource-policy": keep -data: - bot_config.toml: | - {{ .Values.config.core_bot_config | nindent 4 }} -{{- end }} diff --git a/helm-chart/templates/core/configmap-env-config.yaml b/helm-chart/templates/core/configmap-env-config.yaml deleted file mode 100644 index 17b1d627..00000000 --- a/helm-chart/templates/core/configmap-env-config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-maibot-core-env-config - namespace: {{ .Release.Namespace }} -data: - .env: | - HOST=0.0.0.0 - PORT=8000 - WEBUI_ENABLED={{ if .Values.core.webui.enabled }}true{{ else }}false{{ end }} - WEBUI_MODE=production - WEBUI_HOST=0.0.0.0 - WEBUI_PORT=8001 diff --git a/helm-chart/templates/core/configmap-model-config.yaml b/helm-chart/templates/core/configmap-model-config.yaml deleted file mode 100644 index bf122244..00000000 --- a/helm-chart/templates/core/configmap-model-config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if or .Release.IsInstall (and .Values.core.webui.enabled .Values.config.enable_config_override_with_webui) (and (not .Values.core.webui.enabled) .Values.config.enable_config_override_without_webui) }} -# 渲染规则: -# 初次安装,或配置了覆盖规则 -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-maibot-core-model-config - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/resource-policy": keep -data: - model_config.toml: | - {{ .Values.config.core_model_config | nindent 4 }} -{{- end }} diff --git a/helm-chart/templates/core/pvc.yaml b/helm-chart/templates/core/pvc.yaml index e7fe15c7..66c6ac12 100644 --- a/helm-chart/templates/core/pvc.yaml +++ b/helm-chart/templates/core/pvc.yaml @@ -4,13 +4,30 @@ metadata: name: {{ .Release.Name }}-maibot-core namespace: {{ .Release.Namespace }} spec: - {{- if .Values.core.persistence.accessModes }} + {{- if .Values.core.persistence.data.accessModes }} accessModes: - {{ toYaml .Values.core.persistence.accessModes | nindent 4 }} + {{ toYaml .Values.core.persistence.data.accessModes | nindent 4 }} {{- end }} resources: requests: - storage: {{ .Values.core.persistence.size }} - {{- if .Values.core.persistence.storageClass }} - storageClassName: {{ .Values.core.persistence.storageClass | default nil }} + storage: {{ .Values.core.persistence.data.size }} + {{- if .Values.core.persistence.data.storageClass }} + storageClassName: {{ .Values.core.persistence.data.storageClass | default nil }} + {{- end }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-maibot-core-config + namespace: {{ .Release.Namespace }} +spec: + {{- if .Values.core.persistence.config.accessModes }} + accessModes: + {{ toYaml .Values.core.persistence.config.accessModes | nindent 4 }} + {{- end }} + resources: + requests: + storage: {{ .Values.core.persistence.config.size }} + {{- if .Values.core.persistence.config.storageClass }} + storageClassName: {{ .Values.core.persistence.config.storageClass | default nil }} {{- end }} diff --git a/helm-chart/templates/core/statefulset.yaml b/helm-chart/templates/core/statefulset.yaml index 2bc8d14c..ff075ab0 100644 --- a/helm-chart/templates/core/statefulset.yaml +++ b/helm-chart/templates/core/statefulset.yaml @@ -7,7 +7,7 @@ metadata: app: {{ .Release.Name }}-maibot-core spec: serviceName: {{ .Release.Name }}-maibot-core - replicas: 1 + replicas: 0 # post-install任务初始化完毕后自动扩容至1 selector: matchLabels: app: {{ .Release.Name }}-maibot-core @@ -31,9 +31,9 @@ spec: value: "9943b855e72199d0f5016ea39052f1b6" {{- if .Values.core.webui.enabled }} - name: MAIBOT_WEBUI_ENABLED - value: "true" + value: {{ .Values.core.webui.enabled }} {{- end}} - image: {{ .Values.core.image.repository | default "sengokucola/maibot" }}:{{ .Values.core.image.tag | default "0.11.5-beta" }} + image: {{ .Values.core.image.repository | default "sengokucola/maibot" }}:{{ .Values.core.image.tag | default "0.11.6-beta" }} imagePullPolicy: {{ .Values.core.image.pullPolicy }} ports: - containerPort: 8000 @@ -56,59 +56,18 @@ spec: readOnly: true subPath: k8s-init.sh - mountPath: /MaiMBot/.env - name: env-config - readOnly: true + name: config subPath: .env - {{- if not .Values.core.webui.enabled }} - mountPath: /MaiMBot/config/model_config.toml - name: model-config - readOnly: true + name: config subPath: model_config.toml - mountPath: /MaiMBot/config/bot_config.toml - name: bot-config - readOnly: true + name: config subPath: bot_config.toml - {{- end }} {{- if .Values.statistics_dashboard.enabled }} - mountPath: /MaiMBot/statistics name: statistics {{- end }} - {{- if .Values.core.webui.enabled }} - - mountPath: /MaiMBot/webui-cm-sync - name: webui-cm-sync - {{- end }} - {{- if .Values.core.webui.enabled }} - - name: webui-cm-sync - image: {{ .Values.core.webui.cm_sync.image.repository | default "reg.mikumikumi.xyz/maibot/core-webui-cm-sync" }}:{{ .Values.core.webui.cm_sync.image.tag | default "0.11.5-beta" }} - imagePullPolicy: {{ .Values.core.webui.cm_sync.image.pullPolicy }} - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: RELEASE_NAME - value: {{ .Release.Name }} - volumeMounts: - - mountPath: /MaiMBot/webui-cm-sync - name: webui-cm-sync - {{- end }} - {{- if .Values.core.setup_default_plugins }} - initContainers: # 用户插件目录存储在存储卷中,会在启动时覆盖掉容器的默认插件目录。此初始化容器用于默认插件更新后或麦麦首次启动时为用户自动安装默认插件到存储卷中 - - args: - - setup-plugins.py - command: - - python3 - workingDir: /MaiMBot - image: {{ .Values.core.image.repository | default "sengokucola/maibot" }}:{{ .Values.core.image.tag | default "0.11.5-beta" }} - imagePullPolicy: {{ .Values.core.image.pullPolicy }} - name: setup-plugins - resources: { } - volumeMounts: - - mountPath: /MaiMBot/data - name: data - - mountPath: /MaiMBot/setup-plugins.py - name: scripts - readOnly: true - subPath: setup-plugins.py - {{- end }} serviceAccountName: {{ .Release.Name }}-maibot-sa {{- if .Values.core.image.pullSecrets }} imagePullSecrets: @@ -130,38 +89,13 @@ spec: items: - key: k8s-init.sh path: k8s-init.sh - {{- if .Values.core.setup_default_plugins }} - - key: setup-plugins.py - path: setup-plugins.py - {{- end }} name: {{ .Release.Name }}-maibot-scripts name: scripts - - configMap: - items: - - key: .env - path: .env - name: {{ .Release.Name }}-maibot-core-env-config - name: env-config - {{- if not .Values.core.webui.enabled }} - - configMap: - items: - - key: model_config.toml - path: model_config.toml - name: {{ .Release.Name }}-maibot-core-model-config - name: model-config - - configMap: - items: - - key: bot_config.toml - path: bot_config.toml - name: {{ .Release.Name }}-maibot-core-bot-config - name: bot-config - {{- end }} + - name: config + persistentVolumeClaim: + claimName: {{ .Release.Name }}-maibot-core-config {{- if .Values.statistics_dashboard.enabled }} - name: statistics persistentVolumeClaim: claimName: {{ .Release.Name }}-maibot-statistics-dashboard {{- end }} - {{- if .Values.core.webui.enabled }} - - emptyDir: {} - name: webui-cm-sync - {{- end }} diff --git a/helm-chart/templates/napcat/statefulset.yaml b/helm-chart/templates/napcat/statefulset.yaml index 7cca56df..dc022b21 100644 --- a/helm-chart/templates/napcat/statefulset.yaml +++ b/helm-chart/templates/napcat/statefulset.yaml @@ -26,7 +26,7 @@ spec: value: "{{ .Values.napcat.permission.uid }}" - name: TZ value: Asia/Shanghai - image: {{ .Values.napcat.image.repository | default "mlikiowa/napcat-docker" }}:{{ .Values.napcat.image.tag | default "v4.9.73" }} + image: {{ .Values.napcat.image.repository | default "mlikiowa/napcat-docker" }}:{{ .Values.napcat.image.tag | default "v4.9.80" }} imagePullPolicy: {{ .Values.napcat.image.pullPolicy }} livenessProbe: failureThreshold: 3 diff --git a/helm-chart/templates/other/check-eula-privacy.yaml b/helm-chart/templates/other/check-eula-privacy.yaml index 18b0cca2..db997a60 100644 --- a/helm-chart/templates/other/check-eula-privacy.yaml +++ b/helm-chart/templates/other/check-eula-privacy.yaml @@ -1,8 +1,8 @@ # 检查EULA和PRIVACY {{- if not .Values.EULA_AGREE }} -{{- fail "You must accept the EULA by setting 'EULA_AGREE: true'. EULA: https://github.com/MaiM-with-u/MaiBot/blob/main/EULA.md" }} +{{ fail "You must accept the EULA by setting 'EULA_AGREE: true'. EULA: https://github.com/Mai-with-u/MaiBot/blob/main/EULA.md" }} {{- end }} {{- if not .Values.PRIVACY_AGREE }} -{{- fail "You must accept the Privacy Policy by setting 'PRIVACY_AGREE: true'. Privacy Policy: https://github.com/MaiM-with-u/MaiBot/blob/main/PRIVACY.md" }} +{{ fail "You must accept the Privacy Policy by setting 'PRIVACY_AGREE: true'. Privacy Policy: https://github.com/Mai-with-u/MaiBot/blob/main/PRIVACY.md" }} {{- end }} diff --git a/helm-chart/templates/other/configmap-scripts.yaml b/helm-chart/templates/other/configmap-scripts.yaml index dd61ca68..3fe96794 100644 --- a/helm-chart/templates/other/configmap-scripts.yaml +++ b/helm-chart/templates/other/configmap-scripts.yaml @@ -7,8 +7,3 @@ data: # core k8s-init.sh: | {{ .Files.Get "files/k8s-init.sh" | nindent 4 }} - # core的初始化容器 - {{- if .Values.core.setup_default_plugins }} - setup-plugins.py: | - {{ .Files.Get "files/setup-plugins.py" | nindent 4 }} - {{- end }} diff --git a/helm-chart/templates/other/job-gen-adapter-cm.yaml b/helm-chart/templates/other/job-gen-adapter-cm.yaml deleted file mode 100644 index ad01b568..00000000 --- a/helm-chart/templates/other/job-gen-adapter-cm.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# 动态生成adapter配置文件的configmap的job,仅会在部署时运行一次 -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Release.Name }}-maibot-adapter-cm-generator - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -spec: - backoffLimit: 2 - template: - spec: - serviceAccountName: {{ .Release.Name }}-maibot-sa - restartPolicy: Never - containers: - - name: adapter-cm-generator - image: {{ .Values.adapter.cm_generator.image.repository | default "reg.mikumikumi.xyz/maibot/adapter-cm-generator" }}:{{ .Values.adapter.cm_generator.image.tag | default "0.11.5-beta" }} - workingDir: /app - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: RELEASE_NAME - value: {{ .Release.Name }} - - name: DATA_B64 - value: {{ .Values.config.adapter_config | b64enc }} # 将配置文件编码为base64,从环境变量注入 diff --git a/helm-chart/templates/other/job-preprocessor.yaml b/helm-chart/templates/other/job-preprocessor.yaml new file mode 100644 index 00000000..76c1de19 --- /dev/null +++ b/helm-chart/templates/other/job-preprocessor.yaml @@ -0,0 +1,58 @@ +# 预处理脚本,仅会在部署前运行一次 +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-maibot-preprocessor + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: {{ .Release.Name }}-maibot-sa + restartPolicy: Never + containers: + - name: preprocessor + image: {{ .Values.pre_processor.image.repository | default "reg.mikumikumi.xyz/maibot/preprocessor" }}:{{ .Values.pre_processor.image.tag | default "0.11.6-beta" }} + imagePullPolicy: {{ .Values.pre_processor.image.pullPolicy }} + env: + - name: RELEASE_NAME + value: {{ .Release.Name }} + {{- if or .Values.config.override_adapter_config .Release.IsInstall }} + - name: CONFIG_ADAPTER_B64 + value: {{ .Values.config.adapter_config | b64enc | quote }} + {{- end }} + - name: CONFIG_CORE_ENV_B64 + value: {{ tpl .Files.Get "files/.env" . | b64enc | quote }} + {{- if or .Values.config.override_core_bot_config .Release.IsInstall }} + - name: CONFIG_CORE_BOT_B64 + value: {{ .Values.config.core_bot_config | b64enc | quote }} + {{- end }} + {{- if or .Values.config.override_core_model_config .Release.IsInstall }} + - name: CONFIG_CORE_MODEL_B64 + value: {{ .Values.config.core_model_config | b64enc | quote }} + {{- end }} + volumeMounts: + - mountPath: /app/config/adapter/config.toml + name: adapter-config + subPath: config.toml + - mountPath: /app/config/core/.env + name: core-config + subPath: .env + - mountPath: /app/config/core/bot_config.toml + name: core-config + subPath: bot_config.toml + - mountPath: /app/config/core/model_config.toml + name: core-config + subPath: model_config.toml + imagePullSecrets: + {{ toYaml .Values.pre_processor.image.pullSecrets | nindent 8 }} + volumes: + - name: adapter-config + persistentVolumeClaim: + claimName: {{ .Release.Name }}-maibot-adapter-config + - name: core-config + persistentVolumeClaim: + claimName: {{ .Release.Name }}-maibot-core-config diff --git a/helm-chart/templates/other/rbac.yaml b/helm-chart/templates/other/rbac.yaml index 8b8075c7..2edb11b4 100644 --- a/helm-chart/templates/other/rbac.yaml +++ b/helm-chart/templates/other/rbac.yaml @@ -12,11 +12,14 @@ metadata: namespace: {{ .Release.Namespace }} rules: - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "update", "patch"] + resources: ["configmaps", "pods"] + verbs: ["get", "list", "delete"] - apiGroups: ["apps"] resources: ["statefulsets"] - verbs: ["get", "list", "create", "update", "patch"] + verbs: ["get", "list", "update", "patch"] + - apiGroups: ["apps"] + resources: ["statefulsets/scale"] + verbs: ["get", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index dacb8c3d..890d3ac2 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -5,6 +5,14 @@ EULA_AGREE: false PRIVACY_AGREE: false +# 预处理Job的配置 +pre_processor: + image: + repository: # 默认 reg.mikumikumi.xyz/maibot/preprocessor + tag: # 默认 0.11.6-beta + pullPolicy: IfNotPresent + pullSecrets: [ ] + # 麦麦Adapter的部署配置 adapter: @@ -29,26 +37,23 @@ adapter: nodePort: # 仅在设置NodePort类型时有效,不指定则会随机分配端口 persistence: - storageClass: - accessModes: - - ReadWriteOnce - size: 1Gi - - # adapter的动态生成configmap的Job的配置 - cm_generator: - image: - repository: # 默认 reg.mikumikumi.xyz/maibot/adapter-cm-generator - tag: # 默认 0.11.5-beta - pullPolicy: IfNotPresent - pullSecrets: [ ] - + config: + storageClass: + accessModes: + - ReadWriteOnce + size: 10Mi + data: + storageClass: + accessModes: + - ReadWriteOnce + size: 1Gi # 麦麦本体的部署配置 core: image: repository: # 默认 sengokucola/maibot - tag: # 默认 0.11.5-beta + tag: # 默认 0.11.6-beta pullPolicy: IfNotPresent pullSecrets: [ ] @@ -58,23 +63,19 @@ core: tolerations: [ ] persistence: - storageClass: - accessModes: - - ReadWriteOnce - size: 10Gi - - setup_default_plugins: true # 启用一个初始化容器,用于为用户自动安装默认插件到存储卷中 + config: + storageClass: + accessModes: + - ReadWriteOnce + size: 10Mi + data: + storageClass: + accessModes: + - ReadWriteOnce + size: 10Gi webui: # WebUI相关配置 - # 对从helm chart 0.11.5-beta 之前的旧版升级的用户的重要提示: - # 旧版本helm chart没有配置configmap的保留策略,直接升级同时开启WebUI后会导致configmap丢失从而无法启动。 - # 这种情况可以先禁用WebUI更新一次,再启用WebUI更新一次即可解决问题。 enabled: true # 默认启用 - cm_sync: # WebUI的辅助容器配置 - image: - repository: # 默认 reg.mikumikumi.xyz/maibot/core-webui-cm-sync - tag: # 默认 0.11.5-beta - pullPolicy: IfNotPresent service: type: ClusterIP # ClusterIP / NodePort 指定NodePort可以将内网的服务端口映射到物理节点的端口 port: 8001 # 服务端口 @@ -139,7 +140,7 @@ napcat: image: repository: # 默认 mlikiowa/napcat-docker - tag: # 默认 v4.9.73 + tag: # 默认 v4.9.80 pullPolicy: IfNotPresent pullSecrets: [ ] @@ -211,15 +212,17 @@ sqlite_web: path: / pathType: Prefix -# 手动设置麦麦各部分组件的运行配置文件 +# 设置麦麦各部分组件的初始运行配置 +# 考虑到配置文件的操作复杂性增加,k8s的适配复杂度也同步增加,且WebUI可以直接修改配置文件 +# 自0.11.6-beta版本开始,各组件的配置不再存储于k8s的configmap中,而是直接存储于存储卷的实际文件中 +# 从旧版本升级的用户,旧的configmap的配置会自动迁移到新的存储卷的配置文件中 +# 此处的配置只在初次部署时或者指定覆盖时注入到MaiBot中 config: - # 启用WebUI后,配置文件的修改即可在WebUI进行。如果通过WebUI修改了配置,则实际的配置文件将与values中的配置存在差异。 - # 如果用户在k8s的ConfigMap中手动修改了配置文件,则实际的配置文件也会与values中的配置存在差异。 - # 出现上述两种情况时,为了避免helm升级麦麦时,下面values中的配置覆盖掉已有的配置文件而导致配置丢失,可以在这里禁止本次部署时的配置覆盖。 - # 注:由于adapter的配置无法通过WebUI修改,因此下面的adapter_config配置仍然会覆盖已有配置文件。 - enable_config_override_without_webui: true # 未启用WebUI时,默认覆盖 - enable_config_override_with_webui: false # 启用WebUI时,默认不覆盖 + # 指定是否用下面的配置覆盖MaiBot现有的配置文件 + override_adapter_config: false + override_core_bot_config: false + override_core_model_config: false # adapter的config.toml adapter_config: | @@ -256,7 +259,7 @@ config: # core的model_config.toml core_model_config: | [inner] - version = "1.7.8" + version = "1.8.2" # 配置文件版本号迭代规则同bot_config.toml @@ -303,7 +306,7 @@ config: api_provider = "DeepSeek" # API服务商名称(对应在api_providers中配置的服务商名称) price_in = 2.0 # 输入价格(用于API调用统计,单位:元/ M token)(可选,若无该字段,默认值为0) price_out = 8.0 # 输出价格(用于API调用统计,单位:元/ M token)(可选,若无该字段,默认值为0) - #force_stream_mode = true # 强制流式输出模式(若模型不支持非流式输出,请取消该注释,启用强制流式输出,若无该字段,默认值为false) + # force_stream_mode = true # 强制流式输出模式(若模型不支持非流式输出,请取消该注释,启用强制流式输出,若无该字段,默认值为false) [[models]] model_identifier = "deepseek-ai/DeepSeek-V3.2-Exp" @@ -313,6 +316,7 @@ config: price_out = 3.0 [models.extra_params] # 可选的额外参数配置 enable_thinking = false # 不启用思考 + # temperature = 0.5 # 可选:为该模型单独指定温度,会覆盖任务配置中的温度 [[models]] model_identifier = "deepseek-ai/DeepSeek-V3.2-Exp" @@ -321,7 +325,33 @@ config: price_in = 2.0 price_out = 3.0 [models.extra_params] # 可选的额外参数配置 - enable_thinking = true # 不启用思考 + enable_thinking = true # 启用思考 + # temperature = 0.7 # 可选:为该模型单独指定温度,会覆盖任务配置中的温度 + + [[models]] + model_identifier = "Qwen/Qwen3-Next-80B-A3B-Instruct" + name = "qwen3-next-80b" + api_provider = "SiliconFlow" + price_in = 1.0 + price_out = 4.0 + + [[models]] + model_identifier = "zai-org/GLM-4.6" + name = "siliconflow-glm-4.6" + api_provider = "SiliconFlow" + price_in = 3.5 + price_out = 14.0 + [models.extra_params] # 可选的额外参数配置 + enable_thinking = false # 不启用思考 + + [[models]] + model_identifier = "zai-org/GLM-4.6" + name = "siliconflow-glm-4.6-think" + api_provider = "SiliconFlow" + price_in = 3.5 + price_out = 14.0 + [models.extra_params] # 可选的额外参数配置 + enable_thinking = true # 启用思考 [[models]] model_identifier = "deepseek-ai/DeepSeek-R1" @@ -365,63 +395,74 @@ config: model_list = ["siliconflow-deepseek-v3.2"] # 使用的模型列表,每个子项对应上面的模型名称(name) temperature = 0.2 # 模型温度,新V3建议0.1-0.3 max_tokens = 2048 # 最大输出token数 + slow_threshold = 15.0 # 慢请求阈值(秒),模型等待回复时间超过此值会输出警告日志 [model_task_config.utils_small] # 在麦麦的一些组件中使用的小模型,消耗量较大,建议使用速度较快的小模型 model_list = ["qwen3-30b","qwen3-next-80b"] temperature = 0.7 max_tokens = 2048 + slow_threshold = 10.0 [model_task_config.tool_use] #工具调用模型,需要使用支持工具调用的模型 model_list = ["qwen3-30b","qwen3-next-80b"] temperature = 0.7 max_tokens = 800 + slow_threshold = 10.0 [model_task_config.replyer] # 首要回复模型,还用于表达器和表达方式学习 - model_list = ["siliconflow-deepseek-v3.2-think","siliconflow-glm-4.6-think","siliconflow-glm-4.6"] + model_list = ["siliconflow-deepseek-v3.2","siliconflow-deepseek-v3.2-think","siliconflow-glm-4.6","siliconflow-glm-4.6-think"] temperature = 0.3 # 模型温度,新V3建议0.1-0.3 max_tokens = 2048 + slow_threshold = 25.0 [model_task_config.planner] #决策:负责决定麦麦该什么时候回复的模型 model_list = ["siliconflow-deepseek-v3.2"] temperature = 0.3 max_tokens = 800 + slow_threshold = 12.0 [model_task_config.vlm] # 图像识别模型 model_list = ["qwen3-vl-30"] max_tokens = 256 + slow_threshold = 15.0 [model_task_config.voice] # 语音识别模型 model_list = ["sensevoice-small"] + slow_threshold = 12.0 - #嵌入模型 + # 嵌入模型 [model_task_config.embedding] model_list = ["bge-m3"] + slow_threshold = 5.0 - #------------LPMM知识库模型------------ + # ------------LPMM知识库模型------------ [model_task_config.lpmm_entity_extract] # 实体提取模型 model_list = ["siliconflow-deepseek-v3.2"] temperature = 0.2 max_tokens = 800 + slow_threshold = 20.0 [model_task_config.lpmm_rdf_build] # RDF构建模型 model_list = ["siliconflow-deepseek-v3.2"] temperature = 0.2 max_tokens = 800 + slow_threshold = 20.0 [model_task_config.lpmm_qa] # 问答模型 model_list = ["siliconflow-deepseek-v3.2"] temperature = 0.7 max_tokens = 800 + slow_threshold = 20.0 # core的bot_config.toml core_bot_config: | [inner] - version = "6.21.8" + version = "6.23.5" #----以下是给开发人员阅读的,如果你只是部署了麦麦,不需要阅读---- - #如果你想要修改配置文件,请递增version的值 - #如果新增项目,请阅读src/config/official_configs.py中的说明 + # 如果你想要修改配置文件,请递增version的值 + # 如果新增项目,请阅读src/config/official_configs.py中的说明 # # 版本格式:主版本号.次版本号.修订号,版本号递增规则如下: # 主版本号:MMC版本更新 @@ -441,7 +482,7 @@ config: [personality] # 建议120字以内,描述人格特质 和 身份特征 personality = "是一个女大学生,现在在读大二,会刷贴吧。" - #アイデンティティがない 生まれないらららら + # アイデンティティがない 生まれないらららら # 描述麦麦说话的表达风格,表达习惯,如要修改,可以酌情新增内容 reply_style = "请回复的平淡一些,简短一些,说中文,不要刻意突出自身学科背景。可以参考贴吧,知乎和微博的回复风格。" @@ -498,12 +539,16 @@ config: # 注意:如果为群聊,则需要设置为group,如果设置为私聊,则需要设置为private ] + reflect = false # 是否启用表达反思(Bot主动向管理员询问表达方式是否合适) + reflect_operator_id = "" # 表达反思操作员ID,格式:platform:id:type (例如 "qq:123456:private" 或 "qq:654321:group") + allow_reflect = [] # 允许进行表达反思的聊天流ID列表,格式:["qq:123456:private", "qq:654321:group", ...],只有在此列表中的聊天流才会提出问题并跟踪。如果列表为空,则所有聊天流都可以进行表达反思(前提是 reflect = true) - [chat] #麦麦的聊天设置 - talk_value = 1 #聊天频率,越小越沉默,范围0-1 + + [chat] # 麦麦的聊天设置 + talk_value = 1 # 聊天频率,越小越沉默,范围0-1 mentioned_bot_reply = true # 是否启用提及必回复 max_context_size = 30 # 上下文长度 - planner_smooth = 2 #规划器平滑,增大数值会减小planner负荷,略微降低反应速度,推荐1-5,0为关闭,必须大于等于0 + planner_smooth = 2 # 规划器平滑,增大数值会减小planner负荷,略微降低反应速度,推荐1-5,0为关闭,必须大于等于0 enable_talk_value_rules = true # 是否启用动态发言频率规则 @@ -523,6 +568,7 @@ config: [memory] max_agent_iterations = 3 # 记忆思考深度(最低为1(不深入思考)) + enable_jargon_detection = true # 记忆检索过程中是否启用黑话识别 [jargon] all_global = true # 是否开启全局黑话模式,注意,此功能关闭后,已经记录的全局黑话不会改变,需要手动删除 @@ -556,8 +602,8 @@ config: ban_msgs_regex = [ # 需要过滤的消息(原始消息)匹配的正则表达式,匹配到的消息将被过滤,若不了解正则表达式请勿修改 - #"https?://[^\\s]+", # 匹配https链接 - #"\\d{4}-\\d{2}-\\d{2}", # 匹配日期 + # "https?://[^\\s]+", # 匹配https链接 + # "\\d{4}-\\d{2}-\\d{2}", # 匹配日期 ] @@ -586,7 +632,7 @@ config: keyword_rules = [ { keywords = ["人机", "bot", "机器", "入机", "robot", "机器人", "ai", "AI"], reaction = "有人说你是人机或者机器人,否定这一事实,攻击或反问他。注意,只有认为你是机器人才回复,否则不要否认" }, { keywords = ["测试关键词回复", "test"], reaction = "回答测试成功" }, - #{ keywords = ["你好", "hello"], reaction = "你好,有什么可以帮你?" } + # { keywords = ["你好", "hello"], reaction = "你好,有什么可以帮你?" } # 在此处添加更多规则,格式同上 ] @@ -659,6 +705,7 @@ config: chat_prompts = [] - #此系统暂时移除,无效配置 + # 此系统暂时移除,无效配置 [relationship] enable_relationship = true # 是否启用关系系统 +