mirror of https://github.com/Mai-with-u/MaiBot.git
Merge branch 'dev' into groupnickname
commit
faebf51f9e
|
|
@ -6,29 +6,39 @@ echo 你需要选择启动方式,输入字母来选择:
|
||||||
echo V = 不知道什么意思就输入 V
|
echo V = 不知道什么意思就输入 V
|
||||||
echo C = 输入 C 使用 Conda 环境
|
echo C = 输入 C 使用 Conda 环境
|
||||||
echo.
|
echo.
|
||||||
choice /C CV /N /M "在下方输入字母并回车 (C/V)?" /T 10 /D V
|
choice /C CV /N /M "不知道什么意思就输入 V (C/V)?" /T 10 /D V
|
||||||
|
|
||||||
set "ENV_TYPE="
|
set "ENV_TYPE="
|
||||||
if %ERRORLEVEL% == 1 set "ENV_TYPE=CONDA"
|
if %ERRORLEVEL% == 1 set "ENV_TYPE=CONDA"
|
||||||
if %ERRORLEVEL% == 2 set "ENV_TYPE=VENV"
|
if %ERRORLEVEL% == 2 set "ENV_TYPE=VENV"
|
||||||
|
|
||||||
if "%ENV_TYPE%" == "CONDA" (
|
if "%ENV_TYPE%" == "CONDA" goto activate_conda
|
||||||
|
if "%ENV_TYPE%" == "VENV" goto activate_venv
|
||||||
|
|
||||||
|
REM 如果 choice 超时或返回意外值,默认使用 venv
|
||||||
|
echo WARN: Invalid selection or timeout from choice. Defaulting to VENV.
|
||||||
|
set "ENV_TYPE=VENV"
|
||||||
|
goto activate_venv
|
||||||
|
|
||||||
|
:activate_conda
|
||||||
set /p CONDA_ENV_NAME="请输入要使用的 Conda 环境名称: "
|
set /p CONDA_ENV_NAME="请输入要使用的 Conda 环境名称: "
|
||||||
if not defined CONDA_ENV_NAME (
|
if not defined CONDA_ENV_NAME (
|
||||||
echo 错误: 未输入 Conda 环境名称.
|
echo 错误: 未输入 Conda 环境名称.
|
||||||
pause
|
pause
|
||||||
exit /b 1
|
exit /b 1
|
||||||
)
|
)
|
||||||
echo 选择: Conda '%CONDA_ENV_NAME%'
|
echo 选择: Conda '!CONDA_ENV_NAME!'
|
||||||
REM 激活Conda环境
|
REM 激活Conda环境
|
||||||
call conda activate %CONDA_ENV_NAME%
|
call conda activate !CONDA_ENV_NAME!
|
||||||
if %ERRORLEVEL% neq 0 (
|
if !ERRORLEVEL! neq 0 (
|
||||||
echo 错误: Conda环境 '%CONDA_ENV_NAME%' 激活失败. 请确保Conda已安装并正确配置, 且 '%CONDA_ENV_NAME%' 环境存在.
|
echo 错误: Conda环境 '!CONDA_ENV_NAME!' 激活失败. 请确保Conda已安装并正确配置, 且 '!CONDA_ENV_NAME!' 环境存在.
|
||||||
pause
|
pause
|
||||||
exit /b 1
|
exit /b 1
|
||||||
)
|
)
|
||||||
) else (
|
goto env_activated
|
||||||
echo Selected: venv (default)
|
|
||||||
|
:activate_venv
|
||||||
|
echo Selected: venv (default or selected)
|
||||||
REM 查找venv虚拟环境
|
REM 查找venv虚拟环境
|
||||||
set "venv_path=%~dp0venv\Scripts\activate.bat"
|
set "venv_path=%~dp0venv\Scripts\activate.bat"
|
||||||
if not exist "%venv_path%" (
|
if not exist "%venv_path%" (
|
||||||
|
|
@ -43,8 +53,9 @@ if "%ENV_TYPE%" == "CONDA" (
|
||||||
pause
|
pause
|
||||||
exit /b 1
|
exit /b 1
|
||||||
)
|
)
|
||||||
)
|
goto env_activated
|
||||||
|
|
||||||
|
:env_activated
|
||||||
echo Environment activated successfully!
|
echo Environment activated successfully!
|
||||||
|
|
||||||
REM --- 后续脚本执行 ---
|
REM --- 后续脚本执行 ---
|
||||||
|
|
|
||||||
|
|
@ -48,11 +48,10 @@ def process_text_file(file_path):
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# 新增用户确认提示
|
# 新增用户确认提示
|
||||||
print("=== 重要操作确认,请认真阅读以下内容哦 ===")
|
print("=== 数据预处理脚本 ===")
|
||||||
print("如果你并非第一次导入知识")
|
print(f"本脚本将处理 '{RAW_DATA_PATH}' 目录下的所有 .txt 文件。")
|
||||||
print("请先删除data/import.json文件,备份data/openie.json文件")
|
print(f"处理后的段落数据将合并,并以 MM-DD-HH-SS-imported-data.json 的格式保存在 '{IMPORTED_DATA_PATH}' 目录中。")
|
||||||
print("在进行知识库导入之前")
|
print("请确保原始数据已放置在正确的目录中。")
|
||||||
print("请修改config/lpmm_config.toml中的配置项")
|
|
||||||
confirm = input("确认继续执行?(y/n): ").strip().lower()
|
confirm = input("确认继续执行?(y/n): ").strip().lower()
|
||||||
if confirm != "y":
|
if confirm != "y":
|
||||||
logger.error("操作已取消")
|
logger.error("操作已取消")
|
||||||
|
|
|
||||||
|
|
@ -289,7 +289,6 @@ def _to_emoji_objects(data):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[加载错误] 处理数据库记录时出错 ({full_path}): {str(e)}")
|
logger.error(f"[加载错误] 处理数据库记录时出错 ({full_path}): {str(e)}")
|
||||||
load_errors += 1
|
load_errors += 1
|
||||||
return emoji_objects, load_errors
|
|
||||||
return emoji_objects, load_errors
|
return emoji_objects, load_errors
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
import os
|
import os
|
||||||
import toml
|
import toml
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
|
||||||
|
# import argparse
|
||||||
from .global_logger import logger
|
from .global_logger import logger
|
||||||
|
|
||||||
PG_NAMESPACE = "paragraph"
|
PG_NAMESPACE = "paragraph"
|
||||||
|
|
@ -37,7 +38,8 @@ def _load_config(config, config_file_path):
|
||||||
# Check if all top-level keys from default config exist in the file config
|
# Check if all top-level keys from default config exist in the file config
|
||||||
for key in config.keys():
|
for key in config.keys():
|
||||||
if key not in file_config:
|
if key not in file_config:
|
||||||
print(f"警告: 配置文件 '{config_file_path}' 缺少必需的顶级键: '{key}'。请检查配置文件。")
|
logger.critical(f"警告: 配置文件 '{config_file_path}' 缺少必需的顶级键: '{key}'。请检查配置文件。")
|
||||||
|
logger.critical("请通过template/lpmm_config_template.toml文件进行更新")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
if "llm_providers" in file_config:
|
if "llm_providers" in file_config:
|
||||||
|
|
@ -68,16 +70,11 @@ def _load_config(config, config_file_path):
|
||||||
logger.info(f"从文件中读取配置: {config_file_path}")
|
logger.info(f"从文件中读取配置: {config_file_path}")
|
||||||
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Configurations for the pipeline")
|
|
||||||
parser.add_argument(
|
|
||||||
"--config_path",
|
|
||||||
type=str,
|
|
||||||
default="lpmm_config.toml",
|
|
||||||
help="Path to the configuration file",
|
|
||||||
)
|
|
||||||
|
|
||||||
global_config = dict(
|
global_config = dict(
|
||||||
{
|
{
|
||||||
|
"lpmm": {
|
||||||
|
"version": "0.1.0",
|
||||||
|
},
|
||||||
"llm_providers": {
|
"llm_providers": {
|
||||||
"localhost": {
|
"localhost": {
|
||||||
"base_url": "https://api.siliconflow.cn/v1",
|
"base_url": "https://api.siliconflow.cn/v1",
|
||||||
|
|
@ -136,8 +133,8 @@ global_config = dict(
|
||||||
)
|
)
|
||||||
|
|
||||||
# _load_config(global_config, parser.parse_args().config_path)
|
# _load_config(global_config, parser.parse_args().config_path)
|
||||||
file_path = os.path.abspath(__file__)
|
# file_path = os.path.abspath(__file__)
|
||||||
dir_path = os.path.dirname(file_path)
|
# dir_path = os.path.dirname(file_path)
|
||||||
root_path = os.path.join(dir_path, os.pardir, os.pardir, os.pardir, os.pardir)
|
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
|
||||||
config_path = os.path.join(root_path, "config", "lpmm_config.toml")
|
config_path = os.path.join(ROOT_PATH, "config", "lpmm_config.toml")
|
||||||
_load_config(global_config, config_path)
|
_load_config(global_config, config_path)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import time
|
||||||
import random
|
import random
|
||||||
from typing import List, Dict
|
from typing import List, Dict
|
||||||
from ...common.database import db
|
from ...common.database import db
|
||||||
from maim_message import UserInfo, Seg
|
from maim_message import UserInfo
|
||||||
# import re
|
# import re
|
||||||
# import traceback
|
# import traceback
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,6 @@
|
||||||
|
[lpmm]
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
# LLM API 服务提供商,可配置多个
|
# LLM API 服务提供商,可配置多个
|
||||||
[[llm_providers]]
|
[[llm_providers]]
|
||||||
name = "localhost"
|
name = "localhost"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue