Merge branch 'dev' into groupnickname

pull/914/head
未來星織 2025-05-02 22:31:54 +09:00 committed by GitHub
commit faebf51f9e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 39 additions and 30 deletions

View File

@ -6,29 +6,39 @@ echo 你需要选择启动方式,输入字母来选择:
echo V = 不知道什么意思就输入 V
echo C = 输入 C 使用 Conda 环境
echo.
choice /C CV /N /M "在下方输入字母并回车 (C/V)?" /T 10 /D V
choice /C CV /N /M "不知道什么意思就输入 V (C/V)?" /T 10 /D V
set "ENV_TYPE="
if %ERRORLEVEL% == 1 set "ENV_TYPE=CONDA"
if %ERRORLEVEL% == 2 set "ENV_TYPE=VENV"
if "%ENV_TYPE%" == "CONDA" (
if "%ENV_TYPE%" == "CONDA" goto activate_conda
if "%ENV_TYPE%" == "VENV" goto activate_venv
REM 如果 choice 超时或返回意外值,默认使用 venv
echo WARN: Invalid selection or timeout from choice. Defaulting to VENV.
set "ENV_TYPE=VENV"
goto activate_venv
:activate_conda
set /p CONDA_ENV_NAME="请输入要使用的 Conda 环境名称: "
if not defined CONDA_ENV_NAME (
echo 错误: 未输入 Conda 环境名称.
pause
exit /b 1
)
echo 选择: Conda '%CONDA_ENV_NAME%'
echo 选择: Conda '!CONDA_ENV_NAME!'
REM 激活Conda环境
call conda activate %CONDA_ENV_NAME%
if %ERRORLEVEL% neq 0 (
echo 错误: Conda环境 '%CONDA_ENV_NAME%' 激活失败. 请确保Conda已安装并正确配置, 且 '%CONDA_ENV_NAME%' 环境存在.
call conda activate !CONDA_ENV_NAME!
if !ERRORLEVEL! neq 0 (
echo 错误: Conda环境 '!CONDA_ENV_NAME!' 激活失败. 请确保Conda已安装并正确配置, 且 '!CONDA_ENV_NAME!' 环境存在.
pause
exit /b 1
)
) else (
echo Selected: venv (default)
goto env_activated
:activate_venv
echo Selected: venv (default or selected)
REM 查找venv虚拟环境
set "venv_path=%~dp0venv\Scripts\activate.bat"
if not exist "%venv_path%" (
@ -43,8 +53,9 @@ if "%ENV_TYPE%" == "CONDA" (
pause
exit /b 1
)
)
goto env_activated
:env_activated
echo Environment activated successfully!
REM --- 后续脚本执行 ---

View File

@ -48,11 +48,10 @@ def process_text_file(file_path):
def main():
# 新增用户确认提示
print("=== 重要操作确认,请认真阅读以下内容哦 ===")
print("如果你并非第一次导入知识")
print("请先删除data/import.json文件备份data/openie.json文件")
print("在进行知识库导入之前")
print("请修改config/lpmm_config.toml中的配置项")
print("=== 数据预处理脚本 ===")
print(f"本脚本将处理 '{RAW_DATA_PATH}' 目录下的所有 .txt 文件。")
print(f"处理后的段落数据将合并,并以 MM-DD-HH-SS-imported-data.json 的格式保存在 '{IMPORTED_DATA_PATH}' 目录中。")
print("请确保原始数据已放置在正确的目录中。")
confirm = input("确认继续执行?(y/n): ").strip().lower()
if confirm != "y":
logger.error("操作已取消")

View File

@ -289,7 +289,6 @@ def _to_emoji_objects(data):
except Exception as e:
logger.error(f"[加载错误] 处理数据库记录时出错 ({full_path}): {str(e)}")
load_errors += 1
return emoji_objects, load_errors
return emoji_objects, load_errors

View File

@ -1,7 +1,8 @@
import os
import toml
import sys
import argparse
# import argparse
from .global_logger import logger
PG_NAMESPACE = "paragraph"
@ -37,7 +38,8 @@ def _load_config(config, config_file_path):
# Check if all top-level keys from default config exist in the file config
for key in config.keys():
if key not in file_config:
print(f"警告: 配置文件 '{config_file_path}' 缺少必需的顶级键: '{key}'。请检查配置文件。")
logger.critical(f"警告: 配置文件 '{config_file_path}' 缺少必需的顶级键: '{key}'。请检查配置文件。")
logger.critical("请通过template/lpmm_config_template.toml文件进行更新")
sys.exit(1)
if "llm_providers" in file_config:
@ -68,16 +70,11 @@ def _load_config(config, config_file_path):
logger.info(f"从文件中读取配置: {config_file_path}")
parser = argparse.ArgumentParser(description="Configurations for the pipeline")
parser.add_argument(
"--config_path",
type=str,
default="lpmm_config.toml",
help="Path to the configuration file",
)
global_config = dict(
{
"lpmm": {
"version": "0.1.0",
},
"llm_providers": {
"localhost": {
"base_url": "https://api.siliconflow.cn/v1",
@ -136,8 +133,8 @@ global_config = dict(
)
# _load_config(global_config, parser.parse_args().config_path)
file_path = os.path.abspath(__file__)
dir_path = os.path.dirname(file_path)
root_path = os.path.join(dir_path, os.pardir, os.pardir, os.pardir, os.pardir)
config_path = os.path.join(root_path, "config", "lpmm_config.toml")
# file_path = os.path.abspath(__file__)
# dir_path = os.path.dirname(file_path)
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
config_path = os.path.join(ROOT_PATH, "config", "lpmm_config.toml")
_load_config(global_config, config_path)

View File

@ -7,7 +7,7 @@ import time
import random
from typing import List, Dict
from ...common.database import db
from maim_message import UserInfo, Seg
from maim_message import UserInfo
# import re
# import traceback

View File

@ -1,3 +1,6 @@
[lpmm]
version = "0.1.0"
# LLM API 服务提供商,可配置多个
[[llm_providers]]
name = "localhost"