feat: 重构资料包模型与会话绑定主链路

2026-06-04 00:43:13 +08:00
parent ddf5e7d15c
commit d0841e533f
18 changed files with 1000 additions and 263 deletions
--- a/apps/documents/migrations/0002_submissionbatch_uploadeddocument_chapter_code_and_more.py
+++ b/apps/documents/migrations/0002_submissionbatch_uploadeddocument_chapter_code_and_more.py
@@ -0,0 +1,103 @@
+# Generated by Django 5.2.14 on 2026-06-03 16:39
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("documents", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SubmissionBatch",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "batch_id",
+                    models.CharField(db_index=True, max_length=64, unique=True),
+                ),
+                (
+                    "product_name",
+                    models.CharField(blank=True, db_index=True, max_length=255),
+                ),
+                (
+                    "workflow_type",
+                    models.CharField(default="registration", max_length=64),
+                ),
+                (
+                    "conversation_id",
+                    models.CharField(blank=True, db_index=True, max_length=64),
+                ),
+                ("file_count", models.PositiveIntegerField(default=0)),
+                ("page_count", models.PositiveIntegerField(default=0)),
+                ("chapter_summary", models.JSONField(blank=True, default=list)),
+                (
+                    "import_status",
+                    models.CharField(db_index=True, default="pending", max_length=32),
+                ),
+                ("exception_count", models.PositiveIntegerField(default=0)),
+                ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "ordering": ["-created_at"],
+            },
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="chapter_code",
+            field=models.CharField(blank=True, max_length=32),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="chapter_match_status",
+            field=models.CharField(blank=True, max_length=32),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="document_role",
+            field=models.CharField(blank=True, max_length=64),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="needs_manual_review",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="page_count",
+            field=models.PositiveIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="page_count_confidence",
+            field=models.CharField(blank=True, max_length=32),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="relative_path",
+            field=models.CharField(blank=True, max_length=500),
+        ),
+        migrations.AddField(
+            model_name="uploadeddocument",
+            name="batch",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="documents",
+                to="documents.submissionbatch",
+            ),
+        ),
+    ]
--- a/apps/documents/models.py
+++ b/apps/documents/models.py
@@ -1,6 +1,48 @@
 from django.db import models


+class SubmissionBatch(models.Model):
+    """
+    资料包主对象，承接导入、会话绑定和目录汇总结果。
+
+    Documents 模块负责维护资料包与文件的关系，
+    不在模型层耦合 Agent 执行细节。
+    """
+
+    STATUS_PENDING = "pending"
+    STATUS_PROCESSING = "processing"
+    STATUS_COMPLETED = "completed"
+    STATUS_REVIEW_REQUIRED = "review_required"
+    STATUS_FAILED = "failed"
+
+    batch_id = models.CharField(max_length=64, unique=True, db_index=True)
+    product_name = models.CharField(max_length=255, blank=True, db_index=True)
+    workflow_type = models.CharField(max_length=64, default="registration")
+    conversation_id = models.CharField(max_length=64, blank=True, db_index=True)
+    file_count = models.PositiveIntegerField(default=0)
+    page_count = models.PositiveIntegerField(default=0)
+    chapter_summary = models.JSONField(default=list, blank=True)
+    import_status = models.CharField(max_length=32, default=STATUS_PENDING, db_index=True)
+    exception_count = models.PositiveIntegerField(default=0)
+    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    def __str__(self) -> str:
+        return self.product_name or self.batch_id
+
+    def get_import_status_display_text(self) -> str:
+        return {
+            self.STATUS_PENDING: "待导入",
+            self.STATUS_PROCESSING: "处理中",
+            self.STATUS_COMPLETED: "已完成",
+            self.STATUS_REVIEW_REQUIRED: "待复核",
+            self.STATUS_FAILED: "失败",
+        }.get(self.import_status, self.import_status)
+
+
 class UploadedDocument(models.Model):
    """
    保存用户上传文档的元数据和入库状态。
@@ -13,11 +55,25 @@ class UploadedDocument(models.Model):
    STATUS_INDEXED = "indexed"
    STATUS_FAILED = "failed"

+    batch = models.ForeignKey(
+        SubmissionBatch,
+        related_name="documents",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+    )
    scenario_id = models.CharField(max_length=100, db_index=True)
    original_name = models.CharField(max_length=255)
    file = models.FileField(upload_to="documents/%Y%m%d/")
    file_type = models.CharField(max_length=20)
    size = models.PositiveIntegerField(default=0)
+    relative_path = models.CharField(max_length=500, blank=True)
+    chapter_code = models.CharField(max_length=32, blank=True)
+    document_role = models.CharField(max_length=64, blank=True)
+    page_count = models.PositiveIntegerField(default=0)
+    page_count_confidence = models.CharField(max_length=32, blank=True)
+    chapter_match_status = models.CharField(max_length=32, blank=True)
+    needs_manual_review = models.BooleanField(default=False)
    status = models.CharField(max_length=20, default=STATUS_UPLOADED, db_index=True)
    error_message = models.TextField(blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
--- a/apps/documents/services.py
+++ b/apps/documents/services.py
@@ -4,11 +4,12 @@ import xml.etree.ElementTree as ET
 from zipfile import BadZipFile, ZipFile

 from agent_core.rag.ingest import ingest_document
+from apps.chat.services import create_conversation_for_batch

-from .models import UploadedDocument
+from .models import SubmissionBatch, UploadedDocument


-def create_uploaded_document(scenario_id: str, uploaded_file) -> UploadedDocument:
+def create_uploaded_document(scenario_id: str, uploaded_file, batch: SubmissionBatch | None = None) -> UploadedDocument:
    """
    保存上传文件的元数据记录。

@@ -17,15 +18,116 @@ def create_uploaded_document(scenario_id: str, uploaded_file) -> UploadedDocumen
    """
    extension = _detect_extension(uploaded_file.name)
    return UploadedDocument.objects.create(
+        batch=batch,
        scenario_id=scenario_id,
        original_name=uploaded_file.name,
        file=uploaded_file,
        file_type=extension,
        size=uploaded_file.size,
+        relative_path=uploaded_file.name,
        status=UploadedDocument.STATUS_UPLOADED,
    )


+def import_submission_batch(scenario_id: str, uploaded_files: list) -> dict:
+    """
+    导入资料包并建立批次、文档、目录汇总和主会话。
+
+    当前实现保持离线稳定，重点保证：
+    - 资料包记录可落库
+    - 产品名称可解析
+    - 会话可自动绑定
+    - 可直接产出 overview report
+    """
+    batch = SubmissionBatch.objects.create(
+        batch_id=_generate_batch_id(),
+        workflow_type="registration",
+        import_status=SubmissionBatch.STATUS_PROCESSING,
+    )
+    documents = []
+    candidates = []
+    chapter_summary = {}
+    total_pages = 0
+
+    for uploaded_file in uploaded_files:
+        document = create_uploaded_document(scenario_id, uploaded_file, batch=batch)
+        text = extract_text(document)
+        page_count = _estimate_page_count(text)
+        document.page_count = page_count
+        document.page_count_confidence = "estimated"
+        document.document_role = _detect_document_role(document.original_name)
+        document.chapter_code = _detect_chapter_code(document.original_name, text)
+        document.chapter_match_status = "matched" if document.chapter_code else "unknown"
+        document.needs_manual_review = not bool(document.chapter_code)
+        document.save(
+            update_fields=[
+                "page_count",
+                "page_count_confidence",
+                "document_role",
+                "chapter_code",
+                "chapter_match_status",
+                "needs_manual_review",
+                "updated_at",
+            ]
+        )
+        documents.append(document)
+        total_pages += page_count
+        chapter_key = document.chapter_code or "UNCLASSIFIED"
+        chapter_summary[chapter_key] = chapter_summary.get(chapter_key, 0) + 1
+        candidates.extend(_extract_product_candidates(document.original_name, text))
+
+    product_name, warnings = _select_product_name(candidates)
+    conversation = create_conversation_for_batch(batch.batch_id, product_name)
+
+    batch.product_name = product_name
+    batch.conversation_id = conversation.conversation_id
+    batch.file_count = len(documents)
+    batch.page_count = total_pages
+    batch.chapter_summary = [
+        {"chapter_code": chapter_code, "document_count": count}
+        for chapter_code, count in sorted(chapter_summary.items())
+    ]
+    batch.exception_count = len(warnings)
+    batch.import_status = (
+        SubmissionBatch.STATUS_REVIEW_REQUIRED if warnings else SubmissionBatch.STATUS_COMPLETED
+    )
+    batch.save(
+        update_fields=[
+            "product_name",
+            "conversation_id",
+            "file_count",
+            "page_count",
+            "chapter_summary",
+            "exception_count",
+            "import_status",
+            "updated_at",
+        ]
+    )
+    return {
+        "batch_id": batch.batch_id,
+        "conversation_id": conversation.conversation_id,
+        "product_name": batch.product_name,
+        "registration_overview_report": {
+            "batch_id": batch.batch_id,
+            "product_name": batch.product_name,
+            "file_count": batch.file_count,
+            "total_page_count": batch.page_count,
+            "chapter_summary": batch.chapter_summary,
+            "documents": [
+                {
+                    "document_id": document.id,
+                    "original_name": document.original_name,
+                    "chapter_code": document.chapter_code,
+                    "page_count": document.page_count,
+                    "document_role": document.document_role,
+                }
+                for document in documents
+            ],
+            "warnings": warnings,
+        },
+    }
+
+
 def extract_text(document: UploadedDocument) -> str:
    """
    根据文档类型选择合适的文本抽取策略。
@@ -83,6 +185,99 @@ def _detect_extension(file_name: str) -> str:
    return Path(file_name).suffix.lower().lstrip(".")


+def _generate_batch_id() -> str:
+    return f"SUB-20260604-{SubmissionBatch.objects.count() + 1:03d}"
+
+
+def _estimate_page_count(text: str) -> int:
+    stripped = text.strip()
+    if not stripped:
+        return 0
+    line_count = len([line for line in stripped.splitlines() if line.strip()])
+    return max(1, line_count)
+
+
+def _detect_document_role(file_name: str) -> str:
+    normalized = file_name.lower()
+    if "申请表" in file_name:
+        return "application_form"
+    if "说明书" in file_name:
+        return "product_manual"
+    if "产品列表" in file_name:
+        return "product_list"
+    if "声明" in file_name:
+        return "declaration"
+    if normalized.endswith(".pdf"):
+        return "pdf_document"
+    return "general_document"
+
+
+def _detect_chapter_code(file_name: str, text: str) -> str:
+    for source in (file_name, text):
+        match = re.search(r"(CH\d+(?:\.\d+)*)", source, flags=re.IGNORECASE)
+        if match:
+            return match.group(1).upper()
+    if "监管" in file_name or "申请表" in file_name or "说明书" in file_name:
+        return "CH1"
+    return ""
+
+
+def _extract_product_candidates(file_name: str, text: str) -> list[dict]:
+    source_type = _detect_candidate_source(file_name)
+    if not source_type:
+        return []
+    patterns = [
+        r"产品名称[:：]\s*([^\n\r]+)",
+        r"名称[:：]\s*([^\n\r]+检测试剂盒[^\n\r]*)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text)
+        if match:
+            return [{"source_type": source_type, "product_name": match.group(1).strip()}]
+    cleaned = Path(file_name).stem.replace("目标产品", "").replace("说明书", "").strip("-_ ")
+    if cleaned and "申请表" not in cleaned and "产品列表" not in cleaned:
+        return [{"source_type": source_type, "product_name": cleaned}]
+    return []
+
+
+def _detect_candidate_source(file_name: str) -> str:
+    if "申请表" in file_name:
+        return "application_form"
+    if "说明书" in file_name:
+        return "product_manual"
+    if "产品列表" in file_name:
+        return "product_list"
+    return ""
+
+
+def _select_product_name(candidates: list[dict]) -> tuple[str, list[str]]:
+    if not candidates:
+        return "", ["未识别到产品名称，建议人工补录。"]
+
+    priority = {
+        "application_form": 1,
+        "product_manual": 2,
+        "product_list": 3,
+    }
+    sorted_candidates = sorted(
+        candidates,
+        key=lambda item: priority.get(item["source_type"], 99),
+    )
+    top_candidate = sorted_candidates[0]
+    warnings = []
+    conflict_names = {
+        item["product_name"]
+        for item in sorted_candidates
+        if item["product_name"] != top_candidate["product_name"]
+    }
+    if conflict_names:
+        warnings.append(
+            "产品名称来源冲突："
+            + " / ".join([top_candidate["product_name"], *sorted(conflict_names)])
+        )
+    return top_candidate["product_name"], warnings
+
+
 def _read_text_file(path: Path) -> str:
    """优先按 UTF-8 读取；失败时回退到系统默认编码。"""
    try:
--- a/apps/documents/views.py
+++ b/apps/documents/views.py
@@ -5,18 +5,24 @@ from django.views.decorators.http import require_POST
 from apps.scenarios.services import list_scenarios

 from .forms import DocumentUploadForm
-from .models import UploadedDocument
-from .services import create_uploaded_document, index_document
+from .models import SubmissionBatch, UploadedDocument
+from .services import import_submission_batch, index_document


 def document_list(request):
-    # 列表页只负责展示文档元数据和可执行操作，不处理入库细节。
+    # 资料包页展示批次、会话绑定和关键异常，同时保留文档级明细便于演示。
+    keyword = (request.GET.get("keyword") or "").strip()
+    batches = SubmissionBatch.objects.all()
+    if keyword:
+        batches = batches.filter(product_name__icontains=keyword)
    documents = UploadedDocument.objects.all()
    status_counts = {
-        "uploaded": documents.filter(status=UploadedDocument.STATUS_UPLOADED).count(),
-        "indexed": documents.filter(status=UploadedDocument.STATUS_INDEXED).count(),
-        "failed": documents.filter(status=UploadedDocument.STATUS_FAILED).count(),
-        "total": documents.count(),
+        "pending": batches.filter(import_status=SubmissionBatch.STATUS_PENDING).count(),
+        "completed": batches.filter(import_status=SubmissionBatch.STATUS_COMPLETED).count(),
+        "review_required": batches.filter(
+            import_status=SubmissionBatch.STATUS_REVIEW_REQUIRED
+        ).count(),
+        "total": batches.count(),
    }
    processing_pipeline = [
        {"title": "原始文件接收", "detail": "校验格式、大小和场景归属后保存原件。"},
@@ -35,6 +41,8 @@ def document_list(request):
        "documents/document_list.html",
        {
            "documents": documents,
+            "batches": batches,
+            "keyword": keyword,
            "status_counts": status_counts,
            "processing_pipeline": processing_pipeline,
            "exception_items": exception_items,
@@ -43,12 +51,18 @@ def document_list(request):


 def upload(request):
-    # 上传成功后仅保存文件和元数据，是否入库由用户显式触发。
+    # 上传成功后直接创建资料包并绑定主会话。
    if request.method == "POST":
        form = DocumentUploadForm(request.POST, request.FILES)
        if form.is_valid():
-            create_uploaded_document(form.cleaned_data["scenario_id"], form.cleaned_data["file"])
-            messages.success(request, "文件已上传，可继续执行入库。")
+            result = import_submission_batch(
+                form.cleaned_data["scenario_id"],
+                [form.cleaned_data["file"]],
+            )
+            messages.success(
+                request,
+                f"资料包已导入，已绑定会话 {result['conversation_id']}。",
+            )
            return redirect("documents:list")
    else:
        form = DocumentUploadForm()