From dfc5a6473e343841c942eb56c5f28476940a1bb5 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 19:43:19 +0800 Subject: [PATCH 1/7] feat: add assignment knowledge evidence baseline --- KNOWLEDGE_RAG_STAGE10.md | 85 ++++++++++++++++++ routes/api.py | 44 +++++++++- services/knowledge_rag.py | 126 +++++++++++++++++++++++++++ tests/test_knowledge_rag.py | 167 ++++++++++++++++++++++++++++++++++++ utils/guidance_generator.py | 28 +++++- 5 files changed, 444 insertions(+), 6 deletions(-) create mode 100644 KNOWLEDGE_RAG_STAGE10.md create mode 100644 services/knowledge_rag.py create mode 100644 tests/test_knowledge_rag.py diff --git a/KNOWLEDGE_RAG_STAGE10.md b/KNOWLEDGE_RAG_STAGE10.md new file mode 100644 index 0000000..5cfc9e2 --- /dev/null +++ b/KNOWLEDGE_RAG_STAGE10.md @@ -0,0 +1,85 @@ +# CodeSense 阶段十：知识检索与 RAG 系统地图、指标基线 + +主题键：`CodeSense:knowledge-rag:stage10` +验证角度：性能、资源与边界条件 +本次实现是有界的显式证据检索基线，不是向量数据库或生产级 RAG 索引。 + +## 1. 变更前的事实基线 + +以下结论来自当前仓库代码，而不是对线上系统的推测： + +- 仓库没有独立的 RAG、向量索引或 retriever 模块。 +- `models.AssignmentKnowledgePoint` 是已有的作业级知识点关联表； + `add_to_assignment()` 是教师/自动检测流程写入知识点的现有入口。 +- `models.KnowledgePointScore` 保存学生个人知识点评分，属于私有画像，不能作为跨学生检索内容。 +- `POST /api/ask_question` 原先把作业标题、描述、学生代码和问题交给答案生成器，返回答案，但没有返回知识证据、引用或“没有检索结果”的明确状态。 + +在实现前先加入了一个期望新行为的红灯用例： +`tests/test_knowledge_rag.py::test_ask_question_exposes_retrieval_evidence_and_fallback_state`。 +未修改路由时该用例以 `KeyError: 'knowledge_retrieval'` 失败，说明旧响应没有该字段；修复后转为通过。 + +## 2. 当前系统地图 + +```text +教师/已有自动检测流程 + │ AssignmentKnowledgePoint.add_to_assignment() + ▼ +assignment_knowledge_points ──(当前 assignment_id，有界最多 8 条)──┐ + ▼ +学生 POST /api/ask_question ──► services/knowledge_rag.py + │ │ + │ ├─ grounded：生成 [K1]...[Kn] 证据 + │ └─ no_result：NO_KNOWLEDGE_EVIDENCE 回退 + ▼ ▼ +guidance_generator ◄── 有界知识上下文（禁止编造引用） + │ + ├─ 普通 JSON：answer + knowledge_retrieval + └─ SSE：delta + done.answer + done.data.knowledge_retrieval +``` + +## 3. 生命周期与边界 + +1. 作业创建或后续识别流程绑定知识点，写入现有 `assignment_knowledge_points` 表。 +2. 学生提问时只按当前作业 ID读取显式绑定知识点，按权重降序、主键升序排序，最多取 8 条。 +3. 生成器只收到这组有限上下文；回答末尾追加确定性的证据回执。 +4. 没有知识点时不猜测、不读取任何学生画像，返回 `NO_KNOWLEDGE_EVIDENCE`，并说明回答仅基于题目和代码。 +5. 日志只记录状态、候选数、命中数、延迟、引用完整度和是否回退，不记录学生代码、问题或知识点私有分数。 + +本次没有新增数据库表/字段、权限规则、部署配置、Redis 依赖或外部向量服务；因此不改变数据库结构、权限、部署和现有接口的必需字段。原有 `answer` 字段仍保留，新增信息仅位于响应数据和答案末尾的证据回执中。 + +## 4. 指标定义与本地验证 + +| 指标 | 定义 | +| --- | --- | +| `candidate_count` | 当前作业查询到的候选绑定数 | +| `hit_count` | 生成有效证据的条数 | +| `retrieval_hit_rate` | `hit_count / candidate_count`；无候选时为 `0.0` | +| `retrieval_latency_ms` | 单次显式查询从开始到结果构建的本地耗时 | +| `citation_completeness` | 具有稳定 `evidence_id` 和 `[K]` 标记的证据占比 | +| `fallback` | 无结果时的可解释回退对象；当前代码为 `NO_KNOWLEDGE_EVIDENCE` | + +验证命令： + +```powershell +D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.venv\Scripts\python.exe -m pytest tests/test_knowledge_rag.py -q --disable-warnings +``` + +覆盖结果：4 passed。用例包含无知识点回退、有知识点 `[K1]` 引用与指标、SSE 首尾事件兼容，以及不读取学生私有评分的边界。 + +另外在隔离的 SQLite 测量环境中对同一作业的 2 条显式知识点连续检索 50 次，实测结果为： +`status=grounded`、`candidate_count=2`、`hit_count=2`、`retrieval_hit_rate=1.0`、 +`citation_completeness=1.0`、平均 `0.39 ms`、P95 `0.79 ms`。这是本机进程内的 +检索基线，不是生产端到端 SLA；测试启动时 Redis 不可用并自动回退到文件系统会话， +该环境现象也不纳入检索延迟结论。 + +## 5. 风险、回滚与未解决问题 + +- 主要取舍：复用现有显式绑定，换取低改动和可解释性；当前不做自然语言相关性排序、向量搜索或大规模召回。 +- 性能边界：每次最多读取 8 条当前作业知识点；`retrieval_latency_ms` 是应用内基线，不等同于生产端到端延迟。 +- 兼容性边界：普通 JSON 保留原有答案字段；SSE 保留 `start/delta/done` 事件，新增字段只在 `done` 中出现。 +- 未验证：真实 Redis、外部 AI 服务、生产数据库、并发压力和浏览器全流程；这些不应被本地单元测试结果替代。 +- 若后续发现答案展示不适配，可回滚本次提交；不需要数据库回滚或迁移。下一步再引入向量索引前，应先确定数据来源、脱敏、权限、更新策略和相关性评估集。 + +## 6. 后续维护说明 + +先运行上面的定向测试，再运行完整测试集。新增检索源时必须保持作业范围、证据 ID、回退码和学生隐私边界；任何需要数据库、权限、部署或外部索引变更的方案先作为待决策项，不在本小范围 PR 内直接落地。 diff --git a/routes/api.py b/routes/api.py index 4125181..f504a30 100644 --- a/routes/api.py +++ b/routes/api.py @@ -33,6 +33,11 @@ from services.ai_evaluator import AIEvaluator from services.api_keys import api_keys # 导入 API 密钥管理器 from services.demo_database import current_demo_run_id +from services.knowledge_rag import ( + build_knowledge_prompt_context, + render_knowledge_receipt, + retrieve_assignment_knowledge, +) from tasks.submission_tasks import evaluate_submission_async from tasks.submission_queue import ( SubmissionQueueUnavailable, @@ -326,6 +331,23 @@ def _text_chunks(text, size=120): yield text[index:index + size] +def _retrieve_knowledge_context(assignment_id): + """Retrieve assignment evidence and emit bounded operational metrics.""" + retrieval = retrieve_assignment_knowledge(assignment_id) + metrics = retrieval["metrics"] + current_app.logger.info( + "knowledge_rag status=%s candidates=%s hits=%s latency_ms=%.2f " + "citation_completeness=%.3f fallback=%s", + retrieval["status"], + metrics["candidate_count"], + metrics["hit_count"], + metrics["retrieval_latency_ms"], + metrics["citation_completeness"], + bool(retrieval.get("fallback")), + ) + return retrieval + + @api.route('/submit', methods=['POST']) @login_required @student_required @@ -723,6 +745,12 @@ def ask_question(): if not can_access_assignment(assignment, current_user): return error_response("您无权访问此作业", 403) + knowledge_retrieval = _retrieve_knowledge_context(assignment_id) + knowledge_prompt_context = build_knowledge_prompt_context( + knowledge_retrieval + ) + knowledge_receipt = render_knowledge_receipt(knowledge_retrieval) + # 仅对合法且有权限的请求计入冷却时间；同时容忍旧版或损坏的 # session 值，避免 fromisoformat 异常把一个普通请求变成 500。 now = datetime.utcnow() @@ -758,6 +786,7 @@ def stream_answer(): assignment_title=assignment.title, assignment_description=assignment.description, language=language, + knowledge_context=knowledge_prompt_context, ): if not chunk: continue @@ -779,6 +808,7 @@ def stream_answer(): formatted_answer = answer else: formatted_answer = '很抱歉，我无法理解您的问题或无法基于当前代码生成回答。请尝试重新表述您的问题或提供更多代码上下文。' + formatted_answer += knowledge_receipt if student_id: try: @@ -801,7 +831,11 @@ def stream_answer(): 'done': True, 'content': formatted_answer, 'answer': formatted_answer, - 'data': {'answer': formatted_answer}, + 'data': { + 'answer': formatted_answer, + 'knowledge_retrieval': knowledge_retrieval, + }, + 'knowledge_retrieval': knowledge_retrieval, }) except Exception as stream_error: db.session.rollback() @@ -820,7 +854,8 @@ def stream_answer(): question=question, assignment_title=assignment.title, assignment_description=assignment.description, - language=language + language=language, + knowledge_context=knowledge_prompt_context, ) # 输出调试信息 @@ -847,6 +882,8 @@ def stream_answer(): formatted_answer = f"

{escaped_answer}

" else: formatted_answer = "很抱歉，我无法理解您的问题或无法基于当前代码生成回答。请尝试重新表述您的问题或提供更多代码上下文。" + + formatted_answer += knowledge_receipt # 记录学生提问日志 if student_id: @@ -871,7 +908,8 @@ def stream_answer(): success=True, message="问题回答成功", data={ - 'answer': formatted_answer + 'answer': formatted_answer, + 'knowledge_retrieval': knowledge_retrieval, } ) diff --git a/services/knowledge_rag.py b/services/knowledge_rag.py new file mode 100644 index 0000000..39f1f9d --- /dev/null +++ b/services/knowledge_rag.py @@ -0,0 +1,126 @@ +"""Bounded, assignment-scoped knowledge retrieval for student answers. + +This is an explicit-evidence baseline, not a vector database. It reads only +the knowledge points attached to the current assignment and returns stable, +non-sensitive citations plus a deterministic no-result state. Keeping the +retriever here makes a future index interchangeable without changing the +student-facing answer route. +""" + +from __future__ import annotations + +import time + +from models import AssignmentKnowledgePoint, KnowledgePointScore + + +MAX_EVIDENCE = 8 +NO_KNOWLEDGE_EVIDENCE = { + "code": "NO_KNOWLEDGE_EVIDENCE", + "message": "当前作业没有已标注知识点，回答仅基于题目和代码。", +} + + +def _created_at_value(record): + created_at = getattr(record, "created_at", None) + return created_at.isoformat() if created_at else None + + +def _result(status, evidence, candidate_count, started_at, fallback=None): + hit_count = len(evidence) + latency_ms = max(0.0, (time.perf_counter() - started_at) * 1000.0) + metrics = { + "candidate_count": int(candidate_count), + "hit_count": hit_count, + "retrieval_hit_rate": round(hit_count / candidate_count, 3) + if candidate_count + else 0.0, + "retrieval_latency_ms": round(latency_ms, 2), + "citation_completeness": round( + sum(1 for item in evidence if item.get("evidence_id") and item.get("citation")) + / hit_count, + 3, + ) if hit_count else 0.0, + } + return { + "status": status, + "evidence": evidence, + "metrics": metrics, + "fallback": fallback, + } + + +def retrieve_assignment_knowledge(assignment_id, *, limit=MAX_EVIDENCE): + """Retrieve bounded, explicit knowledge evidence for one assignment. + + The retrieval is intentionally assignment-scoped and does not inspect a + student's private ``KnowledgePointScore`` rows. Evidence order follows + the teacher/AI-maintained weight and then the stable row id. + """ + + started_at = time.perf_counter() + try: + assignment_id = int(assignment_id) + except (TypeError, ValueError): + return _result("no_result", [], 0, started_at, NO_KNOWLEDGE_EVIDENCE.copy()) + + bounded_limit = max(1, min(int(limit), MAX_EVIDENCE)) + records = ( + AssignmentKnowledgePoint.query + .filter_by(assignment_id=assignment_id) + .order_by( + AssignmentKnowledgePoint.weight.desc(), + AssignmentKnowledgePoint.id.asc(), + ) + .limit(bounded_limit) + .all() + ) + + evidence = [] + for record in records: + code = str(record.knowledge_point or "").strip() + if not code: + continue + name = KnowledgePointScore.KNOWLEDGE_POINTS.get(code, code) + evidence.append({ + "evidence_id": f"assignment-kp:{record.id}", + "citation": f"[K{len(evidence) + 1}]", + "source_type": "assignment_knowledge_point", + "title": name, + "content": f"当前作业显式绑定知识点：{name}（{code}）。", + "created_at": _created_at_value(record), + }) + + if not evidence: + return _result("no_result", [], len(records), started_at, NO_KNOWLEDGE_EVIDENCE.copy()) + return _result("grounded", evidence, len(records), started_at) + + +def build_knowledge_prompt_context(retrieval): + """Create a bounded prompt section that makes citation limits explicit.""" + + if retrieval.get("status") != "grounded": + return NO_KNOWLEDGE_EVIDENCE["message"] + " 不要编造知识库引用。" + + lines = [ + "以下是当前作业已检索到的知识证据。只能使用这些证据，不要扩展为未提供的资料；引用时使用对应标记：" + ] + lines.extend( + f"{item['citation']} {item['content']}" + for item in retrieval.get("evidence", []) + ) + return "\n".join(lines) + + +def render_knowledge_receipt(retrieval): + """Render a deterministic evidence receipt for the final student answer.""" + + if retrieval.get("status") != "grounded": + return f"\n\n> 知识证据回退：{NO_KNOWLEDGE_EVIDENCE['message']}" + + lines = ["\n\n### 参考知识证据"] + lines.extend( + f"- {item['citation']} {item['title']}" + for item in retrieval.get("evidence", []) + ) + return "\n".join(lines) diff --git a/tests/test_knowledge_rag.py b/tests/test_knowledge_rag.py new file mode 100644 index 0000000..67d82bb --- /dev/null +++ b/tests/test_knowledge_rag.py @@ -0,0 +1,167 @@ +import json + +import pytest + +from app import create_app +from config import TestingConfig as _TestingConfig +from models import Assignment, AssignmentKnowledgePoint, KnowledgePointScore, User, db +from routes import api as api_routes +from services.knowledge_rag import retrieve_assignment_knowledge + + +@pytest.fixture +def knowledge_context(tmp_path, monkeypatch): + database_path = tmp_path / "knowledge_rag.db" + monkeypatch.setattr( + _TestingConfig, + "SQLALCHEMY_DATABASE_URI", + f"sqlite:///{database_path}", + ) + app = create_app("testing") + with app.app_context(): + db.create_all() + student = User( + student_id="rag-student", + username="rag-student", + usertype="学生", + ) + student.password = "password" + assignment = Assignment( + title="数组边界题", + description="请处理数组输入并说明边界条件。", + creator_id="rag-student", + ) + db.session.add_all([student, assignment]) + db.session.commit() + assignment_id = assignment.id + + client = app.test_client() + login = client.post( + "/login", + data={"username": "rag-student", "password": "password"}, + ) + assert login.status_code in {302, 303} + yield app, client, assignment_id + + with app.app_context(): + db.session.remove() + db.drop_all() + + +def test_ask_question_exposes_retrieval_evidence_and_fallback_state( + knowledge_context, monkeypatch +): + _, client, assignment_id = knowledge_context + monkeypatch.setattr( + api_routes, + "generate_answer_to_question", + lambda **_: "请检查数组下标与边界条件。", + ) + + response = client.post( + "/api/ask_question", + json={ + "assignment_id": assignment_id, + "code": "int main(){return 0;}", + "question": "数组边界怎么检查？", + }, + ) + + assert response.status_code == 200 + data = response.json["data"] + assert data["knowledge_retrieval"]["status"] == "no_result" + assert data["knowledge_retrieval"]["fallback"]["code"] == "NO_KNOWLEDGE_EVIDENCE" + assert "没有已标注知识点" in data["answer"] + + +def test_ask_question_returns_scoped_citations_and_metrics(knowledge_context, monkeypatch): + app, client, assignment_id = knowledge_context + with app.app_context(): + AssignmentKnowledgePoint.add_to_assignment( + assignment_id, + "array", + weight=1.5, + auto_detected=True, + ) + + captured = {} + + def fake_answer(**kwargs): + captured.update(kwargs) + return "先检查数组下标是否始终落在有效范围内。" + + monkeypatch.setattr(api_routes, "generate_answer_to_question", fake_answer) + response = client.post( + "/api/ask_question", + json={ + "assignment_id": assignment_id, + "code": "int main(){return 0;}", + "question": "数组边界怎么检查？", + }, + ) + + assert response.status_code == 200 + data = response.json["data"] + retrieval = data["knowledge_retrieval"] + assert retrieval["status"] == "grounded" + assert retrieval["metrics"]["candidate_count"] == 1 + assert retrieval["metrics"]["hit_count"] == 1 + assert retrieval["metrics"]["retrieval_hit_rate"] == 1.0 + assert retrieval["metrics"]["citation_completeness"] == 1.0 + assert retrieval["evidence"][0]["citation"] == "[K1]" + assert "数组" in captured["knowledge_context"] + assert "[K1]" in data["answer"] + assert "参考知识证据" in data["answer"] + + +def test_ask_question_sse_includes_retrieval_receipt(knowledge_context, monkeypatch): + _, client, assignment_id = knowledge_context + captured = {} + + def fake_stream(**kwargs): + captured.update(kwargs) + return iter(["先手动追踪边界值。"]) + + monkeypatch.setattr(api_routes, "generate_answer_to_question_stream", fake_stream) + response = client.post( + "/api/ask_question", + json={ + "assignment_id": assignment_id, + "code": "int main(){return 0;}", + "question": "边界值怎么检查？", + }, + headers={"Accept": "text/event-stream"}, + ) + + events = [ + json.loads(line[6:]) + for line in response.data.decode("utf-8").splitlines() + if line.startswith("data: ") + ] + assert response.status_code == 200 + assert [event["type"] for event in events] == ["start", "delta", "done"] + done = events[-1] + assert done["knowledge_retrieval"]["status"] == "no_result" + assert done["data"]["knowledge_retrieval"]["fallback"]["code"] == ( + "NO_KNOWLEDGE_EVIDENCE" + ) + assert "没有已标注知识点" in done["answer"] + assert "不要编造知识库引用" in captured["knowledge_context"] + + +def test_retriever_does_not_read_student_private_scores(knowledge_context): + app, _, assignment_id = knowledge_context + with app.app_context(): + db.session.add( + KnowledgePointScore( + student_id="rag-student", + knowledge_point="array", + score=99.0, + ) + ) + db.session.commit() + retrieval = retrieve_assignment_knowledge(assignment_id) + + assert retrieval["status"] == "no_result" + assert retrieval["metrics"]["candidate_count"] == 0 + assert retrieval["metrics"]["hit_count"] == 0 diff --git a/utils/guidance_generator.py b/utils/guidance_generator.py index 8faa743..d106042 100644 --- a/utils/guidance_generator.py +++ b/utils/guidance_generator.py @@ -417,7 +417,15 @@ def generate_guidance(code, assignment_title, assignment_description, language=" # 使用基于规则的生成器 return generate_rule_based_guidance(code, assignment_title, assignment_description, language) -def generate_answer_to_question(code, question, assignment_title, assignment_description, language="cpp", _stream=False): +def generate_answer_to_question( + code, + question, + assignment_title, + assignment_description, + language="cpp", + _stream=False, + knowledge_context=None, +): """ 根据学生的提问和代码生成回答 code: 学生当前编写的代码 @@ -480,6 +488,13 @@ def generate_answer_to_question(code, question, assignment_title, assignment_des 请根据以上原则，给出引导性回答（不超过300字，重点突出，语气友好）。 如果问题涉及代码错误，指出问题的"方向"而非"答案"。 """ + prompt += f""" + +## 知识证据边界 +{knowledge_context or '当前没有已检索的知识库证据。不要编造知识点引用，回答仅基于题目和代码。'} +如果使用上面的证据，请在相关内容后标注对应的 [K] 引用；没有证据时不要生成引用标记。 +""" + print(f"发送问题到大模型API，提示词长度: {len(prompt)}") if _stream: @@ -524,8 +539,14 @@ def generate_guidance_stream(code, assignment_title, assignment_description, lan yield from result -def generate_answer_to_question_stream(code, question, assignment_title, - assignment_description, language="cpp"): +def generate_answer_to_question_stream( + code, + question, + assignment_title, + assignment_description, + language="cpp", + knowledge_context=None, +): """Stream an answer to a code question while keeping JSON callers intact.""" result = generate_answer_to_question( code, @@ -534,6 +555,7 @@ def generate_answer_to_question_stream(code, question, assignment_title, assignment_description, language, _stream=True, + knowledge_context=knowledge_context, ) if isinstance(result, str): yield result From b4ce906ce3892bbab6d991d96a52dbddebdb42b9 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 20:00:39 +0800 Subject: [PATCH 2/7] fix: expose explicit no-result fallback metric --- KNOWLEDGE_RAG_STAGE10.md | 1 + services/knowledge_rag.py | 1 + tests/test_knowledge_rag.py | 3 +++ 3 files changed, 5 insertions(+) diff --git a/KNOWLEDGE_RAG_STAGE10.md b/KNOWLEDGE_RAG_STAGE10.md index 5cfc9e2..6f5bc56 100644 --- a/KNOWLEDGE_RAG_STAGE10.md +++ b/KNOWLEDGE_RAG_STAGE10.md @@ -56,6 +56,7 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） | `retrieval_hit_rate` | `hit_count / candidate_count`；无候选时为 `0.0` | | `retrieval_latency_ms` | 单次显式查询从开始到结果构建的本地耗时 | | `citation_completeness` | 具有稳定 `evidence_id` 和 `[K]` 标记的证据占比 | +| `no_result_fallback` | 是否发生无结果回退；发生时为 `true` | | `fallback` | 无结果时的可解释回退对象；当前代码为 `NO_KNOWLEDGE_EVIDENCE` | 验证命令： diff --git a/services/knowledge_rag.py b/services/knowledge_rag.py index 39f1f9d..52d3aeb 100644 --- a/services/knowledge_rag.py +++ b/services/knowledge_rag.py @@ -41,6 +41,7 @@ def _result(status, evidence, candidate_count, started_at, fallback=None): / hit_count, 3, ) if hit_count else 0.0, + "no_result_fallback": bool(fallback), } return { "status": status, diff --git a/tests/test_knowledge_rag.py b/tests/test_knowledge_rag.py index 67d82bb..3bda07a 100644 --- a/tests/test_knowledge_rag.py +++ b/tests/test_knowledge_rag.py @@ -71,6 +71,7 @@ def test_ask_question_exposes_retrieval_evidence_and_fallback_state( data = response.json["data"] assert data["knowledge_retrieval"]["status"] == "no_result" assert data["knowledge_retrieval"]["fallback"]["code"] == "NO_KNOWLEDGE_EVIDENCE" + assert data["knowledge_retrieval"]["metrics"]["no_result_fallback"] is True assert "没有已标注知识点" in data["answer"] @@ -108,6 +109,7 @@ def fake_answer(**kwargs): assert retrieval["metrics"]["hit_count"] == 1 assert retrieval["metrics"]["retrieval_hit_rate"] == 1.0 assert retrieval["metrics"]["citation_completeness"] == 1.0 + assert retrieval["metrics"]["no_result_fallback"] is False assert retrieval["evidence"][0]["citation"] == "[K1]" assert "数组" in captured["knowledge_context"] assert "[K1]" in data["answer"] @@ -142,6 +144,7 @@ def fake_stream(**kwargs): assert [event["type"] for event in events] == ["start", "delta", "done"] done = events[-1] assert done["knowledge_retrieval"]["status"] == "no_result" + assert done["knowledge_retrieval"]["metrics"]["no_result_fallback"] is True assert done["data"]["knowledge_retrieval"]["fallback"]["code"] == ( "NO_KNOWLEDGE_EVIDENCE" ) From 6d6a4511c7f9533d475b60454ef67806cc9ff583 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 20:20:39 +0800 Subject: [PATCH 3/7] fix: degrade safely when knowledge source fails --- KNOWLEDGE_RAG_STAGE10.md | 8 +++-- services/knowledge_rag.py | 54 +++++++++++++++++++++++-------- tests/test_knowledge_rag.py | 64 +++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 17 deletions(-) diff --git a/KNOWLEDGE_RAG_STAGE10.md b/KNOWLEDGE_RAG_STAGE10.md index 6f5bc56..310aa3c 100644 --- a/KNOWLEDGE_RAG_STAGE10.md +++ b/KNOWLEDGE_RAG_STAGE10.md @@ -43,7 +43,8 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） 2. 学生提问时只按当前作业 ID读取显式绑定知识点，按权重降序、主键升序排序，最多取 8 条。 3. 生成器只收到这组有限上下文；回答末尾追加确定性的证据回执。 4. 没有知识点时不猜测、不读取任何学生画像，返回 `NO_KNOWLEDGE_EVIDENCE`，并说明回答仅基于题目和代码。 -5. 日志只记录状态、候选数、命中数、延迟、引用完整度和是否回退，不记录学生代码、问题或知识点私有分数。 +5. 知识源查询异常时回滚当前查询事务，返回 `KNOWLEDGE_RETRIEVAL_UNAVAILABLE`，仍允许回答链路继续，并明确说明知识证据暂不可用。 +6. 日志只记录状态、候选数、命中数、延迟、引用完整度和是否回退，不记录学生代码、问题或知识点私有分数。 本次没有新增数据库表/字段、权限规则、部署配置、Redis 依赖或外部向量服务；因此不改变数据库结构、权限、部署和现有接口的必需字段。原有 `answer` 字段仍保留，新增信息仅位于响应数据和答案末尾的证据回执中。 @@ -56,7 +57,8 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） | `retrieval_hit_rate` | `hit_count / candidate_count`；无候选时为 `0.0` | | `retrieval_latency_ms` | 单次显式查询从开始到结果构建的本地耗时 | | `citation_completeness` | 具有稳定 `evidence_id` 和 `[K]` 标记的证据占比 | -| `no_result_fallback` | 是否发生无结果回退；发生时为 `true` | +| `no_result_fallback` | 是否发生“没有已标注知识点”的回退；发生时为 `true` | +| `retrieval_error_fallback` | 知识源查询异常时是否安全降级；发生时为 `true` | | `fallback` | 无结果时的可解释回退对象；当前代码为 `NO_KNOWLEDGE_EVIDENCE` | 验证命令： @@ -65,7 +67,7 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.venv\Scripts\python.exe -m pytest tests/test_knowledge_rag.py -q --disable-warnings ``` -覆盖结果：4 passed。用例包含无知识点回退、有知识点 `[K1]` 引用与指标、SSE 首尾事件兼容，以及不读取学生私有评分的边界。 +覆盖结果：6 passed。用例包含无知识点回退、有知识点 `[K1]` 引用与指标、SSE 首尾事件兼容、知识源异常安全降级、回答链路降级，以及不读取学生私有评分的边界。 另外在隔离的 SQLite 测量环境中对同一作业的 2 条显式知识点连续检索 50 次，实测结果为： `status=grounded`、`candidate_count=2`、`hit_count=2`、`retrieval_hit_rate=1.0`、 diff --git a/services/knowledge_rag.py b/services/knowledge_rag.py index 52d3aeb..a069a14 100644 --- a/services/knowledge_rag.py +++ b/services/knowledge_rag.py @@ -9,16 +9,22 @@ from __future__ import annotations +import logging import time -from models import AssignmentKnowledgePoint, KnowledgePointScore +from models import AssignmentKnowledgePoint, KnowledgePointScore, db MAX_EVIDENCE = 8 +logger = logging.getLogger(__name__) NO_KNOWLEDGE_EVIDENCE = { "code": "NO_KNOWLEDGE_EVIDENCE", "message": "当前作业没有已标注知识点，回答仅基于题目和代码。", } +RETRIEVAL_UNAVAILABLE = { + "code": "KNOWLEDGE_RETRIEVAL_UNAVAILABLE", + "message": "知识证据暂时不可用，回答仅基于题目和代码。", +} def _created_at_value(record): @@ -41,7 +47,12 @@ def _result(status, evidence, candidate_count, started_at, fallback=None): / hit_count, 3, ) if hit_count else 0.0, - "no_result_fallback": bool(fallback), + "no_result_fallback": bool( + fallback and fallback.get("code") == NO_KNOWLEDGE_EVIDENCE["code"] + ), + "retrieval_error_fallback": bool( + fallback and fallback.get("code") == RETRIEVAL_UNAVAILABLE["code"] + ), } return { "status": status, @@ -65,17 +76,30 @@ def retrieve_assignment_knowledge(assignment_id, *, limit=MAX_EVIDENCE): except (TypeError, ValueError): return _result("no_result", [], 0, started_at, NO_KNOWLEDGE_EVIDENCE.copy()) - bounded_limit = max(1, min(int(limit), MAX_EVIDENCE)) - records = ( - AssignmentKnowledgePoint.query - .filter_by(assignment_id=assignment_id) - .order_by( - AssignmentKnowledgePoint.weight.desc(), - AssignmentKnowledgePoint.id.asc(), + try: + bounded_limit = max(1, min(int(limit), MAX_EVIDENCE)) + records = ( + AssignmentKnowledgePoint.query + .filter_by(assignment_id=assignment_id) + .order_by( + AssignmentKnowledgePoint.weight.desc(), + AssignmentKnowledgePoint.id.asc(), + ) + .limit(bounded_limit) + .all() + ) + except Exception: + db.session.rollback() + logger.exception( + "knowledge evidence retrieval failed; using safe answer-only fallback" + ) + return _result( + "unavailable", + [], + 0, + started_at, + RETRIEVAL_UNAVAILABLE.copy(), ) - .limit(bounded_limit) - .all() - ) evidence = [] for record in records: @@ -101,7 +125,8 @@ def build_knowledge_prompt_context(retrieval): """Create a bounded prompt section that makes citation limits explicit.""" if retrieval.get("status") != "grounded": - return NO_KNOWLEDGE_EVIDENCE["message"] + " 不要编造知识库引用。" + fallback = retrieval.get("fallback") or NO_KNOWLEDGE_EVIDENCE + return fallback["message"] + " 不要编造知识库引用。" lines = [ "以下是当前作业已检索到的知识证据。只能使用这些证据，不要扩展为未提供的资料；引用时使用对应标记：" @@ -117,7 +142,8 @@ def render_knowledge_receipt(retrieval): """Render a deterministic evidence receipt for the final student answer.""" if retrieval.get("status") != "grounded": - return f"\n\n> 知识证据回退：{NO_KNOWLEDGE_EVIDENCE['message']}" + fallback = retrieval.get("fallback") or NO_KNOWLEDGE_EVIDENCE + return f"\n\n> 知识证据回退：{fallback['message']}" lines = ["\n\n### 参考知识证据"] lines.extend( diff --git a/tests/test_knowledge_rag.py b/tests/test_knowledge_rag.py index 3bda07a..c2706f1 100644 --- a/tests/test_knowledge_rag.py +++ b/tests/test_knowledge_rag.py @@ -6,6 +6,7 @@ from config import TestingConfig as _TestingConfig from models import Assignment, AssignmentKnowledgePoint, KnowledgePointScore, User, db from routes import api as api_routes +from services import knowledge_rag from services.knowledge_rag import retrieve_assignment_knowledge @@ -168,3 +169,66 @@ def test_retriever_does_not_read_student_private_scores(knowledge_context): assert retrieval["status"] == "no_result" assert retrieval["metrics"]["candidate_count"] == 0 assert retrieval["metrics"]["hit_count"] == 0 + + +def test_retriever_returns_safe_fallback_when_knowledge_source_is_unavailable( + knowledge_context, monkeypatch +): + app, _, assignment_id = knowledge_context + + class BrokenQuery: + def filter_by(self, **_kwargs): + raise RuntimeError("knowledge table unavailable") + + with app.app_context(): + monkeypatch.setattr( + knowledge_rag.AssignmentKnowledgePoint, + "query", + BrokenQuery(), + ) + retrieval = retrieve_assignment_knowledge(assignment_id) + + assert retrieval["status"] == "unavailable" + assert retrieval["fallback"]["code"] == "KNOWLEDGE_RETRIEVAL_UNAVAILABLE" + assert retrieval["metrics"]["no_result_fallback"] is False + assert retrieval["metrics"]["retrieval_error_fallback"] is True + assert "暂时不可用" in knowledge_rag.build_knowledge_prompt_context(retrieval) + assert "暂时不可用" in knowledge_rag.render_knowledge_receipt(retrieval) + + +def test_ask_question_continues_with_answer_only_when_knowledge_source_is_unavailable( + knowledge_context, monkeypatch +): + app, client, assignment_id = knowledge_context + + class BrokenQuery: + def filter_by(self, **_kwargs): + raise RuntimeError("knowledge table unavailable") + + with app.app_context(): + monkeypatch.setattr( + knowledge_rag.AssignmentKnowledgePoint, + "query", + BrokenQuery(), + ) + + monkeypatch.setattr( + api_routes, + "generate_answer_to_question", + lambda **_: "请先检查边界条件。", + ) + response = client.post( + "/api/ask_question", + json={ + "assignment_id": assignment_id, + "code": "int main(){return 0;}", + "question": "边界值怎么检查？", + }, + ) + + assert response.status_code == 200 + retrieval = response.json["data"]["knowledge_retrieval"] + assert retrieval["status"] == "unavailable" + assert retrieval["fallback"]["code"] == "KNOWLEDGE_RETRIEVAL_UNAVAILABLE" + assert retrieval["metrics"]["retrieval_error_fallback"] is True + assert "知识证据暂时不可用" in response.json["data"]["answer"] From 93ab2f193c0596269eb83b26c20d177e52ad4ce6 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 20:37:05 +0800 Subject: [PATCH 4/7] fix: distinguish knowledge fallback metrics in logs --- routes/api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/routes/api.py b/routes/api.py index f504a30..4666910 100644 --- a/routes/api.py +++ b/routes/api.py @@ -337,13 +337,16 @@ def _retrieve_knowledge_context(assignment_id): metrics = retrieval["metrics"] current_app.logger.info( "knowledge_rag status=%s candidates=%s hits=%s latency_ms=%.2f " - "citation_completeness=%.3f fallback=%s", + "citation_completeness=%.3f no_result_fallback=%s " + "retrieval_error_fallback=%s fallback_code=%s", retrieval["status"], metrics["candidate_count"], metrics["hit_count"], metrics["retrieval_latency_ms"], metrics["citation_completeness"], - bool(retrieval.get("fallback")), + metrics["no_result_fallback"], + metrics["retrieval_error_fallback"], + (retrieval.get("fallback") or {}).get("code"), ) return retrieval From 1cc54d2490e0ffa3489675a9f55924cdea685189 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 20:39:22 +0800 Subject: [PATCH 5/7] docs: refresh stage10 verification baseline --- KNOWLEDGE_RAG_STAGE10.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/KNOWLEDGE_RAG_STAGE10.md b/KNOWLEDGE_RAG_STAGE10.md index 310aa3c..d59d4c3 100644 --- a/KNOWLEDGE_RAG_STAGE10.md +++ b/KNOWLEDGE_RAG_STAGE10.md @@ -75,6 +75,14 @@ D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.v 检索基线，不是生产端到端 SLA；测试启动时 Redis 不可用并自动回退到文件系统会话， 该环境现象也不纳入检索延迟结论。 +在加入知识源异常降级和日志指标区分后的完整验证命令为： + +```powershell +D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.venv\Scripts\python.exe -m pytest -q --disable-warnings +``` + +实测结果：`658 passed`，退出码 0，耗时 `13:23`。 + ## 5. 风险、回滚与未解决问题 - 主要取舍：复用现有显式绑定，换取低改动和可解释性；当前不做自然语言相关性排序、向量搜索或大规模召回。 From eac16dd9ad35886c450d419a16b3bc16168007f7 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 21:43:34 +0800 Subject: [PATCH 6/7] fix: normalize knowledge receipt rendering --- KNOWLEDGE_RAG_STAGE10.md | 9 +++-- routes/api.py | 72 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/KNOWLEDGE_RAG_STAGE10.md b/KNOWLEDGE_RAG_STAGE10.md index d59d4c3..77edc45 100644 --- a/KNOWLEDGE_RAG_STAGE10.md +++ b/KNOWLEDGE_RAG_STAGE10.md @@ -59,7 +59,10 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） | `citation_completeness` | 具有稳定 `evidence_id` 和 `[K]` 标记的证据占比 | | `no_result_fallback` | 是否发生“没有已标注知识点”的回退；发生时为 `true` | | `retrieval_error_fallback` | 知识源查询异常时是否安全降级；发生时为 `true` | -| `fallback` | 无结果时的可解释回退对象；当前代码为 `NO_KNOWLEDGE_EVIDENCE` | +| `fallback` | 回退时的可解释对象；无知识点为 `NO_KNOWLEDGE_EVIDENCE`，查询异常为 `KNOWLEDGE_RETRIEVAL_UNAVAILABLE` | + +以上是检索记录层指标：`candidate_count` 是上限截断后的候选数， +`citation_completeness` 只衡量证据字段是否完整，不能代表答案中的引用正确率。 验证命令： @@ -67,7 +70,7 @@ guidance_generator ◄── 有界知识上下文（禁止编造引用） D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.venv\Scripts\python.exe -m pytest tests/test_knowledge_rag.py -q --disable-warnings ``` -覆盖结果：6 passed。用例包含无知识点回退、有知识点 `[K1]` 引用与指标、SSE 首尾事件兼容、知识源异常安全降级、回答链路降级，以及不读取学生私有评分的边界。 +覆盖结果：9 passed。用例包含无知识点回退、有知识点 `[K1]` 引用与指标、SSE 首尾事件兼容、知识源异常安全降级、回答链路降级、作业隔离、8 条截断排序、标题转义，以及不读取学生私有评分的边界。 另外在隔离的 SQLite 测量环境中对同一作业的 2 条显式知识点连续检索 50 次，实测结果为： `status=grounded`、`candidate_count=2`、`hit_count=2`、`retrieval_hit_rate=1.0`、 @@ -81,7 +84,7 @@ D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.v D:\xproject\新建文件夹\CodeSense-main\pr-student-learning-route-worktree\.venv\Scripts\python.exe -m pytest -q --disable-warnings ``` -实测结果：`658 passed`，退出码 0，耗时 `13:23`。 +实测结果：`661 passed`，退出码 0，耗时 `14:30`。 ## 5. 风险、回滚与未解决问题 diff --git a/routes/api.py b/routes/api.py index 4666910..7f42c8b 100644 --- a/routes/api.py +++ b/routes/api.py @@ -28,6 +28,9 @@ generate_answer_to_question_stream, ) # 导入指导生成函数和答案生成函数 from utils.code_advisor import generate_code_advice # 导入新的代码建议系统 +from utils.markdown_formatter import ( + enhance_code_blocks as format_markdown_code_blocks, +) from utils.sse import sse_event, sse_response, stream_text_chunks, wants_sse from utils.upload_safety import UploadValidationError, validate_upload from services.ai_evaluator import AIEvaluator @@ -47,10 +50,15 @@ import json import os from datetime import datetime +from html import escape api = Blueprint('api', __name__, url_prefix='/api') _SUPPORTED_LANGUAGES = frozenset({'cpp', 'c++', 'c', 'python', 'py', 'java'}) +_QUESTION_ANSWER_FALLBACK = ( + '很抱歉，我无法理解您的问题或无法基于当前代码生成回答。' + '请尝试重新表述您的问题或提供更多代码上下文。' +) def _json_object(): @@ -351,6 +359,25 @@ def _retrieve_knowledge_context(assignment_id): return retrieval +def _format_question_answer(answer, knowledge_receipt, language, student_id): + """Format the answer and evidence receipt through one Markdown path.""" + + answer_text = str(answer or _QUESTION_ANSWER_FALLBACK) + combined = answer_text + str(knowledge_receipt or '') + try: + return format_markdown_code_blocks( + combined, + default_lang=language, + ) + except Exception: + current_app.logger.exception( + '学生提问 Markdown 格式化失败 student_id=%s', + student_id, + ) + escaped_answer = escape(combined).replace('\n', '
') + return f'

{escaped_answer}

' + + @api.route('/submit', methods=['POST']) @login_required @student_required @@ -801,17 +828,12 @@ def stream_answer(): }) answer = ''.join(chunks) - if answer: - try: - formatted_answer = enhance_code_blocks( - enhance_markdown(answer), - default_lang=language, - ) - except Exception: - formatted_answer = answer - else: - formatted_answer = '很抱歉，我无法理解您的问题或无法基于当前代码生成回答。请尝试重新表述您的问题或提供更多代码上下文。' - formatted_answer += knowledge_receipt + formatted_answer = _format_question_answer( + answer, + knowledge_receipt, + language, + student_id, + ) if student_id: try: @@ -865,28 +887,12 @@ def stream_answer(): current_app.logger.debug('学生提问 AI 回答已生成 student_id=%s answer_length=%s', student_id, len(answer) if answer else 0) - # 使用markdown库正确地将Markdown转换为HTML - if answer: - try: - # 增强Markdown格式，确保标题正确渲染 - answer = enhance_markdown(answer) - - # 增强代码块 - enhanced_answer = enhance_code_blocks(answer, default_lang=language) - - # 直接返回Markdown文本，不转换为HTML - formatted_answer = enhanced_answer - - # 输出调试信息 - except Exception: - current_app.logger.exception('学生提问 Markdown 格式化失败 student_id=%s', student_id) - # 如果Markdown转换失败，至少返回纯文本 - escaped_answer = answer.replace('<', '<').replace('>', '>').replace('\n', '
') - formatted_answer = f"

{escaped_answer}

" - else: - formatted_answer = "很抱歉，我无法理解您的问题或无法基于当前代码生成回答。请尝试重新表述您的问题或提供更多代码上下文。" - - formatted_answer += knowledge_receipt + formatted_answer = _format_question_answer( + answer, + knowledge_receipt, + language, + student_id, + ) # 记录学生提问日志 if student_id: From 08dae271a97dfedeaba134246a754dd8d02bae83 Mon Sep 17 00:00:00 2001 From: xjx-star Date: Mon, 14 Sep 2026 21:43:54 +0800 Subject: [PATCH 7/7] test: cover stage10 retrieval boundaries --- services/knowledge_rag.py | 12 +++++++- tests/test_knowledge_rag.py | 61 +++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/services/knowledge_rag.py b/services/knowledge_rag.py index a069a14..18273be 100644 --- a/services/knowledge_rag.py +++ b/services/knowledge_rag.py @@ -10,13 +10,16 @@ from __future__ import annotations import logging +import re import time +from html import escape from models import AssignmentKnowledgePoint, KnowledgePointScore, db MAX_EVIDENCE = 8 logger = logging.getLogger(__name__) +_MARKDOWN_SPECIAL = re.compile(r"([\\`*_\[\]{}()#+.!|>~-])") NO_KNOWLEDGE_EVIDENCE = { "code": "NO_KNOWLEDGE_EVIDENCE", "message": "当前作业没有已标注知识点，回答仅基于题目和代码。", @@ -32,6 +35,13 @@ def _created_at_value(record): return created_at.isoformat() if created_at else None +def _safe_markdown_label(value): + """Escape a knowledge label before inserting it into Markdown output.""" + + escaped = escape(str(value or ""), quote=True) + return _MARKDOWN_SPECIAL.sub(r"\\\1", escaped) + + def _result(status, evidence, candidate_count, started_at, fallback=None): hit_count = len(evidence) latency_ms = max(0.0, (time.perf_counter() - started_at) * 1000.0) @@ -147,7 +157,7 @@ def render_knowledge_receipt(retrieval): lines = ["\n\n### 参考知识证据"] lines.extend( - f"- {item['citation']} {item['title']}" + f"- {item['citation']} {_safe_markdown_label(item['title'])}" for item in retrieval.get("evidence", []) ) return "\n".join(lines) diff --git a/tests/test_knowledge_rag.py b/tests/test_knowledge_rag.py index c2706f1..1ece3f5 100644 --- a/tests/test_knowledge_rag.py +++ b/tests/test_knowledge_rag.py @@ -232,3 +232,64 @@ def filter_by(self, **_kwargs): assert retrieval["fallback"]["code"] == "KNOWLEDGE_RETRIEVAL_UNAVAILABLE" assert retrieval["metrics"]["retrieval_error_fallback"] is True assert "知识证据暂时不可用" in response.json["data"]["answer"] + + +def test_retriever_is_scoped_to_one_assignment(knowledge_context): + app, _, assignment_id = knowledge_context + with app.app_context(): + other_assignment = Assignment( + title="其他作业", + description="不应被当前作业检索到。", + creator_id="rag-student", + ) + db.session.add(other_assignment) + db.session.commit() + AssignmentKnowledgePoint.add_to_assignment( + other_assignment.id, + "other-only", + ) + retrieval = retrieve_assignment_knowledge(assignment_id) + + assert retrieval["status"] == "no_result" + assert all("other-only" not in item["content"] for item in retrieval["evidence"]) + + +def test_retriever_caps_evidence_and_orders_by_weight(knowledge_context): + app, _, assignment_id = knowledge_context + with app.app_context(): + db.session.add_all( + [ + AssignmentKnowledgePoint( + assignment_id=assignment_id, + knowledge_point=f"custom-{index}", + weight=float(index), + ) + for index in range(10) + ] + ) + db.session.commit() + retrieval = retrieve_assignment_knowledge(assignment_id) + + assert retrieval["status"] == "grounded" + assert len(retrieval["evidence"]) == 8 + assert "custom-9" in retrieval["evidence"][0]["content"] + assert "custom-2" in retrieval["evidence"][-1]["content"] + assert retrieval["metrics"]["candidate_count"] == 8 + + +def test_knowledge_receipt_escapes_untrusted_title(): + receipt = knowledge_rag.render_knowledge_receipt( + { + "status": "grounded", + "evidence": [ + { + "citation": "[K1]", + "title": " [unsafe]", + } + ], + } + ) + + assert "