diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 4071f21..6576c80 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -4,5 +4,9 @@ from app.api.chat import router as chat_router from app.api.auth import router as auth_router from app.api.retrieve import router as retrieve_router +from app.api.conversations import router as conversations_router -__all__ = ["health_router", "upload_router", "documents_router", "chat_router", "auth_router", "retrieve_router"] +__all__ = [ + "health_router", "upload_router", "documents_router", "chat_router", + "auth_router", "retrieve_router", "conversations_router", +] diff --git a/backend/app/api/chat.py b/backend/app/api/chat.py index fc66326..104550f 100644 --- a/backend/app/api/chat.py +++ b/backend/app/api/chat.py @@ -2,27 +2,36 @@ POST /chat — RAG chat endpoint. Pipeline: - 1. Run retrieval pipeline - 2. Pass RetrievalResult to ResponseGenerator - 3. Stream SSE tokens + citations to client + 1. Resolve (or create) the conversation and persist the user message + 2. Load recent history under its own token budget + 3. Condense an elliptical follow-up into a standalone retrieval query + 4. Run the retrieval pipeline + 5. Pass RetrievalResult + history to ResponseGenerator + 6. Stream SSE tokens + citations, then persist the assistant message -No prompt logic here — all business logic lives in app/llm/. +No prompt logic here — all business logic lives in app/llm/ and app/services/. """ +import asyncio import logging -from typing import Annotated +from dataclasses import asdict +from typing import Annotated, AsyncGenerator from fastapi import APIRouter, Body, Depends, HTTPException, Request from fastapi.responses import StreamingResponse from sqlalchemy.ext.asyncio import AsyncSession from app.api.dependencies import get_current_user, get_db -from app.database.models import User +from app.database.models import Conversation, User +from app.database.session import AsyncSessionLocal from app.schemas.chat import ChatRequest, ChatResponse, CitationOut from app.retrieval.retrieval_pipeline import RetrievalResult from app.retrieval import run_retrieval_pipeline from app.retrieval.exceptions import NoResultsError, EmbeddingError, VectorSearchError from app.llm import get_response_generator +from app.llm.condenser import condense_query +from app.llm.models import StreamCapture +from app.llm.stream import conversation_event from app.llm.exceptions import ( NoContextError, ProviderUnavailableError, @@ -32,11 +41,61 @@ ) from app.core.config import get_settings from app.core.middleware import limiter +from app.services import conversation_service from app.web_search import build_web_context, search_web logger = logging.getLogger(__name__) router = APIRouter() +_NO_RESULTS_ANSWER = "I could not find relevant information in the available documents." + +# asyncio holds only weak references to tasks, so a fire-and-forget write can be +# garbage-collected mid-flight. Anything still writing is kept here until it is done. +_pending_writes: set[asyncio.Task] = set() + + +async def _save_assistant_message( + conversation_id: str, + *, + content: str, + status: str, + mode: str, + model: str, + citations: list | None, +) -> None: + """ + Write the assistant's reply using a session of its own. + + A fresh session from the sessionmaker, deliberately: this runs from the + streaming generator's `finally`, which on a client disconnect executes while + the request is being torn down. The request-scoped session that `get_db` + yielded is closed by then, and reusing it raises rather than saving the + partial answer the user actually saw. + """ + async with AsyncSessionLocal() as session: + conversation = await session.get(Conversation, conversation_id) + if conversation is None: + # Deleted while the answer was still streaming. Nothing to attach to. + logger.info("[chat] conversation %s vanished mid-stream", conversation_id) + return + await conversation_service.append_message( + session, + conversation, + role="assistant", + content=content, + status=status, + mode=mode, + model=model, + citations=citations, + ) + + +def _persist_in_background(coro) -> asyncio.Task: + task = asyncio.create_task(coro) + _pending_writes.add(task) + task.add_done_callback(_pending_writes.discard) + return task + @router.post("/chat") @limiter.limit(lambda: get_settings().rate_limit_chat) @@ -47,24 +106,92 @@ async def chat( db: AsyncSession = Depends(get_db), ): """ - Streaming RAG chat. + Streaming RAG chat with conversation memory. - Returns text/event-stream when streaming is enabled, - or a JSON ChatResponse when streaming is disabled. + Returns text/event-stream when streaming is enabled, or a JSON ChatResponse + when it is disabled. """ settings = get_settings() + # --- 1. Conversation ------------------------------------------------------ + # Resolved before a single byte is streamed: once StreamingResponse has been + # returned the status code is already 200, and a "conversation not found" + # would have to be reported as an SSE error event the client cannot tell from + # a provider outage. + created = False + if chat_request.conversation_id: + conversation = await conversation_service.get_conversation( + db, current_user.id, chat_request.conversation_id + ) + if conversation is None: + # 404 rather than 403 for another user's id -- see app/api/conversations.py. + raise HTTPException(status_code=404, detail="Conversation not found.") + else: + conversation = await conversation_service.create_conversation( + db, current_user.id, mode=chat_request.mode + ) + created = True + + # --- 2. The message being answered --------------------------------------- + if chat_request.regenerate: + # Retry re-answers the question already on record and replaces the stale + # reply, so the thread does not grow a duplicate turn every time the user + # is unhappy with an answer. The text comes from the database, not the + # request, so a retry cannot quietly change the question. + last_user = await conversation_service.last_message(db, conversation.id, "user") + if last_user is None: + raise HTTPException(status_code=409, detail="This conversation has nothing to retry.") + message_text = last_user.content + user_message_id = last_user.id + + stale = await conversation_service.last_message(db, conversation.id, "assistant") + if stale is not None: + await conversation_service.delete_message(db, conversation.id, stale.id) + else: + message_text = chat_request.message + user_message = await conversation_service.append_message( + db, conversation, role="user", content=message_text, mode=chat_request.mode + ) + user_message_id = user_message.id + + # --- 3. History ----------------------------------------------------------- + history = await conversation_service.load_history( + db, + conversation.id, + turns=settings.chat_history_turns, + max_tokens=settings.chat_history_max_tokens, + exclude_message_id=user_message_id, + ) + + generator = get_response_generator() + + # --- 4. Query condensing -------------------------------------------------- + # LLM mode never retrieves, so there is nothing to condense for. Everything + # else rewrites the follow-up into a standalone query -- for RETRIEVAL ONLY. + retrieval_query = message_text + if history and chat_request.mode != "llm": + condensed = await condense_query(generator.provider, message_text, history) + retrieval_query = condensed.query + + if retrieval_query != message_text: + logger.info( + "[chat] conversation=%s retrieval_query=%r original=%r", + conversation.id, retrieval_query, message_text, + ) + + # --- 5. Retrieval --------------------------------------------------------- # LLM mode intentionally skips retrieval. DocuQuery and Hybrid use the # same grounded retrieval pipeline, while Hybrid's prompt also allows # general reasoning beyond the retrieved excerpts. + no_results = False if chat_request.mode == "llm": retrieval_result = RetrievalResult( - query=chat_request.message, chunks=[], context="", citations=[], total_retrieved=0 + query=message_text, chunks=[], context="", citations=[], total_retrieved=0 ) else: try: retrieval_result = await run_retrieval_pipeline( - query=chat_request.message, + query=retrieval_query, db=db, user_id=current_user.id, top_k=chat_request.top_k, @@ -74,55 +201,128 @@ async def chat( logger.error("[chat] Retrieval error: %s", exc) raise HTTPException(status_code=503, detail="Retrieval service unavailable.") except NoResultsError: - if chat_request.mode == "hybrid": - retrieval_result = RetrievalResult( - query=chat_request.message, chunks=[], context="", citations=[], total_retrieved=0 - ) - else: - # Return a graceful no-results response rather than a 404. - if settings.llm_streaming_enabled: - async def _no_results(): - yield "data: I could not find relevant information in the available documents.\n\n" - yield "event: citations\ndata: []\n\n" - yield "data: [DONE]\n\n" - return StreamingResponse(_no_results(), media_type="text/event-stream") - return ChatResponse( - answer="I could not find relevant information in the available documents.", - citations=[], - model=settings.llm_model, - conversation_id=chat_request.conversation_id, - ) + retrieval_result = RetrievalResult( + query=retrieval_query, chunks=[], context="", citations=[], total_retrieved=0 + ) + # Hybrid answers from general knowledge when nothing was retrieved; + # DocuQuery says so instead of guessing. + no_results = chat_request.mode != "hybrid" if chat_request.mode == "hybrid": - web_sources = await search_web(chat_request.message) + web_sources = await search_web(retrieval_query) web_context = build_web_context(web_sources) if web_context: retrieval_result.context = f"{retrieval_result.context}\n\n{web_context}".strip() retrieval_result.web_sources = web_sources - generator = get_response_generator() + # --- 6a. No-results short circuit ---------------------------------------- + if no_results: + await _save_assistant_message( + conversation.id, + content=_NO_RESULTS_ANSWER, + status="complete", + mode=chat_request.mode, + model=settings.llm_model, + citations=[], + ) + if settings.llm_streaming_enabled: + async def _no_results_stream() -> AsyncGenerator[str, None]: + if created: + yield conversation_event(conversation.id) + yield f"data: {_NO_RESULTS_ANSWER}\n\n" + yield "event: citations\ndata: []\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(_no_results_stream(), media_type="text/event-stream") + + return ChatResponse( + answer=_NO_RESULTS_ANSWER, + citations=[], + model=settings.llm_model, + conversation_id=conversation.id, + retrieval_query=retrieval_query, + ) - # --- Streaming path --- + # --- 6b. Streaming path --------------------------------------------------- if settings.llm_streaming_enabled: - async def _stream(): + conversation_id = conversation.id + + async def _stream() -> AsyncGenerator[str, None]: + capture = StreamCapture() + completed = False + failed = False try: - async for event in generator.stream(chat_request.message, retrieval_result, chat_request.mode): + if created: + yield conversation_event(conversation_id) + async for event in generator.stream( + message_text, retrieval_result, chat_request.mode, + history=history, capture=capture, + ): yield event + completed = True except RateLimitError: + failed = True yield "event: error\ndata: Rate limit reached. Please try again shortly.\n\n" except GenerationTimeoutError: + failed = True yield "event: error\ndata: The request timed out. Please try again.\n\n" except ProviderUnavailableError: + failed = True yield "event: error\ndata: AI service is temporarily unavailable.\n\n" except (NoContextError, MalformedResponseError) as exc: + failed = True logger.error("[chat] Generation error: %s", exc) yield "event: error\ndata: Failed to generate a response.\n\n" + finally: + # Reached on every exit: normal completion, a provider error, and a + # client disconnect -- which throws GeneratorExit in at the yield + # above, so neither `completed` nor `failed` gets set. + # + # The three cases are kept apart deliberately. Treating a + # disconnect as an error would file every closed tab under + # "generation failed", and an abandoned answer would come back + # marked as something that never worked. + text = capture.text + if completed: + status = "complete" + elif failed: + status = "error" + elif text: + status = "partial" + else: + # Gone before the first token. There is no answer to keep, and + # nothing failed, so no row is written at all. + status = None + + if status is not None: + task = _persist_in_background( + _save_assistant_message( + conversation_id, + content=text, + status=status, + mode=chat_request.mode, + model=settings.llm_model, + citations=[asdict(c) for c in capture.citations], + ) + ) + try: + await asyncio.shield(task) + except asyncio.CancelledError: + # The request is being cancelled around us. The write is a + # separate, shielded task, so it survives and finishes; + # _pending_writes holds the reference until it does. + logger.info( + "[chat] cancelled during save of %s message for %s", + status, conversation_id, + ) return StreamingResponse(_stream(), media_type="text/event-stream") - # --- Non-streaming path --- + # --- 6c. Non-streaming path ---------------------------------------------- try: - llm_response = await generator.generate(chat_request.message, retrieval_result, chat_request.mode) + llm_response = await generator.generate( + message_text, retrieval_result, chat_request.mode, history=history + ) except RateLimitError: raise HTTPException(status_code=429, detail="Rate limit reached. Please try again shortly.") except GenerationTimeoutError: @@ -134,20 +334,32 @@ async def _stream(): logger.error("[chat] Generation error: %s", exc) raise HTTPException(status_code=500, detail="Failed to generate a response.") + citations = [ + CitationOut( + document_id=c.document_id, + filename=c.filename, + page=c.page, + chunk_index=c.chunk_index, + source_type=c.source_type, + title=c.title, + url=c.url, + ) + for c in llm_response.citations + ] + + await _save_assistant_message( + conversation.id, + content=llm_response.answer, + status="complete", + mode=chat_request.mode, + model=llm_response.model, + citations=[asdict(c) for c in llm_response.citations], + ) + return ChatResponse( answer=llm_response.answer, - citations=[ - CitationOut( - document_id=c.document_id, - filename=c.filename, - page=c.page, - chunk_index=c.chunk_index, - source_type=c.source_type, - title=c.title, - url=c.url, - ) - for c in llm_response.citations - ], + citations=citations, model=llm_response.model, - conversation_id=chat_request.conversation_id, + conversation_id=conversation.id, + retrieval_query=retrieval_query, ) diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py new file mode 100644 index 0000000..b953872 --- /dev/null +++ b/backend/app/api/conversations.py @@ -0,0 +1,158 @@ +""" +Conversation CRUD. + +Thin by design: every handler validates, calls one service function, and turns a +`None` or `False` result into 404. + +On 404 vs 403 — a conversation that belongs to another user answers exactly like +one that does not exist. Returning 403 would confirm that the id is real and +someone else's, which is a membership oracle: an attacker could enumerate valid +conversation ids without ever reading one. The service layer makes this the +default by folding `user_id` into every WHERE clause, so a handler cannot forget. +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, HTTPException, Query, Response, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.dependencies import get_current_user, get_db +from app.database.models import User +from app.schemas.conversation import ( + ConversationCreate, + ConversationDetailOut, + ConversationImportRequest, + ConversationImportResult, + ConversationOut, + ConversationPage, + ConversationPatch, + MessageFeedbackPatch, + MessageOut, +) +from app.services import conversation_service + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/conversations", tags=["Conversations"]) + +_NOT_FOUND = "Conversation not found." + + +@router.get("", response_model=ConversationPage) +async def list_conversations( + limit: int = Query(default=30, ge=1, le=100), + offset: int = Query(default=0, ge=0), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """One page of the signed-in user's conversations, pinned first.""" + items, total = await conversation_service.list_conversations( + db, current_user.id, limit=limit, offset=offset + ) + return ConversationPage( + items=[ConversationOut.model_validate(c) for c in items], + total=total, + limit=limit, + offset=offset, + has_more=offset + len(items) < total, + ) + + +@router.post("/import", response_model=ConversationImportResult) +async def import_conversations( + payload: ConversationImportRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + One-time migration of chats that were saved in the browser's localStorage. + + Declared before `/{conversation_id}` so the literal path wins the match. The + 200-conversation cap lives in the request schema, which means an oversized + body is rejected by validation before any database work starts. + + Deliberately not idempotent: the client deletes its legacy key only after a + 2xx, and calling this twice with the same payload duplicates the threads. + """ + imported, skipped = await conversation_service.import_conversations( + db, current_user.id, payload.conversations + ) + return ConversationImportResult(imported=imported, skipped=skipped) + + +@router.post("", response_model=ConversationOut, status_code=status.HTTP_201_CREATED) +async def create_conversation( + payload: ConversationCreate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + conversation = await conversation_service.create_conversation( + db, current_user.id, title=payload.title, mode=payload.mode + ) + return ConversationOut.model_validate(conversation) + + +@router.get("/{conversation_id}", response_model=ConversationDetailOut) +async def get_conversation( + conversation_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """The conversation and every message in it, oldest first.""" + conversation = await conversation_service.get_conversation_with_messages( + db, current_user.id, conversation_id + ) + if conversation is None: + raise HTTPException(status_code=404, detail=_NOT_FOUND) + return ConversationDetailOut.model_validate(conversation) + + +@router.patch("/{conversation_id}", response_model=ConversationOut) +async def patch_conversation( + conversation_id: str, + payload: ConversationPatch, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Rename, pin/unpin, or switch mode. Unset fields are left alone.""" + conversation = await conversation_service.update_conversation( + db, current_user.id, conversation_id, payload + ) + if conversation is None: + raise HTTPException(status_code=404, detail=_NOT_FOUND) + return ConversationOut.model_validate(conversation) + + +@router.delete("/{conversation_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_conversation( + conversation_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Delete a conversation and its messages (ON DELETE CASCADE).""" + deleted = await conversation_service.delete_conversation( + db, current_user.id, conversation_id + ) + if not deleted: + raise HTTPException(status_code=404, detail=_NOT_FOUND) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.patch( + "/{conversation_id}/messages/{message_id}/feedback", response_model=MessageOut +) +async def patch_message_feedback( + conversation_id: str, + message_id: str, + payload: MessageFeedbackPatch, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Set or clear the thumb on one message. `{"feedback": null}` clears it.""" + message = await conversation_service.set_message_feedback( + db, current_user.id, conversation_id, message_id, payload.feedback + ) + if message is None: + raise HTTPException(status_code=404, detail="Message not found.") + return MessageOut.model_validate(message) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 17dfa86..7ce2071 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -61,6 +61,24 @@ def async_database_url(self) -> str: llm_api_key: str = "" ollama_base_url: str = "http://localhost:11434" + # Conversation memory (Phase 8) + # How many prior turns (one turn = a user message and the assistant reply to it) + # are replayed to the provider. Older turns are dropped first. + chat_history_turns: int = 6 + # History gets its own budget, deliberately separate from llm_max_context_tokens. + # Sharing one budget would let a long conversation crowd out retrieved passages, + # which is the opposite of what a document assistant should do under pressure. + chat_history_max_tokens: int = 1000 + # Query condensing: rewrite an elliptical follow-up into a standalone search query. + chat_query_rewrite: bool = True + chat_query_rewrite_max_tokens: int = 80 + # Kept well under llm_timeout -- condensing sits in front of retrieval, so its + # timeout is added to every follow-up's time-to-first-token. On expiry the raw + # message is used instead, which is a worse query but not a failed request. + chat_query_rewrite_timeout: float = 6.0 + # Upper bound on the one-time localStorage import. Mirrored in the request schema. + chat_import_max_conversations: int = 200 + # Authentication. Set a strong value in Railway/Vercel environments. auth_secret: str = DEV_AUTH_SECRET auth_cookie_name: str = "docuquery_session" diff --git a/backend/app/database/models.py b/backend/app/database/models.py index aa64406..ab15716 100644 --- a/backend/app/database/models.py +++ b/backend/app/database/models.py @@ -3,7 +3,10 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING -from sqlalchemy import String, Integer, DateTime, Text, Enum as SAEnum, ForeignKey, JSON +from sqlalchemy import ( + Boolean, String, Integer, DateTime, Text, Enum as SAEnum, ForeignKey, Index, JSON, text, +) +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column, relationship from pgvector.sqlalchemy import Vector @@ -23,6 +26,10 @@ class User(Base): DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False ) + conversations: Mapped[list["Conversation"]] = relationship( + "Conversation", back_populates="user", cascade="all, delete-orphan" + ) + class ProcessingStatus(str, enum.Enum): uploaded = "uploaded" @@ -31,6 +38,81 @@ class ProcessingStatus(str, enum.Enum): failed = "failed" +def _now() -> datetime: + return datetime.now(timezone.utc) + + +class Conversation(Base): + """A saved chat thread. Owned by exactly one user; deleting the user removes it.""" + + __tablename__ = "conversations" + __table_args__ = ( + # The sidebar query is "this user's threads, most recently touched first". + # A composite index with updated_at descending serves it without a sort. + Index("ix_conversations_user_id_updated_at", "user_id", text("updated_at DESC")), + ) + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + user_id: Mapped[str] = mapped_column( + String, ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + title: Mapped[str] = mapped_column(String(200), nullable=False, default="New Chat") + mode: Mapped[str] = mapped_column(String(16), nullable=False, default="docuquery") + pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_now + ) + # Bumped explicitly whenever a message is appended -- `onupdate` only fires when + # some column of this row is itself written, and appending a message does not. + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_now, onupdate=_now + ) + + user: Mapped["User"] = relationship("User", back_populates="conversations") + messages: Mapped[list["Message"]] = relationship( + "Message", + back_populates="conversation", + cascade="all, delete-orphan", + order_by="Message.created_at", + ) + + +class Message(Base): + """ + One turn in a conversation. + + `status` records how the row came to exist, which matters because an assistant + message is written by a background stream that the client can abandon: + + complete the stream finished and the whole answer is here + partial the client disconnected mid-stream; `content` is what was generated + error generation failed; `content` holds whatever preceded the failure + """ + + __tablename__ = "messages" + __table_args__ = ( + Index("ix_messages_conversation_id_created_at", "conversation_id", "created_at"), + ) + + id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + conversation_id: Mapped[str] = mapped_column( + String, ForeignKey("conversations.id", ondelete="CASCADE"), nullable=False + ) + role: Mapped[str] = mapped_column(String(16), nullable=False) + content: Mapped[str] = mapped_column(Text, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="complete") + mode: Mapped[str | None] = mapped_column(String(16), nullable=True) + model: Mapped[str | None] = mapped_column(String(120), nullable=True) + citations: Mapped[list | None] = mapped_column(JSONB, nullable=True) + steps: Mapped[list | None] = mapped_column(JSONB, nullable=True) + feedback: Mapped[str | None] = mapped_column(String(8), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=_now + ) + + conversation: Mapped["Conversation"] = relationship("Conversation", back_populates="messages") + + class Document(Base): __tablename__ = "documents" diff --git a/backend/app/llm/condenser.py b/backend/app/llm/condenser.py new file mode 100644 index 0000000..97c1394 --- /dev/null +++ b/backend/app/llm/condenser.py @@ -0,0 +1,144 @@ +""" +Query condensing: rewrite an elliptical follow-up into a standalone search query. + +"How does it differ from the policy engine?" embeds to almost nothing useful, +because the subject of the sentence is in the previous turn. Retrieval on that raw +text is the single largest source of bad follow-up answers. + +The rewrite feeds RETRIEVAL ONLY. The user's original message is what reaches the +answer prompt, so the assistant never responds to a question the user did not ask +and a bad rewrite degrades the passages rather than changing the question. + +Failure is never fatal here. Condensing sits in front of retrieval, so its latency +is added to every follow-up's time-to-first-token; on timeout or any provider +error the raw message is used instead. A worse query beats a failed request. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass + +from app.core.config import get_settings +from app.llm.exceptions import LLMError +from app.llm.models import HistoryMessage, LLMRequest +from app.llm.prompts import ( + DEFAULT_CONDENSE_PROMPT_VERSION, + build_condense_user_message, + get_condense_prompt, +) +from app.llm.providers.base import BaseLLMProvider + +logger = logging.getLogger(__name__) + +# A standalone rewrite of a chat message is a sentence, not a paragraph. Anything +# longer means the model ignored the instruction and started answering, so the +# result is discarded rather than embedded. +MAX_REWRITE_CHARS = 400 + + +@dataclass +class CondenseResult: + """What retrieval should run on, and how it was arrived at.""" + + query: str + rewritten: bool + latency_ms: float + # "ok" | "timeout" | "provider_error" | "rejected" | "skipped" + outcome: str + + +def _clean(raw: str) -> str: + """ + Reduce the model's reply to a single query string. + + Models sometimes prefix a rewrite with "Standalone query:" or wrap it in + quotes despite the instruction not to; the first non-empty line, unquoted, is + the rewrite in every one of those shapes. + """ + for line in raw.splitlines(): + candidate = line.strip() + if not candidate: + continue + for prefix in ("standalone search query:", "standalone query:", "query:", "rewrite:"): + if candidate.lower().startswith(prefix): + candidate = candidate[len(prefix):].strip() + candidate = candidate.strip("\"'").strip() + if candidate: + return candidate + return "" + + +async def condense_query( + provider: BaseLLMProvider, + message: str, + history: list[HistoryMessage], + *, + version: str = DEFAULT_CONDENSE_PROMPT_VERSION, +) -> CondenseResult: + """ + One LLM call that turns `message` into a standalone query given `history`. + + Returns the raw message unchanged whenever there is no history, condensing is + disabled, the call times out, the provider errors, or the reply does not look + like a query. The caller does not need to handle failure. + """ + settings = get_settings() + + if not history or not settings.chat_query_rewrite: + return CondenseResult(query=message, rewritten=False, latency_ms=0.0, outcome="skipped") + + request = LLMRequest( + system_prompt=get_condense_prompt(version), + user_message=build_condense_user_message(history, message), + context="", + model=settings.llm_model, + # Deterministic: the same follow-up should condense the same way twice, or + # an evaluation of this step measures sampling noise. + temperature=0.0, + max_tokens=settings.chat_query_rewrite_max_tokens, + ) + + t0 = time.perf_counter() + try: + response = await asyncio.wait_for( + provider.generate(request), timeout=settings.chat_query_rewrite_timeout + ) + except (asyncio.TimeoutError, TimeoutError): + elapsed = (time.perf_counter() - t0) * 1000 + logger.warning( + "[condense] timeout after %.0fms — retrieving on the raw message | raw=%r", + elapsed, message, + ) + return CondenseResult(query=message, rewritten=False, latency_ms=elapsed, outcome="timeout") + except LLMError as exc: + elapsed = (time.perf_counter() - t0) * 1000 + logger.warning( + "[condense] provider error (%s) — retrieving on the raw message | raw=%r", + exc.__class__.__name__, message, + ) + return CondenseResult( + query=message, rewritten=False, latency_ms=elapsed, outcome="provider_error" + ) + + elapsed = (time.perf_counter() - t0) * 1000 + rewrite = _clean(response.answer or "") + + if not rewrite or len(rewrite) > MAX_REWRITE_CHARS: + logger.warning( + "[condense] unusable rewrite (%d chars) — retrieving on the raw message | raw=%r", + len(rewrite), message, + ) + return CondenseResult( + query=message, rewritten=False, latency_ms=elapsed, outcome="rejected" + ) + + # Both sides are logged deliberately: a follow-up that retrieves badly is + # almost always diagnosed by comparing what the user typed with what was + # actually searched for. + logger.info( + "[condense] %.0fms | raw=%r -> rewritten=%r", elapsed, message, rewrite + ) + return CondenseResult(query=rewrite, rewritten=True, latency_ms=elapsed, outcome="ok") diff --git a/backend/app/llm/models.py b/backend/app/llm/models.py index e671bca..92e59b6 100644 --- a/backend/app/llm/models.py +++ b/backend/app/llm/models.py @@ -4,6 +4,23 @@ from dataclasses import dataclass, field +# Rough chars-per-token estimate used for every token budget in this layer. +# Accurate enough for English prose; avoids a full tokeniser dependency. Shared +# so the history budget and the context budget measure in the same unit. +CHARS_PER_TOKEN = 4 + + +def estimate_tokens(text: str) -> int: + return len(text) // CHARS_PER_TOKEN + + +@dataclass +class HistoryMessage: + """One earlier turn replayed to the provider. `role` is "user" or "assistant".""" + + role: str + content: str + @dataclass class LLMRequest: @@ -14,6 +31,9 @@ class LLMRequest: model: str temperature: float max_tokens: int + # Prior turns, oldest first, already trimmed to the history budget by the + # caller. Empty for the first message of a conversation. + history: list[HistoryMessage] = field(default_factory=list) @dataclass @@ -35,3 +55,22 @@ class LLMResponse: model: str prompt_tokens: int | None = None completion_tokens: int | None = None + + +@dataclass +class StreamCapture: + """ + Collects what a stream produced, so the caller can persist it. + + The streaming path yields SSE frames, not raw text. Without this the route + would have to parse its own wire format back into an answer in order to save + it -- and it has to save it even when the client hangs up mid-answer, which is + exactly when `tokens` holds a partial reply. + """ + + tokens: list[str] = field(default_factory=list) + citations: list[CitationRecord] = field(default_factory=list) + + @property + def text(self) -> str: + return "".join(self.tokens) diff --git a/backend/app/llm/prompts.py b/backend/app/llm/prompts.py index ec41681..0e9e1a5 100644 --- a/backend/app/llm/prompts.py +++ b/backend/app/llm/prompts.py @@ -47,3 +47,58 @@ def get_system_prompt(version: str = DEFAULT_PROMPT_VERSION, mode: str = "docuqu if mode == "hybrid": return _HYBRID_PROMPT return PROMPT_VERSIONS[version] + + +# --- Query condensing ------------------------------------------------------- +# +# A follow-up like "what about the third one?" is meaningless to a vector search: +# it embeds to nothing useful because the subject lives in the previous turn. This +# prompt rewrites such a message into one standalone question, which is then used +# for RETRIEVAL ONLY -- the user's original wording still goes to the answer +# prompt, so the assistant never replies to a question the user did not ask. +# +# Versioned like the system prompts so a rewrite strategy can be changed and +# measured without silently altering what a past evaluation run meant. + +_CONDENSE_PROMPT_V1 = """You rewrite a follow-up question into a single standalone search query. + +Rules: +1. Resolve every pronoun and reference ("it", "them", "the third one", "that + framework") using the conversation above, so the result stands alone. +2. Preserve the user's intent exactly. Do not answer the question, do not add + information, and do not narrow or broaden what was asked. +3. If the question already stands alone, return it unchanged. +4. Output the rewritten query and nothing else -- no preamble, no quotes, no + explanation. +""" + +CONDENSE_PROMPT_VERSIONS: dict[str, str] = { + "v1": _CONDENSE_PROMPT_V1, +} + +DEFAULT_CONDENSE_PROMPT_VERSION = "v1" + + +def get_condense_prompt(version: str = DEFAULT_CONDENSE_PROMPT_VERSION) -> str: + """Return the query-condensing system prompt for the given version.""" + if version not in CONDENSE_PROMPT_VERSIONS: + raise ValueError(f"Unknown condense prompt version: {version!r}") + return CONDENSE_PROMPT_VERSIONS[version] + + +def build_condense_user_message(history: list, follow_up: str) -> str: + """ + Render the prior turns plus the follow-up into the condenser's user message. + + `history` is a list of app.llm.models.HistoryMessage, oldest first. + """ + lines = [ + f"{'User' if turn.role == 'user' else 'Assistant'}: {turn.content}" + for turn in history + ] + transcript = "\n".join(lines) + return ( + f"Conversation so far:\n{transcript}\n\n" + f"Follow-up question: {follow_up}\n\n" + "Standalone search query:" + ) diff --git a/backend/app/llm/providers/anthropic_provider.py b/backend/app/llm/providers/anthropic_provider.py index 9c9f6b7..bc9b09d 100644 --- a/backend/app/llm/providers/anthropic_provider.py +++ b/backend/app/llm/providers/anthropic_provider.py @@ -9,7 +9,7 @@ import anthropic from anthropic import AsyncAnthropic, APIConnectionError, APIStatusError, RateLimitError as AnthropicRateLimitError -from app.llm.providers.base import BaseLLMProvider +from app.llm.providers.base import BaseLLMProvider, build_anthropic_messages from app.llm.models import LLMRequest, LLMResponse from app.llm.exceptions import ( ProviderUnavailableError, @@ -32,7 +32,7 @@ async def generate(self, request: LLMRequest) -> LLMResponse: response = await self._client.messages.create( model=request.model, system=request.system_prompt, - messages=[{"role": "user", "content": f"{request.context}\n\nQuestion: {request.user_message}"}], + messages=build_anthropic_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, ) @@ -69,7 +69,7 @@ async def stream(self, request: LLMRequest) -> AsyncGenerator[str, None]: async with self._client.messages.stream( model=request.model, system=request.system_prompt, - messages=[{"role": "user", "content": f"{request.context}\n\nQuestion: {request.user_message}"}], + messages=build_anthropic_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, ) as stream: diff --git a/backend/app/llm/providers/base.py b/backend/app/llm/providers/base.py index 3b92459..9043ae8 100644 --- a/backend/app/llm/providers/base.py +++ b/backend/app/llm/providers/base.py @@ -8,6 +8,43 @@ from app.llm.models import LLMRequest, LLMResponse +def build_user_turn(request: LLMRequest) -> str: + """ + The live user turn: retrieved passages followed by the question. + + With no context — LLM mode, and the query condenser — the message is sent on + its own. The older unconditional f-string produced a turn beginning + "\\n\\nQuestion: " in exactly those cases, which is noise the model has to + read past. + """ + if not request.context: + return request.user_message + return f"{request.context}\n\nQuestion: {request.user_message}" + + +def build_chat_messages(request: LLMRequest) -> list[dict[str, str]]: + """ + system prompt, prior turns, then the live user turn — OpenAI/Groq wire format. + + `request.history` is already trimmed and strictly alternating; see + app.services.conversation_service.load_history. + """ + messages: list[dict[str, str]] = [{"role": "system", "content": request.system_prompt}] + messages.extend({"role": turn.role, "content": turn.content} for turn in request.history) + messages.append({"role": "user", "content": build_user_turn(request)}) + return messages + + +def build_anthropic_messages(request: LLMRequest) -> list[dict[str, str]]: + """ + Prior turns then the live user turn. No system entry: Anthropic takes the + system prompt as a separate top-level parameter rather than a message. + """ + messages = [{"role": turn.role, "content": turn.content} for turn in request.history] + messages.append({"role": "user", "content": build_user_turn(request)}) + return messages + + class BaseLLMProvider(ABC): """ Contract every provider must satisfy. diff --git a/backend/app/llm/providers/gemini_provider.py b/backend/app/llm/providers/gemini_provider.py index d21ba55..4de306a 100644 --- a/backend/app/llm/providers/gemini_provider.py +++ b/backend/app/llm/providers/gemini_provider.py @@ -9,7 +9,7 @@ import google.generativeai as genai from google.api_core.exceptions import GoogleAPIError, ResourceExhausted, DeadlineExceeded -from app.llm.providers.base import BaseLLMProvider +from app.llm.providers.base import BaseLLMProvider, build_user_turn from app.llm.models import LLMRequest, LLMResponse from app.llm.exceptions import ( ProviderUnavailableError, @@ -26,23 +26,39 @@ class GeminiProvider(BaseLLMProvider): def __init__(self, api_key: str) -> None: genai.configure(api_key=api_key) - def _get_model(self, model_name: str, temperature: float, max_tokens: int): + def _get_model(self, request: LLMRequest): + # system_instruction is a first-class parameter in google-generativeai + # 0.8.x. Passing it here rather than gluing it onto the prompt text is what + # lets `contents` below be a real multi-turn list instead of one blob. return genai.GenerativeModel( - model_name=model_name, + model_name=request.model, + system_instruction=request.system_prompt, generation_config=genai.GenerationConfig( - temperature=temperature, - max_output_tokens=max_tokens, + temperature=request.temperature, + max_output_tokens=request.max_tokens, ), ) - def _build_prompt(self, request: LLMRequest) -> str: - return f"{request.system_prompt}\n\n{request.context}\n\nQuestion: {request.user_message}" + @staticmethod + def _build_contents(request: LLMRequest) -> list[dict]: + """ + Prior turns plus the live user turn, in Gemini's content format. + + Gemini names the assistant role "model", so the history roles are mapped + rather than passed through. + """ + contents = [ + {"role": "model" if turn.role == "assistant" else "user", "parts": [turn.content]} + for turn in request.history + ] + contents.append({"role": "user", "parts": [build_user_turn(request)]}) + return contents async def generate(self, request: LLMRequest) -> LLMResponse: t0 = time.monotonic() - model = self._get_model(request.model, request.temperature, request.max_tokens) + model = self._get_model(request) try: - response = await model.generate_content_async(self._build_prompt(request)) + response = await model.generate_content_async(self._build_contents(request)) except ResourceExhausted as exc: raise RateLimitError(str(exc)) from exc except DeadlineExceeded as exc: @@ -58,10 +74,10 @@ async def generate(self, request: LLMRequest) -> LLMResponse: async def stream(self, request: LLMRequest) -> AsyncGenerator[str, None]: t0 = time.monotonic() - model = self._get_model(request.model, request.temperature, request.max_tokens) + model = self._get_model(request) try: async for chunk in await model.generate_content_async( - self._build_prompt(request), stream=True + self._build_contents(request), stream=True ): if chunk.text: yield chunk.text diff --git a/backend/app/llm/providers/groq_provider.py b/backend/app/llm/providers/groq_provider.py index 27b9b6b..4683896 100644 --- a/backend/app/llm/providers/groq_provider.py +++ b/backend/app/llm/providers/groq_provider.py @@ -8,7 +8,7 @@ from groq import AsyncGroq, APIConnectionError, APIStatusError, RateLimitError as GroqRateLimitError -from app.llm.providers.base import BaseLLMProvider +from app.llm.providers.base import BaseLLMProvider, build_chat_messages from app.llm.models import LLMRequest, LLMResponse from app.llm.exceptions import ( ProviderUnavailableError, @@ -30,7 +30,7 @@ async def generate(self, request: LLMRequest) -> LLMResponse: try: response = await self._client.chat.completions.create( model=request.model, - messages=self._build_messages(request), + messages=build_chat_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, stream=False, @@ -69,7 +69,7 @@ async def stream(self, request: LLMRequest) -> AsyncGenerator[str, None]: try: stream = await self._client.chat.completions.create( model=request.model, - messages=self._build_messages(request), + messages=build_chat_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, stream=True, @@ -95,10 +95,3 @@ async def health_check(self) -> bool: return len(models.data) > 0 except Exception: return False - - @staticmethod - def _build_messages(request: LLMRequest) -> list[dict]: - return [ - {"role": "system", "content": request.system_prompt}, - {"role": "user", "content": f"{request.context}\n\nQuestion: {request.user_message}"}, - ] diff --git a/backend/app/llm/providers/ollama_provider.py b/backend/app/llm/providers/ollama_provider.py index e3e18cb..b7f7d36 100644 --- a/backend/app/llm/providers/ollama_provider.py +++ b/backend/app/llm/providers/ollama_provider.py @@ -1,4 +1,12 @@ -"""Ollama provider — local inference, no API key required.""" +""" +Ollama provider — local inference, no API key required. + +Talks to `/api/chat`, not `/api/generate`. `/api/generate` accepts a single prompt +string and has nowhere to put prior turns, so conversation history had to be +flattened into the prompt and the model could not tell its own past replies from +the user's questions. `/api/chat` takes the same `messages` list shape the hosted +providers use. +""" from __future__ import annotations @@ -8,7 +16,7 @@ import httpx -from app.llm.providers.base import BaseLLMProvider +from app.llm.providers.base import BaseLLMProvider, build_chat_messages from app.llm.models import LLMRequest, LLMResponse from app.llm.exceptions import ( ProviderUnavailableError, @@ -25,20 +33,21 @@ def __init__(self, base_url: str = "http://localhost:11434", timeout: float = 60 self._base_url = base_url.rstrip("/") self._timeout = timeout - def _build_prompt(self, request: LLMRequest) -> str: - return f"{request.system_prompt}\n\n{request.context}\n\nQuestion: {request.user_message}" - - async def generate(self, request: LLMRequest) -> LLMResponse: - t0 = time.monotonic() - payload = { + def _payload(self, request: LLMRequest, *, stream: bool) -> dict: + return { "model": request.model, - "prompt": self._build_prompt(request), + "messages": build_chat_messages(request), "options": {"temperature": request.temperature, "num_predict": request.max_tokens}, - "stream": False, + "stream": stream, } + + async def generate(self, request: LLMRequest) -> LLMResponse: + t0 = time.monotonic() try: async with httpx.AsyncClient(timeout=self._timeout) as client: - resp = await client.post(f"{self._base_url}/api/generate", json=payload) + resp = await client.post( + f"{self._base_url}/api/chat", json=self._payload(request, stream=False) + ) resp.raise_for_status() except httpx.TimeoutException as exc: raise GenerationTimeoutError(str(exc)) from exc @@ -46,31 +55,33 @@ async def generate(self, request: LLMRequest) -> LLMResponse: raise ProviderUnavailableError(str(exc)) from exc data = resp.json() - answer = data.get("response", "") + answer = (data.get("message") or {}).get("content", "") if not answer: raise MalformedResponseError("Ollama returned empty response.") logger.info("[ollama] generate latency=%.2fs model=%s", time.monotonic() - t0, request.model) - return LLMResponse(answer=answer, citations=[], model=request.model) + return LLMResponse( + answer=answer, + citations=[], + model=request.model, + prompt_tokens=data.get("prompt_eval_count"), + completion_tokens=data.get("eval_count"), + ) async def stream(self, request: LLMRequest) -> AsyncGenerator[str, None]: import json t0 = time.monotonic() - payload = { - "model": request.model, - "prompt": self._build_prompt(request), - "options": {"temperature": request.temperature, "num_predict": request.max_tokens}, - "stream": True, - } try: async with httpx.AsyncClient(timeout=self._timeout) as client: - async with client.stream("POST", f"{self._base_url}/api/generate", json=payload) as resp: + async with client.stream( + "POST", f"{self._base_url}/api/chat", json=self._payload(request, stream=True) + ) as resp: resp.raise_for_status() async for line in resp.aiter_lines(): if not line: continue data = json.loads(line) - token = data.get("response", "") + token = (data.get("message") or {}).get("content", "") if token: yield token if data.get("done"): diff --git a/backend/app/llm/providers/openai_provider.py b/backend/app/llm/providers/openai_provider.py index deca461..e0f5747 100644 --- a/backend/app/llm/providers/openai_provider.py +++ b/backend/app/llm/providers/openai_provider.py @@ -8,7 +8,7 @@ from openai import AsyncOpenAI, APIConnectionError, APIStatusError, RateLimitError as OpenAIRateLimitError -from app.llm.providers.base import BaseLLMProvider +from app.llm.providers.base import BaseLLMProvider, build_chat_messages from app.llm.models import LLMRequest, LLMResponse from app.llm.exceptions import ( ProviderUnavailableError, @@ -30,7 +30,7 @@ async def generate(self, request: LLMRequest) -> LLMResponse: try: response = await self._client.chat.completions.create( model=request.model, - messages=self._build_messages(request), + messages=build_chat_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, stream=False, @@ -68,7 +68,7 @@ async def stream(self, request: LLMRequest) -> AsyncGenerator[str, None]: try: stream = await self._client.chat.completions.create( model=request.model, - messages=self._build_messages(request), + messages=build_chat_messages(request), temperature=request.temperature, max_tokens=request.max_tokens, stream=True, @@ -92,10 +92,3 @@ async def health_check(self) -> bool: return True except Exception: return False - - @staticmethod - def _build_messages(request: LLMRequest) -> list[dict]: - return [ - {"role": "system", "content": request.system_prompt}, - {"role": "user", "content": f"{request.context}\n\nQuestion: {request.user_message}"}, - ] diff --git a/backend/app/llm/response_generator.py b/backend/app/llm/response_generator.py index ded121a..361fee8 100644 --- a/backend/app/llm/response_generator.py +++ b/backend/app/llm/response_generator.py @@ -12,7 +12,14 @@ from typing import AsyncGenerator from app.core.config import get_settings -from app.llm.models import LLMRequest, LLMResponse, CitationRecord +from app.llm.models import ( + CHARS_PER_TOKEN, + CitationRecord, + HistoryMessage, + LLMRequest, + LLMResponse, + StreamCapture, +) from app.llm.prompts import get_system_prompt from app.llm.providers.base import BaseLLMProvider from app.llm.stream import stream_with_citations @@ -21,14 +28,9 @@ logger = logging.getLogger(__name__) -# Rough chars-per-token estimate used for context truncation. -# Accurate enough for English prose; avoids a full tokeniser dependency. -_CHARS_PER_TOKEN = 4 - - def _truncate_context(context: str, max_tokens: int) -> str: """Hard-truncate context to stay within the configured token budget.""" - max_chars = max_tokens * _CHARS_PER_TOKEN + max_chars = max_tokens * CHARS_PER_TOKEN if len(context) <= max_chars: return context truncated = context[:max_chars] @@ -68,11 +70,14 @@ def _build_request( model: str, temperature: float, max_tokens: int, + history: list[HistoryMessage] | None = None, ) -> LLMRequest: - prompt_chars = len(system_prompt) + len(context) + len(message) + history = history or [] + history_chars = sum(len(h.content) for h in history) + prompt_chars = len(system_prompt) + len(context) + len(message) + history_chars logger.info( - "[generator] prompt_chars=%d context_chars=%d", - prompt_chars, len(context), + "[generator] prompt_chars=%d context_chars=%d history_turns=%d history_chars=%d", + prompt_chars, len(context), len(history), history_chars, ) return LLMRequest( system_prompt=system_prompt, @@ -81,6 +86,7 @@ def _build_request( model=model, temperature=temperature, max_tokens=max_tokens, + history=history, ) @@ -95,7 +101,19 @@ class ResponseGenerator: def __init__(self, provider: BaseLLMProvider) -> None: self._provider = provider - async def generate(self, message: str, retrieval_result: RetrievalResult, mode: str = "docuquery") -> LLMResponse: + @property + def provider(self) -> BaseLLMProvider: + """The configured provider. Used by the query condenser, which needs a + plain one-shot completion rather than the retrieval-shaped pipeline.""" + return self._provider + + async def generate( + self, + message: str, + retrieval_result: RetrievalResult, + mode: str = "docuquery", + history: list[HistoryMessage] | None = None, + ) -> LLMResponse: """Non-streaming path — returns a complete LLMResponse with citations attached.""" settings = get_settings() context, citations = self._prepare(retrieval_result, settings.llm_max_context_tokens, allow_empty=mode in ("llm", "hybrid")) @@ -107,6 +125,7 @@ async def generate(self, message: str, retrieval_result: RetrievalResult, mode: model=settings.llm_model, temperature=settings.llm_temperature, max_tokens=settings.llm_max_tokens, + history=history, ) t0 = time.monotonic() @@ -120,7 +139,12 @@ async def generate(self, message: str, retrieval_result: RetrievalResult, mode: return response async def stream( - self, message: str, retrieval_result: RetrievalResult, mode: str = "docuquery" + self, + message: str, + retrieval_result: RetrievalResult, + mode: str = "docuquery", + history: list[HistoryMessage] | None = None, + capture: StreamCapture | None = None, ) -> AsyncGenerator[str, None]: """Streaming path — yields SSE-framed tokens then a citations event.""" settings = get_settings() @@ -133,6 +157,7 @@ async def stream( model=settings.llm_model, temperature=settings.llm_temperature, max_tokens=settings.llm_max_tokens, + history=history, ) t0 = time.monotonic() @@ -141,8 +166,13 @@ async def stream( settings.llm_provider, len(retrieval_result.chunks), ) + # Recorded before the first token, so a client that disconnects one token + # in still leaves behind a partial message with its citations attached. + if capture is not None: + capture.citations = citations + token_gen = self._provider.stream(request) - async for event in stream_with_citations(token_gen, citations): + async for event in stream_with_citations(token_gen, citations, capture=capture): yield event logger.info("[generator] stream end latency=%.2fs", time.monotonic() - t0) diff --git a/backend/app/llm/stream.py b/backend/app/llm/stream.py index b744571..9959d35 100644 --- a/backend/app/llm/stream.py +++ b/backend/app/llm/stream.py @@ -14,7 +14,19 @@ from dataclasses import asdict from typing import AsyncGenerator -from app.llm.models import CitationRecord +from app.llm.models import CitationRecord, StreamCapture + + +def conversation_event(conversation_id: str) -> str: + """ + Announces the conversation this stream belongs to. + + Emitted first, and only when /chat created the conversation itself, so a + client that opened a brand-new chat learns its server id before any token + arrives -- early enough to put the id in the URL without waiting for, or + interrupting, the answer. + """ + return f"event: conversation\ndata: {json.dumps({'id': conversation_id})}\n\n" def token_event(token: str) -> str: @@ -38,13 +50,19 @@ def error_event(message: str) -> str: async def stream_with_citations( token_stream: AsyncGenerator[str, None], citations: list[CitationRecord], + capture: StreamCapture | None = None, ) -> AsyncGenerator[str, None]: """ Wrap a raw token stream with SSE framing. - Yields token events, then a single citations event, then [DONE]. + Yields token events, then a single citations event, then [DONE]. When + `capture` is supplied each token is also recorded there, so the caller can + persist the answer -- including a partial one, if the client disconnects + before [DONE]. """ async for token in token_stream: + if capture is not None: + capture.tokens.append(token) yield token_event(token) yield citations_event(citations) yield done_event() diff --git a/backend/app/main.py b/backend/app/main.py index 20108bb..351a7d5 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -13,6 +13,7 @@ chat_router, auth_router, retrieve_router, + conversations_router, ) settings = get_settings() @@ -68,6 +69,7 @@ def create_app() -> FastAPI: app.include_router(chat_router, tags=["Chat"]) app.include_router(auth_router, tags=["Auth"]) app.include_router(retrieve_router, tags=["Retrieval"]) + app.include_router(conversations_router) return app diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 2e868a4..a2c2a46 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -1,9 +1,23 @@ from app.schemas.document import DocumentOut, DocumentChunkOut from app.schemas.upload import UploadResponse from app.schemas.chat import ChatRequest, ChatResponse, CitationOut +from app.schemas.conversation import ( + ConversationCreate, + ConversationDetailOut, + ConversationImportRequest, + ConversationImportResult, + ConversationOut, + ConversationPage, + ConversationPatch, + MessageFeedbackPatch, + MessageOut, +) __all__ = [ "DocumentOut", "DocumentChunkOut", "UploadResponse", "ChatRequest", "ChatResponse", "CitationOut", + "ConversationCreate", "ConversationDetailOut", "ConversationImportRequest", + "ConversationImportResult", "ConversationOut", "ConversationPage", + "ConversationPatch", "MessageFeedbackPatch", "MessageOut", ] diff --git a/backend/app/schemas/chat.py b/backend/app/schemas/chat.py index d1371cd..bab741c 100644 --- a/backend/app/schemas/chat.py +++ b/backend/app/schemas/chat.py @@ -1,17 +1,33 @@ -"""Chat request/response schemas for Phase 5.""" +"""Chat request/response schemas.""" from __future__ import annotations from typing import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator class ChatRequest(BaseModel): - message: str = Field(..., min_length=1, max_length=4000) + # Empty is allowed only when `regenerate` is set; see the validator below. + message: str = Field(default="", max_length=4000) top_k: int | None = Field(default=None, ge=1, le=20) - # conversation_id reserved for Phase 8 (session memory) + # Absent means "start a new conversation". An id that does not exist, or + # belongs to another user, is a 404 raised before anything is streamed. conversation_id: str | None = None mode: Literal["docuquery", "llm", "hybrid"] = "docuquery" + # Re-answer the last user message in place of the existing assistant reply, + # rather than appending another turn. Requires `conversation_id`; the text + # that gets re-answered is read from the server, not from this request. + regenerate: bool = False + + @model_validator(mode="after") + def _check_message(self) -> "ChatRequest": + if self.regenerate: + if not self.conversation_id: + raise ValueError("regenerate requires conversation_id") + return self + if not self.message.strip(): + raise ValueError("message must not be empty") + return self class CitationOut(BaseModel): @@ -29,3 +45,6 @@ class ChatResponse(BaseModel): citations: list[CitationOut] model: str conversation_id: str | None = None + # The standalone query retrieval actually ran on. Equal to the user's message + # unless the condenser rewrote a follow-up. + retrieval_query: str | None = None diff --git a/backend/app/schemas/conversation.py b/backend/app/schemas/conversation.py new file mode 100644 index 0000000..de6ca8e --- /dev/null +++ b/backend/app/schemas/conversation.py @@ -0,0 +1,118 @@ +"""Conversation and message schemas for Phase 8 (server-side chat history).""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from app.schemas.chat import CitationOut + +ChatMode = Literal["docuquery", "llm", "hybrid"] +MessageRole = Literal["user", "assistant"] +MessageStatus = Literal["complete", "partial", "error"] +Feedback = Literal["up", "down"] + + +# ─── Output ─────────────────────────────────────────────────────────────────── + +class MessageOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: str + conversation_id: str + role: MessageRole + content: str + status: MessageStatus + mode: ChatMode | None = None + model: str | None = None + citations: list[CitationOut] | None = None + steps: list[dict] | None = None + feedback: Feedback | None = None + created_at: datetime + + +class ConversationOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: str + title: str + mode: ChatMode + pinned: bool + created_at: datetime + updated_at: datetime + + +class ConversationDetailOut(ConversationOut): + messages: list[MessageOut] + + +class ConversationPage(BaseModel): + """One page of the sidebar list. `total` is the unpaginated count.""" + + items: list[ConversationOut] + total: int + limit: int + offset: int + has_more: bool + + +# ─── Input ──────────────────────────────────────────────────────────────────── + +class ConversationCreate(BaseModel): + title: str | None = Field(default=None, max_length=200) + mode: ChatMode = "docuquery" + + +class ConversationPatch(BaseModel): + """ + Partial update. Every field is optional and `None` means "leave alone" rather + than "set to null", so a rename cannot accidentally unpin a thread. + """ + + title: str | None = Field(default=None, min_length=1, max_length=200) + pinned: bool | None = None + mode: ChatMode | None = None + + +class MessageFeedbackPatch(BaseModel): + """`feedback: null` clears an existing thumb rather than leaving it unchanged.""" + + feedback: Feedback | None = None + + +# ─── Import (one-time localStorage migration) ───────────────────────────────── + +class ImportMessage(BaseModel): + role: MessageRole + content: str = Field(max_length=100_000) + status: MessageStatus = "complete" + citations: list[CitationOut] | None = None + feedback: Feedback | None = None + created_at: datetime | None = None + + +class ImportConversation(BaseModel): + title: str = Field(default="Imported Chat", max_length=200) + mode: ChatMode = "docuquery" + pinned: bool = False + created_at: datetime | None = None + updated_at: datetime | None = None + messages: list[ImportMessage] = Field(default_factory=list, max_length=500) + + +class ConversationImportRequest(BaseModel): + """ + The payload the browser sends once, from its legacy localStorage blob. + + The cap is enforced here rather than in the route so an oversized body is + rejected by request validation -- a 422 before any database work happens. + """ + + conversations: list[ImportConversation] = Field(max_length=200) + + +class ConversationImportResult(BaseModel): + imported: int + skipped: int diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py index 878cf8a..673742c 100644 --- a/backend/app/services/__init__.py +++ b/backend/app/services/__init__.py @@ -5,4 +5,11 @@ update_document_status, ) +# conversation_service is deliberately NOT re-exported here. It imports +# app.llm.models, which executes app/llm/__init__.py, which imports the retrieval +# pipeline -- and app.retrieval reaches app.services through app.ingestion. Doing +# it eagerly closes that loop, so importing app.retrieval first (as the evaluation +# harness does) fails with a partially initialised module. Callers import the +# submodule directly: `from app.services import conversation_service`. + __all__ = ["save_document", "list_documents", "delete_document", "update_document_status"] diff --git a/backend/app/services/conversation_service.py b/backend/app/services/conversation_service.py new file mode 100644 index 0000000..4336943 --- /dev/null +++ b/backend/app/services/conversation_service.py @@ -0,0 +1,361 @@ +""" +Persistence for conversations and their messages. + +Every read and write here is scoped to a user id. The routes never filter on +ownership themselves -- they pass `user_id` in and translate a `None` result into +404. That keeps the ownership rule in one place, and makes "not yours" and "does +not exist" indistinguishable from outside, which is the intended behaviour: a 403 +would confirm that somebody else's conversation id is real. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.database.models import Conversation, Message +from app.llm.models import HistoryMessage, estimate_tokens +from app.schemas.conversation import ConversationPatch, ImportConversation + +logger = logging.getLogger(__name__) + +TITLE_MAX_CHARS = 60 +DEFAULT_TITLE = "New Chat" + +# Assistant rows in these states are replayed as history. `error` rows are not: +# their content is whatever happened to arrive before the failure, and feeding a +# truncated answer back as if it were a real reply teaches the model to produce +# more of the same. +_HISTORY_STATUSES = ("complete", "partial") + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def title_from_message(content: str) -> str: + """The conversation title is just the first user message, trimmed. No LLM call.""" + flat = " ".join(content.split()) + if not flat: + return DEFAULT_TITLE + if len(flat) <= TITLE_MAX_CHARS: + return flat + return flat[: TITLE_MAX_CHARS - 1].rstrip() + "…" + + +# --- Conversations ---------------------------------------------------------- + +async def list_conversations( + db: AsyncSession, user_id: str, *, limit: int = 30, offset: int = 0 +) -> tuple[list[Conversation], int]: + """One page of a user's conversations, pinned first then most recently updated.""" + total = await db.scalar( + select(func.count()).select_from(Conversation).where(Conversation.user_id == user_id) + ) + rows = await db.scalars( + select(Conversation) + .where(Conversation.user_id == user_id) + .order_by(Conversation.pinned.desc(), Conversation.updated_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(rows), int(total or 0) + + +async def create_conversation( + db: AsyncSession, user_id: str, *, title: str | None = None, mode: str = "docuquery" +) -> Conversation: + conversation = Conversation( + user_id=user_id, title=title or DEFAULT_TITLE, mode=mode, pinned=False + ) + db.add(conversation) + await db.commit() + await db.refresh(conversation) + return conversation + + +async def get_conversation( + db: AsyncSession, user_id: str, conversation_id: str +) -> Conversation | None: + """The conversation, or None when it does not exist *or* belongs to someone else.""" + return await db.scalar( + select(Conversation).where( + Conversation.id == conversation_id, Conversation.user_id == user_id + ) + ) + + +async def get_conversation_with_messages( + db: AsyncSession, user_id: str, conversation_id: str +) -> Conversation | None: + return await db.scalar( + select(Conversation) + .options(selectinload(Conversation.messages)) + .where(Conversation.id == conversation_id, Conversation.user_id == user_id) + ) + + +async def update_conversation( + db: AsyncSession, user_id: str, conversation_id: str, patch: ConversationPatch +) -> Conversation | None: + conversation = await get_conversation(db, user_id, conversation_id) + if conversation is None: + return None + + # `exclude_unset` so a body of {"title": "x"} cannot also write pinned=None. + for field, value in patch.model_dump(exclude_unset=True).items(): + if value is not None: + setattr(conversation, field, value) + + await db.commit() + await db.refresh(conversation) + return conversation + + +async def delete_conversation(db: AsyncSession, user_id: str, conversation_id: str) -> bool: + """True if a row was deleted. Messages go with it via ON DELETE CASCADE.""" + result = await db.execute( + delete(Conversation).where( + Conversation.id == conversation_id, Conversation.user_id == user_id + ) + ) + await db.commit() + return result.rowcount > 0 + + +# --- Messages --------------------------------------------------------------- + +async def append_message( + db: AsyncSession, + conversation: Conversation, + *, + role: str, + content: str, + status: str = "complete", + mode: str | None = None, + model: str | None = None, + citations: list | None = None, + steps: list | None = None, +) -> Message: + """ + Add a message and bump the thread's `updated_at`. + + The bump is explicit: `onupdate` fires only when a column of `conversations` is + itself written, and inserting into `messages` does not touch that row. The + sidebar orders on `updated_at`, so without this a busy thread would sink. + """ + message = Message( + conversation_id=conversation.id, + role=role, + content=content, + status=status, + mode=mode, + model=model, + citations=citations, + steps=steps, + ) + db.add(message) + + conversation.updated_at = _now() + if role == "user" and conversation.title == DEFAULT_TITLE: + conversation.title = title_from_message(content) + + await db.commit() + await db.refresh(message) + return message + + +async def set_message_feedback( + db: AsyncSession, user_id: str, conversation_id: str, message_id: str, feedback: str | None +) -> Message | None: + """ + Set or clear a thumb on one message. + + The join to `conversations` is what enforces ownership: a message id alone says + nothing about who owns it. + """ + message = await db.scalar( + select(Message) + .join(Conversation, Message.conversation_id == Conversation.id) + .where( + Message.id == message_id, + Message.conversation_id == conversation_id, + Conversation.user_id == user_id, + ) + ) + if message is None: + return None + + message.feedback = feedback + await db.commit() + await db.refresh(message) + return message + + +async def delete_message(db: AsyncSession, conversation_id: str, message_id: str) -> None: + """Remove one message. Used by retry, which regenerates the last assistant reply.""" + await db.execute( + delete(Message).where( + Message.id == message_id, Message.conversation_id == conversation_id + ) + ) + await db.commit() + + +async def last_message(db: AsyncSession, conversation_id: str, role: str) -> Message | None: + return await db.scalar( + select(Message) + .where(Message.conversation_id == conversation_id, Message.role == role) + .order_by(Message.created_at.desc(), Message.id.desc()) + .limit(1) + ) + + +# --- History ---------------------------------------------------------------- + +def _pair_into_turns(messages: list[Message]) -> list[tuple[Message, Message]]: + """ + Group an oldest-first message list into (user, assistant) turns. + + Only complete pairs survive. A user message whose answer failed, or an assistant + message with no user message before it, is dropped rather than replayed alone -- + partly because a half turn is poor context, and partly because Anthropic's + Messages API rejects a history that does not strictly alternate roles, so an + unpaired message would break one provider and not the others. + """ + turns: list[tuple[Message, Message]] = [] + pending: Message | None = None + for message in messages: + if message.role == "user": + pending = message + elif pending is not None: + turns.append((pending, message)) + pending = None + return turns + + +async def load_history( + db: AsyncSession, + conversation_id: str, + *, + turns: int, + max_tokens: int, + exclude_message_id: str | None = None, +) -> list[HistoryMessage]: + """ + The last `turns` complete exchanges, oldest first, trimmed to `max_tokens`. + + `exclude_message_id` keeps the message currently being answered out of its own + history: it is already persisted by the time this runs, and it is handed to the + provider separately as the live user message. + + The budget is applied by dropping whole turns from the oldest end. Cutting + mid-turn would leave a question without its answer, or an answer with no + question. + """ + if turns <= 0 or max_tokens <= 0: + return [] + + # Fetch newest-first under a bounded LIMIT, then reverse. Ordering by id as a + # tiebreaker keeps the sequence stable when two rows share a timestamp. + query = ( + select(Message) + .where( + Message.conversation_id == conversation_id, + Message.content != "", + Message.status.in_(_HISTORY_STATUSES), + ) + .order_by(Message.created_at.desc(), Message.id.desc()) + # +2 so that an excluded message cannot cost us a whole turn. + .limit(turns * 2 + 2) + ) + if exclude_message_id is not None: + query = query.where(Message.id != exclude_message_id) + + rows = list(await db.scalars(query)) + rows.reverse() + + recent = _pair_into_turns(rows)[-turns:] + + # Drop whole turns from the oldest end until the budget is met. + while recent: + used = sum( + estimate_tokens(user.content) + estimate_tokens(assistant.content) + for user, assistant in recent + ) + if used <= max_tokens: + break + recent.pop(0) + + history: list[HistoryMessage] = [] + for user, assistant in recent: + history.append(HistoryMessage(role="user", content=user.content)) + history.append(HistoryMessage(role="assistant", content=assistant.content)) + return history + + +# --- One-time import from localStorage -------------------------------------- + +async def import_conversations( + db: AsyncSession, user_id: str, payload: list[ImportConversation] +) -> tuple[int, int]: + """ + Bulk-insert conversations exported from the browser's legacy localStorage key. + + Returns (imported, skipped). A conversation with no messages is skipped: the old + store created a row the moment "New Chat" was clicked, so a long-lived browser + accumulates empty threads nobody wants restored. + + Client timestamps are trusted only for ordering. They are user-writable data, so + nothing security-relevant hangs off them; the worst a forged value does is + misplace a row in the caller's own sidebar. + """ + imported = 0 + skipped = 0 + + for incoming in payload: + if not incoming.messages: + skipped += 1 + continue + + created = incoming.created_at or _now() + conversation = Conversation( + user_id=user_id, + title=incoming.title or DEFAULT_TITLE, + mode=incoming.mode, + pinned=incoming.pinned, + created_at=created, + updated_at=incoming.updated_at or created, + ) + db.add(conversation) + await db.flush() + + for order, message in enumerate(incoming.messages): + db.add( + Message( + conversation_id=conversation.id, + role=message.role, + content=message.content, + status=message.status, + mode=incoming.mode, + citations=( + [c.model_dump() for c in message.citations] + if message.citations + else None + ), + feedback=message.feedback, + # Messages the old store never timestamped would otherwise all + # land on the same instant and come back in arbitrary order. + # Spacing them by their position preserves the thread. + created_at=message.created_at or created + timedelta(milliseconds=order), + ) + ) + imported += 1 + + await db.commit() + logger.info("[conversations] imported=%d skipped=%d user=%s", imported, skipped, user_id) + return imported, skipped diff --git a/backend/evaluation/results/ingest_eval-055e6b8f749602d3.json b/backend/evaluation/results/ingest_eval-055e6b8f749602d3.json new file mode 100644 index 0000000..85485f2 --- /dev/null +++ b/backend/evaluation/results/ingest_eval-055e6b8f749602d3.json @@ -0,0 +1,335 @@ +{ + "environment": { + "captured_at": "2026-09-24T08:29:22.992234+00:00", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", + "python": "3.13.1", + "cpu_count_logical": 12, + "numpy": "2.2.6", + "cpu_count_physical": 8, + "total_ram_gb": 15.68, + "onnxruntime": "1.26.0", + "git_commit": "de19497", + "git_dirty": false + }, + "config": { + "chunk_size": 800, + "chunk_overlap": 120, + "embedding_model": "BAAI/bge-base-en-v1.5", + "label": "phase8-followup" + }, + "user_id": "eval-055e6b8f749602d3", + "corpus_dir": "C:\\Users\\Madhav\\AppData\\Local\\Temp\\claude\\C--Users-Madhav-PROJECTS-productions-DocuQuery-\\7effdc90-94ce-485a-abd3-c1a93b59bcf9\\scratchpad\\followup-corpus", + "model_load_ms": 2016.9, + "model_first_inference_ms": 32.19, + "chunker_import_ms": 44326.35, + "documents_attempted": 15, + "documents_indexed": 15, + "documents_failed": 0, + "total_pages": 1108, + "total_chunks": 5574, + "total_chars": 2959235, + "total_bytes": 27002963, + "documents": [ + { + "document_id": "a8b865d8-af23-4e51-bc1f-ceadd25688b6", + "filename": "frb-scf23.pdf", + "file_type": "pdf", + "file_size_bytes": 10136125, + "parse_ms": 1657.07, + "clean_ms": 40.37, + "pages": 51, + "chars": 131446, + "chunk_ms": 15.43, + "chunks": 211, + "chunk_chars_mean": 689.6, + "embed_ms": 145628.52, + "embedding_dim": 384, + "rss_before_embed_mb": 507.7, + "rss_after_embed_mb": 1000.5, + "store_ms": 753.4, + "total_ms": 148094.79, + "status": "indexed" + }, + { + "document_id": "ffea2976-8bb2-46d1-92c6-bd6612aa4803", + "filename": "NIST.AI.100-1.pdf", + "file_type": "pdf", + "file_size_bytes": 1946127, + "parse_ms": 244.55, + "clean_ms": 29.61, + "pages": 48, + "chars": 106427, + "chunk_ms": 11.43, + "chunks": 173, + "chunk_chars_mean": 678.4, + "embed_ms": 66720.25, + "embedding_dim": 384, + "rss_before_embed_mb": 1003.0, + "rss_after_embed_mb": 1004.7, + "store_ms": 588.7, + "total_ms": 67594.54, + "status": "indexed" + }, + { + "document_id": "832c4183-916c-4e04-87e0-a9c4c6e4fd59", + "filename": "NIST.CSWP.04162018.pdf", + "file_type": "pdf", + "file_size_bytes": 1062822, + "parse_ms": 514.77, + "clean_ms": 41.28, + "pages": 55, + "chars": 125350, + "chunk_ms": 16.2, + "chunks": 231, + "chunk_chars_mean": 594.9, + "embed_ms": 164145.93, + "embedding_dim": 384, + "rss_before_embed_mb": 1006.5, + "rss_after_embed_mb": 1491.5, + "store_ms": 477.58, + "total_ms": 165195.76, + "status": "indexed" + }, + { + "document_id": "6253f709-fe56-4456-98a6-59513e53991d", + "filename": "NIST.SP.800-171r2.pdf", + "file_type": "pdf", + "file_size_bytes": 1539170, + "parse_ms": 531.17, + "clean_ms": 52.64, + "pages": 114, + "chars": 286206, + "chunk_ms": 9.87, + "chunks": 536, + "chunk_chars_mean": 580.7, + "embed_ms": 208961.48, + "embedding_dim": 384, + "rss_before_embed_mb": 1496.1, + "rss_after_embed_mb": 1499.7, + "store_ms": 2453.85, + "total_ms": 212009.01, + "status": "indexed" + }, + { + "document_id": "f006d31f-0fa9-4298-a29d-c17feb42a692", + "filename": "NIST.SP.800-190.pdf", + "file_type": "pdf", + "file_size_bytes": 666229, + "parse_ms": 656.73, + "clean_ms": 41.26, + "pages": 63, + "chars": 159541, + "chunk_ms": 14.31, + "chunks": 316, + "chunk_chars_mean": 556.4, + "embed_ms": 90876.29, + "embedding_dim": 384, + "rss_before_embed_mb": 1505.1, + "rss_after_embed_mb": 1505.6, + "store_ms": 1318.35, + "total_ms": 92906.94, + "status": "indexed" + }, + { + "document_id": "0f972c7d-ea18-4721-b3cc-d6f6d548107c", + "filename": "NIST.SP.800-207.pdf", + "file_type": "pdf", + "file_size_bytes": 966908, + "parse_ms": 520.33, + "clean_ms": 42.11, + "pages": 59, + "chars": 154234, + "chunk_ms": 14.6, + "chunks": 313, + "chunk_chars_mean": 544.3, + "embed_ms": 151014.26, + "embedding_dim": 384, + "rss_before_embed_mb": 1505.8, + "rss_after_embed_mb": 1505.8, + "store_ms": 759.12, + "total_ms": 152350.42, + "status": "indexed" + }, + { + "document_id": "79d5b0c3-b79c-4bef-8902-af3b0c01222f", + "filename": "NIST.SP.800-218.pdf", + "file_type": "pdf", + "file_size_bytes": 739891, + "parse_ms": 177.24, + "clean_ms": 8.82, + "pages": 36, + "chars": 105276, + "chunk_ms": 3.62, + "chunks": 205, + "chunk_chars_mean": 565.3, + "embed_ms": 151110.73, + "embedding_dim": 384, + "rss_before_embed_mb": 1505.8, + "rss_after_embed_mb": 1506.1, + "store_ms": 843.16, + "total_ms": 152143.57, + "status": "indexed" + }, + { + "document_id": "c5c2bebb-bb1c-4335-8bcf-b68c8e41e605", + "filename": "NIST.SP.800-37r2.pdf", + "file_type": "pdf", + "file_size_bytes": 2270327, + "parse_ms": 948.65, + "clean_ms": 54.68, + "pages": 183, + "chars": 593241, + "chunk_ms": 18.54, + "chunks": 1082, + "chunk_chars_mean": 599.2, + "embed_ms": 343164.7, + "embedding_dim": 384, + "rss_before_embed_mb": 1509.3, + "rss_after_embed_mb": 1189.1, + "store_ms": 7492.47, + "total_ms": 351679.04, + "status": "indexed" + }, + { + "document_id": "6855fdf2-146e-428c-bb3d-8217729353b4", + "filename": "NIST.SP.800-61r2.pdf", + "file_type": "pdf", + "file_size_bytes": 1744473, + "parse_ms": 951.6, + "clean_ms": 67.57, + "pages": 80, + "chars": 226699, + "chunk_ms": 19.89, + "chunks": 395, + "chunk_chars_mean": 631.3, + "embed_ms": 185931.46, + "embedding_dim": 384, + "rss_before_embed_mb": 1191.3, + "rss_after_embed_mb": 1191.5, + "store_ms": 1735.48, + "total_ms": 188706.0, + "status": "indexed" + }, + { + "document_id": "d4bd18d9-d5bb-42d6-aac6-6239b1c26159", + "filename": "NIST.SP.800-63-3.pdf", + "file_type": "pdf", + "file_size_bytes": 1599268, + "parse_ms": 839.24, + "clean_ms": 39.88, + "pages": 76, + "chars": 149237, + "chunk_ms": 16.3, + "chunks": 293, + "chunk_chars_mean": 553.4, + "embed_ms": 152936.72, + "embedding_dim": 384, + "rss_before_embed_mb": 1191.8, + "rss_after_embed_mb": 1155.3, + "store_ms": 1228.61, + "total_ms": 155060.75, + "status": "indexed" + }, + { + "document_id": "246599b5-df0c-479b-94a8-aacdd07e8f19", + "filename": "NIST.SP.800-88r1.pdf", + "file_type": "pdf", + "file_size_bytes": 714563, + "parse_ms": 653.0, + "clean_ms": 32.68, + "pages": 65, + "chars": 161457, + "chunk_ms": 12.35, + "chunks": 335, + "chunk_chars_mean": 529.5, + "embed_ms": 162708.97, + "embedding_dim": 384, + "rss_before_embed_mb": 1155.6, + "rss_after_embed_mb": 1147.3, + "store_ms": 1598.03, + "total_ms": 165005.03, + "status": "indexed" + }, + { + "document_id": "4de6ee3c-8861-4cb6-9663-688d6d14bc28", + "filename": "nistspecialpublication800-122.pdf", + "file_type": "pdf", + "file_size_bytes": 818516, + "parse_ms": 647.4, + "clean_ms": 46.66, + "pages": 59, + "chars": 173592, + "chunk_ms": 16.47, + "chunks": 335, + "chunk_chars_mean": 559.7, + "embed_ms": 163225.53, + "embedding_dim": 384, + "rss_before_embed_mb": 1147.3, + "rss_after_embed_mb": 1112.9, + "store_ms": 1371.26, + "total_ms": 165307.32, + "status": "indexed" + }, + { + "document_id": "d8938c08-450a-4026-8ef1-9245794f1bff", + "filename": "nistspecialpublication800-145.pdf", + "file_type": "pdf", + "file_size_bytes": 85781, + "parse_ms": 96.9, + "clean_ms": 3.81, + "pages": 7, + "chars": 10784, + "chunk_ms": 1.36, + "chunks": 21, + "chunk_chars_mean": 533.5, + "embed_ms": 7160.51, + "embedding_dim": 384, + "rss_before_embed_mb": 1112.9, + "rss_after_embed_mb": 1113.0, + "store_ms": 96.42, + "total_ms": 7359.0, + "status": "indexed" + }, + { + "document_id": "bd84ebe1-ad44-40dc-8b94-0847aac9d0e6", + "filename": "nistspecialpublication800-34r1.pdf", + "file_type": "pdf", + "file_size_bytes": 952921, + "parse_ms": 1079.22, + "clean_ms": 132.31, + "pages": 149, + "chars": 386997, + "chunk_ms": 55.77, + "chunks": 780, + "chunk_chars_mean": 529.8, + "embed_ms": 406736.79, + "embedding_dim": 384, + "rss_before_embed_mb": 1113.1, + "rss_after_embed_mb": 1066.1, + "store_ms": 3530.57, + "total_ms": 411534.66, + "status": "indexed" + }, + { + "document_id": "3206074a-68b9-4da9-97c3-6b4e49550405", + "filename": "nistspecialpublication800-92.pdf", + "file_type": "pdf", + "file_size_bytes": 1759842, + "parse_ms": 182.32, + "clean_ms": 15.31, + "pages": 63, + "chars": 188748, + "chunk_ms": 4.45, + "chunks": 348, + "chunk_chars_mean": 598.2, + "embed_ms": 46760.09, + "embedding_dim": 384, + "rss_before_embed_mb": 1066.5, + "rss_after_embed_mb": 1058.0, + "store_ms": 1237.28, + "total_ms": 48199.45, + "status": "indexed" + } + ] +} \ No newline at end of file diff --git a/backend/evaluation/results/persistence_check_phase8.txt b/backend/evaluation/results/persistence_check_phase8.txt new file mode 100644 index 0000000..56f145e --- /dev/null +++ b/backend/evaluation/results/persistence_check_phase8.txt @@ -0,0 +1,117 @@ +------------------------------------------------------------ + 1. Sign up in browser A +------------------------------------------------------------ +{ + "id": "c07204e6-aefc-412d-a41c-6bc1a3774aca", + "name": "Persistence Check", + "email": "persistence-1790239473@example.invalid", + "createdAt": "2026-09-24T08:44:33.596132Z" +} +cookie jar A now holds: 1 session cookie +------------------------------------------------------------ + 2. Sign in as the SAME user in browser B (a separate cookie jar) +------------------------------------------------------------ +{ + "id": "c07204e6-aefc-412d-a41c-6bc1a3774aca", + "name": "Persistence Check", + "email": "persistence-1790239473@example.invalid", + "createdAt": "2026-09-24T08:44:33.596132Z" +} +jars A and B share no state. Both cookies encode the same user id, so only +the signature at the tail distinguishes them -- they are separate sessions: + A session cookie: ...iDRDzw1yvXdcAOx4 + B session cookie: ...TLu-I7R934x7RrrE +------------------------------------------------------------ + 3. Browser B sees no chats yet +------------------------------------------------------------ +{ + "items": [], + "total": 0, + "limit": 30, + "offset": 0, + "has_more": false +} +------------------------------------------------------------ + 4. Browser A sends a message with no conversation_id +------------------------------------------------------------ +POST /chat {"message": "What is a policy enforcement point?", "mode": "llm"} + +--- first frames of the SSE stream --- +event: conversation +data: {"id": "b632b811-67ac-4e5a-b980-6d3864cf751a"} + +event: token +data: "A" + + ... + +server created conversation: b632b811-67ac-4e5a-b980-6d3864cf751a +------------------------------------------------------------ + 5. Browser B — never told about that conversation — lists its chats +------------------------------------------------------------ +{ + "items": [ + { + "id": "b632b811-67ac-4e5a-b980-6d3864cf751a", + "title": "What is a policy enforcement point?", + "mode": "llm", + "pinned": false, + "created_at": "2026-09-24T08:44:34.758230Z", + "updated_at": "2026-09-24T08:44:37.638317Z" + } + ], + "total": 1, + "limit": 30, + "offset": 0, + "has_more": false +} +------------------------------------------------------------ + 6. Browser B opens the conversation and reads the transcript +------------------------------------------------------------ +{ + "id": "b632b811-67ac-4e5a-b980-6d3864cf751a", + "title": "What is a policy enforcement point?", + "mode": "llm", + "pinned": false +} +messages: + [user complete] What is a policy enforcement point? + [assistant complete] A **Policy Enforcement Point (PEP)** is a component within a securi... +------------------------------------------------------------ + 7. Browser B logs out +------------------------------------------------------------ + POST /auth/logout -> HTTP 204 + GET /conversations with the logged-out jar -> HTTP 401 +------------------------------------------------------------ + 8. Log back in (jar C) and list again +------------------------------------------------------------ + C session cookie: ...ZJc0_EttnTEtRUec (a newly minted session) + +{ + "items": [ + { + "id": "b632b811-67ac-4e5a-b980-6d3864cf751a", + "title": "What is a policy enforcement point?", + "mode": "llm", + "pinned": false, + "created_at": "2026-09-24T08:44:34.758230Z", + "updated_at": "2026-09-24T08:44:37.638317Z" + } + ], + "total": 1, + "limit": 30, + "offset": 0, + "has_more": false +} +------------------------------------------------------------ + 9. And the transcript is still there +------------------------------------------------------------ +messages: + [user complete] What is a policy enforcement point? + [assistant complete] A **Policy Enforcement Point (PEP)** is a component within a securi... +------------------------------------------------------------ + 10. A different account cannot see it +------------------------------------------------------------ + their GET /conversations -> total=0 + their GET /conversations/b632b811... -> HTTP 404 (404, not 403) +------------------------------------------------------------ diff --git a/backend/evaluation/results/retrieval_eval_followup_phase8.json b/backend/evaluation/results/retrieval_eval_followup_phase8.json new file mode 100644 index 0000000..a35c564 --- /dev/null +++ b/backend/evaluation/results/retrieval_eval_followup_phase8.json @@ -0,0 +1,896 @@ +{ + "environment": { + "captured_at": "2026-09-24T08:30:07.974473+00:00", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", + "python": "3.13.1", + "cpu_count_logical": 12, + "numpy": "2.2.6", + "cpu_count_physical": 8, + "total_ram_gb": 15.68, + "onnxruntime": "1.26.0", + "git_commit": "de19497", + "git_dirty": true + }, + "run": { + "label": "phase8", + "mode": "followup", + "user_id": "eval-055e6b8f749602d3", + "slice_file": "evaluation/sets/manual.json", + "items": 15, + "top_k": 10, + "similarity_threshold": 0.3, + "embedding_model": "BAAI/bge-base-en-v1.5", + "llm_model": "qwen/qwen3.8-27b", + "condense_prompt_version": "v1", + "relevance_level": "document", + "turn_1_answer": "proxy: the top retrieved passage from turn 1, truncated to 600 chars. No answer was generated, so the condenser sees grounded context rather than a written reply." + }, + "provenance": { + "set_status": "DRAFT - not reviewed, not used to produce any reported number.", + "slice_description": "Two-turn exchanges where turn 2 is elliptical and meaningless alone. Measures the value of query condensing: turn-2 retrieval with the raw text vs. with a standalone rewrite.", + "verification": "The multi_hop and followup slices are NOT mechanically verified. Their ground-truth document assignments come from document scope and still need a human to confirm the answer is on the page claimed." + }, + "metrics": { + "raw": { + "queries": 15, + "recall_at_1": 0.4, + "recall_at_3": 0.5333, + "recall_at_5": 0.5333, + "recall_at_10": 0.8 + }, + "model_rewrite": { + "queries": 15, + "recall_at_1": 0.9333, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0 + }, + "reference_rewrite": { + "queries": 15, + "recall_at_1": 0.8667, + "recall_at_3": 0.9333, + "recall_at_5": 0.9333, + "recall_at_10": 0.9333 + } + }, + "rewrite_outcomes": { + "ok": 15 + }, + "rewrite_latency_ms": { + "p50": 194.357, + "p95": 1120.962, + "p99": 1120.962, + "min": 138.891, + "max": 1120.962, + "mean": 299.541, + "n": 15 + }, + "per_item": [ + { + "id": "fu-01", + "turn_1": "What is the purpose of a policy enforcement point in a zero trust architecture?", + "turn_2_raw": "How does it differ from the policy engine?", + "model_rewrite": "How does a policy enforcement point differ from the policy engine in a zero trust architecture?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 1120.96, + "reference_rewrite": "How does a policy enforcement point differ from a policy engine in a zero trust architecture?", + "gt_manifest_id": "nist-sp-800-207", + "gt_filename": "NIST.SP.800-207.pdf", + "turn_1_top_filename": "NIST.SP.800-207.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.AI.100-1.pdf", + "NIST.SP.800-207.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-02", + "turn_1": "What are the three cloud service models defined by NIST?", + "turn_2_raw": "Which of them gives the customer the least control?", + "model_rewrite": "Which of the three cloud service models defined by NIST gives the customer the least control?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 164.06, + "reference_rewrite": "Which NIST cloud service model gives the customer the least control?", + "gt_manifest_id": "nist-sp-800-145", + "gt_filename": "nistspecialpublication800-145.pdf", + "turn_1_top_filename": "nistspecialpublication800-145.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf" + ], + "model_rewrite": [ + "nistspecialpublication800-145.pdf", + "nistspecialpublication800-145.pdf", + "nistspecialpublication800-145.pdf", + "nistspecialpublication800-145.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf", + "NIST.SP.800-207.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": false + } + }, + { + "id": "fu-03", + "turn_1": "What are the phases of the incident response life cycle?", + "turn_2_raw": "What happens in the third one?", + "model_rewrite": "What happens in the third phase of the incident response life cycle?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 305.38, + "reference_rewrite": "What happens during the containment, eradication and recovery phase of incident response?", + "gt_manifest_id": "nist-sp-800-61r2", + "gt_filename": "NIST.SP.800-61r2.pdf", + "turn_1_top_filename": "NIST.SP.800-61r2.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-61r2.pdf", + "nistspecialpublication800-92.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-61r2.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-61r2.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-61r2.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-04", + "turn_1": "What is the difference between clearing and purging media?", + "turn_2_raw": "When is destruction required instead?", + "model_rewrite": "When is destruction required instead of clearing or purging media?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 155.39, + "reference_rewrite": "When is physical destruction of media required instead of clearing or purging?", + "gt_manifest_id": "nist-sp-800-88r1", + "gt_filename": "NIST.SP.800-88r1.pdf", + "turn_1_top_filename": "NIST.SP.800-88r1.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-190.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "nistspecialpublication800-92.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-05", + "turn_1": "What are the core functions of the Cybersecurity Framework?", + "turn_2_raw": "Which one covers incident response?", + "model_rewrite": "Which function of the Cybersecurity Framework covers incident response?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 138.89, + "reference_rewrite": "Which Cybersecurity Framework core function covers incident response?", + "gt_manifest_id": "nist-cswp-cyberframework-1-1", + "gt_filename": "NIST.CSWP.04162018.pdf", + "turn_1_top_filename": "NIST.CSWP.04162018.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-171r2.pdf" + ], + "model_rewrite": [ + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-61r2.pdf" + ], + "reference_rewrite": [ + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-61r2.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-06", + "turn_1": "What are the four functions of the AI Risk Management Framework?", + "turn_2_raw": "What does the first one involve?", + "model_rewrite": "What does the first function of the AI Risk Management Framework involve?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 631.86, + "reference_rewrite": "What does the Govern function of the AI Risk Management Framework involve?", + "gt_manifest_id": "nist-ai-100-1", + "gt_filename": "NIST.AI.100-1.pdf", + "turn_1_top_filename": "NIST.AI.100-1.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.AI.100-1.pdf", + "nistspecialpublication800-34r1.pdf" + ], + "model_rewrite": [ + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf" + ], + "reference_rewrite": [ + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf", + "NIST.AI.100-1.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-07", + "turn_1": "What is a container image?", + "turn_2_raw": "What are the main risks associated with them?", + "model_rewrite": "What are the main risks associated with container images?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 332.4, + "reference_rewrite": "What are the main security risks associated with container images?", + "gt_manifest_id": "nist-sp-800-190", + "gt_filename": "NIST.SP.800-190.pdf", + "turn_1_top_filename": "NIST.SP.800-190.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-122.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.AI.100-1.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.AI.100-1.pdf", + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-63-3.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-37r2.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-08", + "turn_1": "What are the steps of the Risk Management Framework?", + "turn_2_raw": "Which step comes immediately after categorisation?", + "model_rewrite": "Which step comes immediately after categorisation in the Risk Management Framework?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 164.3, + "reference_rewrite": "Which Risk Management Framework step comes immediately after the categorise step?", + "gt_manifest_id": "nist-sp-800-37r2", + "gt_filename": "NIST.SP.800-37r2.pdf", + "turn_1_top_filename": "NIST.SP.800-37r2.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-37r2.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-88r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-09", + "turn_1": "What is controlled unclassified information?", + "turn_2_raw": "How many security requirements protect it?", + "model_rewrite": "How many security requirements protect Controlled Unclassified Information?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 142.86, + "reference_rewrite": "How many security requirements does NIST specify for protecting controlled unclassified information?", + "gt_manifest_id": "nist-sp-800-171r2", + "gt_filename": "NIST.SP.800-171r2.pdf", + "turn_1_top_filename": "NIST.SP.800-171r2.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-171r2.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-190.pdf", + "NIST.SP.800-190.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-10", + "turn_1": "What is the definition of personally identifiable information?", + "turn_2_raw": "What are the confidentiality impact levels for it?", + "model_rewrite": "What are the confidentiality impact levels for personally identifiable information?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 142.29, + "reference_rewrite": "What are the confidentiality impact levels for personally identifiable information?", + "gt_manifest_id": "nist-sp-800-122", + "gt_filename": "nistspecialpublication800-122.pdf", + "turn_1_top_filename": "NIST.SP.800-37r2.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-88r1.pdf", + "nistspecialpublication800-122.pdf" + ], + "model_rewrite": [ + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf" + ], + "reference_rewrite": [ + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-122.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-11", + "turn_1": "What are the practices of the Secure Software Development Framework?", + "turn_2_raw": "Which group of them deals with the organisation rather than the code?", + "model_rewrite": "Which group of practices in the Secure Software Development Framework (SSDF) deals with the organization rather than the code?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 224.17, + "reference_rewrite": "Which Secure Software Development Framework practice group deals with preparing the organisation rather than the code?", + "gt_manifest_id": "nist-sp-800-218", + "gt_filename": "NIST.SP.800-218.pdf", + "turn_1_top_filename": "NIST.SP.800-218.pdf", + "retrieved_filenames": { + "raw": [ + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-122.pdf", + "NIST.CSWP.04162018.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-88r1.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-61r2.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf", + "NIST.SP.800-218.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-12", + "turn_1": "What is the purpose of a business impact analysis?", + "turn_2_raw": "What outputs does it produce?", + "model_rewrite": "What outputs does a business impact analysis produce?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 405.63, + "reference_rewrite": "What outputs does a business impact analysis produce?", + "gt_manifest_id": "nist-sp-800-34r1", + "gt_filename": "nistspecialpublication800-34r1.pdf", + "turn_1_top_filename": "NIST.SP.800-37r2.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-145.pdf", + "NIST.SP.800-88r1.pdf", + "nistspecialpublication800-122.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-171r2.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-37r2.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-37r2.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-61r2.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.CSWP.04162018.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-37r2.pdf" + ] + }, + "hit_at_5": { + "raw": false, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-13", + "turn_1": "What are the authenticator assurance levels?", + "turn_2_raw": "What is required at the highest one?", + "model_rewrite": "What is required at the highest Authenticator Assurance Level (AAL)?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 156.93, + "reference_rewrite": "What is required at authenticator assurance level 3?", + "gt_manifest_id": "nist-sp-800-63-3", + "gt_filename": "NIST.SP.800-63-3.pdf", + "turn_1_top_filename": "NIST.SP.800-63-3.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-92.pdf", + "NIST.SP.800-63-3.pdf", + "nistspecialpublication800-92.pdf", + "NIST.SP.800-63-3.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-34r1.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-92.pdf", + "NIST.SP.800-63-3.pdf", + "nistspecialpublication800-34r1.pdf" + ], + "model_rewrite": [ + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf" + ], + "reference_rewrite": [ + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-171r2.pdf", + "NIST.SP.800-63-3.pdf", + "NIST.SP.800-63-3.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-14", + "turn_1": "What is log management?", + "turn_2_raw": "What are the main challenges in doing it?", + "model_rewrite": "What are the main challenges in computer security log management?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 194.36, + "reference_rewrite": "What are the main challenges in log management?", + "gt_manifest_id": "nist-sp-800-92", + "gt_filename": "nistspecialpublication800-92.pdf", + "turn_1_top_filename": "nistspecialpublication800-92.pdf", + "retrieved_filenames": { + "raw": [ + "nistspecialpublication800-92.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-122.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-92.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "NIST.SP.800-37r2.pdf", + "nistspecialpublication800-34r1.pdf", + "nistspecialpublication800-92.pdf" + ], + "model_rewrite": [ + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "NIST.SP.800-171r2.pdf" + ], + "reference_rewrite": [ + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf", + "nistspecialpublication800-92.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + }, + { + "id": "fu-15", + "turn_1": "What does the Survey of Consumer Finances measure?", + "turn_2_raw": "How did median family net worth change in the most recent period it covers?", + "model_rewrite": "How did median family net worth change in the most recent period covered by the Survey of Consumer Finances?", + "rewrite_outcome": "ok", + "rewrite_latency_ms": 213.62, + "reference_rewrite": "How did median family net worth change between 2019 and 2022 according to the Survey of Consumer Finances?", + "gt_manifest_id": "frb-scf-2023", + "gt_filename": "frb-scf23.pdf", + "turn_1_top_filename": "frb-scf23.pdf", + "retrieved_filenames": { + "raw": [ + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf" + ], + "model_rewrite": [ + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf" + ], + "reference_rewrite": [ + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf", + "frb-scf23.pdf" + ] + }, + "hit_at_5": { + "raw": true, + "model_rewrite": true, + "reference_rewrite": true + } + } + ] +} \ No newline at end of file diff --git a/backend/evaluation/run_retrieval_eval.py b/backend/evaluation/run_retrieval_eval.py index 2120273..bc83bc3 100644 --- a/backend/evaluation/run_retrieval_eval.py +++ b/backend/evaluation/run_retrieval_eval.py @@ -18,10 +18,16 @@ relevant chunk in the top k. Both names are reported with that caveat attached rather than presented as two independent results. +Follow-up mode (--mode followup) measures something different: turn 2 of a +two-turn exchange, retrieved three ways -- on the user's literal follow-up, on +the condenser's rewrite of it, and on the slice's hand-written reference rewrite. +That is the number query condensing has to justify itself with. + Usage: python evaluation/run_retrieval_eval.py --user-id eval-xxxx python evaluation/run_retrieval_eval.py --user-id eval-xxxx --no-threshold python evaluation/run_retrieval_eval.py --user-id eval-xxxx --filter-document + python evaluation/run_retrieval_eval.py --user-id eval-xxxx --mode followup """ from __future__ import annotations @@ -102,9 +108,241 @@ def score_level(per_query: list[list[bool]]) -> dict: return out +# --- Follow-up mode --------------------------------------------------------- +# +# Measures the one thing query condensing exists for: whether turn 2 of a +# conversation retrieves better when the elliptical follow-up is rewritten into a +# standalone query than when it is embedded literally. +# +# Three variants run over the same items, so the columns are comparable: +# +# raw the user's literal turn-2 text -- what shipped before +# model_rewrite app.llm.condenser's output, i.e. the deployed path +# reference_rewrite the hand-written rewrite in the slice, a ceiling for what +# a perfect condenser could achieve on this corpus +# +# Ground truth in this slice names documents by their manifest id, never by +# database document_id, because the latter is regenerated on every ingest. The +# join is through the filename the corpus was ingested under. + +FOLLOWUP_LEVEL = "document" + +# The condenser needs something to stand in for the assistant's turn-1 reply. +# Generating one would put a second, non-deterministic LLM call in the middle of +# a measurement; the top retrieved passage is what a grounded answer would have +# been written from, so it is used instead. This is a proxy, and it is recorded +# in the result file as one. +PROXY_ANSWER_CHARS = 600 + + +def load_manifest_filenames(path: str) -> dict[str, str]: + """manifest id -> the filename the corpus was ingested under.""" + import json + from pathlib import Path + + manifest = json.loads(Path(path).read_text(encoding="utf-8")) + return {doc["id"]: doc["filename"] for doc in manifest["documents"]} + + +def load_followup_items(path: str) -> tuple[list[dict], dict]: + import json + from pathlib import Path + + blob = json.loads(Path(path).read_text(encoding="utf-8")) + slice_ = blob["slices"]["followup"] + provenance = { + "set_status": blob.get("status"), + "slice_description": slice_.get("description"), + "verification": blob.get("verification", {}).get("not_verified"), + } + return slice_["items"], provenance + + +async def retrieve_filenames(db, query: str, user_id: str, top_k: int) -> list[str]: + """Filenames of the top-k chunks, in rank order. Empty when nothing clears.""" + try: + result = await run_retrieval_pipeline( + query=query, db=db, user_id=user_id, top_k=top_k, filters=None + ) + except NoResultsError: + return [] + return [chunk.filename for chunk in result.chunks] + + +async def retrieve_chunks(db, query: str, user_id: str, top_k: int): + try: + result = await run_retrieval_pipeline( + query=query, db=db, user_id=user_id, top_k=top_k, filters=None + ) + except NoResultsError: + return [] + return result.chunks + + +def recall_at(hits: list[list[bool]], k: int) -> float: + if not hits: + return 0.0 + return round(sum(1 for rels in hits if any(rels[:k])) / len(hits), 4) + + +async def run_followup(args) -> None: + from app.llm import get_response_generator + from app.llm.condenser import condense_query + from app.llm.models import HistoryMessage + + settings = get_settings() + settings.retrieval_max_context_chunks = max( + settings.retrieval_max_context_chunks, args.top_k + ) + + items, provenance = load_followup_items(args.slice_file) + filenames = load_manifest_filenames(args.manifest) + print(f"[follow-up] {len(items)} two-turn items from {args.slice_file}") + + variants = ("raw", "model_rewrite", "reference_rewrite") + per_variant: dict[str, list[list[bool]]] = {v: [] for v in variants} + rewrite_latencies: list[float] = [] + rewrite_outcomes: dict[str, int] = {} + rows: list[dict] = [] + + provider = get_response_generator().provider + + engine = create_async_engine(eval_database_url(), pool_pre_ping=True) + Session = async_sessionmaker(engine, expire_on_commit=False) + + async with Session() as db: + for index, item in enumerate(items, 1): + turn_1 = item["turn_1"] + turn_2 = item["turn_2"] + gt_filename = filenames.get(turn_2["gt_document_id"]) + if gt_filename is None: + print(f" [skip] {item['id']}: unknown manifest id {turn_2['gt_document_id']!r}") + continue + + # Turn 1 establishes the conversation the follow-up depends on. + turn_1_chunks = await retrieve_chunks(db, turn_1["query"], args.user_id, args.top_k) + proxy_answer = ( + turn_1_chunks[0].text[:PROXY_ANSWER_CHARS] if turn_1_chunks else "" + ) + history = [ + HistoryMessage(role="user", content=turn_1["query"]), + HistoryMessage(role="assistant", content=proxy_answer), + ] + + condensed = await condense_query(provider, turn_2["query"], history) + rewrite_outcomes[condensed.outcome] = rewrite_outcomes.get(condensed.outcome, 0) + 1 + if condensed.outcome == "ok": + rewrite_latencies.append(condensed.latency_ms) + + queries = { + "raw": turn_2["query"], + "model_rewrite": condensed.query, + "reference_rewrite": item["reference_rewrite"], + } + + retrieved: dict[str, list[str]] = {} + for variant in variants: + names = await retrieve_filenames(db, queries[variant], args.user_id, args.top_k) + retrieved[variant] = names + per_variant[variant].append([name == gt_filename for name in names]) + + rows.append( + { + "id": item["id"], + "turn_1": turn_1["query"], + "turn_2_raw": turn_2["query"], + "model_rewrite": condensed.query, + "rewrite_outcome": condensed.outcome, + "rewrite_latency_ms": round(condensed.latency_ms, 2), + "reference_rewrite": item["reference_rewrite"], + "gt_manifest_id": turn_2["gt_document_id"], + "gt_filename": gt_filename, + "turn_1_top_filename": turn_1_chunks[0].filename if turn_1_chunks else None, + "retrieved_filenames": retrieved, + "hit_at_5": { + variant: any(per_variant[variant][-1][:5]) for variant in variants + }, + } + ) + print(f" [{index}/{len(items)}] {item['id']} scored") + + await engine.dispose() + + metrics = { + variant: { + "queries": len(per_variant[variant]), + **{f"recall_at_{k}": recall_at(per_variant[variant], k) for k in K_VALUES}, + } + for variant in variants + } + + payload = { + "run": { + "label": args.label, + "mode": "followup", + "user_id": args.user_id, + "slice_file": args.slice_file, + "items": len(rows), + "top_k": args.top_k, + "similarity_threshold": settings.retrieval_similarity_threshold, + "embedding_model": settings.embedding_model, + "llm_model": settings.llm_model, + "condense_prompt_version": "v1", + "relevance_level": FOLLOWUP_LEVEL, + "turn_1_answer": ( + "proxy: the top retrieved passage from turn 1, truncated to " + f"{PROXY_ANSWER_CHARS} chars. No answer was generated, so the " + "condenser sees grounded context rather than a written reply." + ), + }, + "provenance": provenance, + "metrics": metrics, + "rewrite_outcomes": rewrite_outcomes, + "rewrite_latency_ms": percentiles(rewrite_latencies), + "per_item": rows, + } + + name = args.out or f"retrieval_eval_followup_{args.label or args.user_id}.json" + save(name, payload) + + print() + print("=== turn-2 retrieval, document level ===") + print(f"{'variant':20s} {'n':>4s} {'R@1':>7s} {'R@3':>7s} {'R@5':>7s}") + for variant in variants: + m = metrics[variant] + print( + f"{variant:20s} {m['queries']:4d} " + f"{m['recall_at_1']:7.3f} {m['recall_at_3']:7.3f} {m['recall_at_5']:7.3f}" + ) + latency = payload["rewrite_latency_ms"] + print() + print(f"rewrite latency ms: P50={latency['p50']} P95={latency['p95']} n={latency['n']}") + print(f"rewrite outcomes: {rewrite_outcomes}") + if provenance.get("set_status"): + print() + print(f"NOTE: {provenance['set_status']}") + + async def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--user-id", required=True) + ap.add_argument( + "--mode", + choices=("standard", "followup"), + default="standard", + help="standard scores the generated eval set; followup scores turn 2 of the " + "two-turn slice with and without query condensing", + ) + ap.add_argument( + "--slice-file", + default="evaluation/sets/manual.json", + help="follow-up mode only: the hand-written slice file", + ) + ap.add_argument( + "--manifest", + default="evaluation/corpus/manifest.json", + help="follow-up mode only: maps the slice's manifest ids to ingested filenames", + ) ap.add_argument("--eval-set", default="eval_set.json") ap.add_argument("--top-k", type=int, default=10) ap.add_argument( @@ -121,6 +359,10 @@ async def main() -> None: ap.add_argument("--out", default=None) args = ap.parse_args() + if args.mode == "followup": + await run_followup(args) + return + settings = get_settings() original_threshold = settings.retrieval_similarity_threshold if args.no_threshold: diff --git a/backend/migrations/versions/e7c4b9d2a6f1_add_conversations_and_messages.py b/backend/migrations/versions/e7c4b9d2a6f1_add_conversations_and_messages.py new file mode 100644 index 0000000..f84cd08 --- /dev/null +++ b/backend/migrations/versions/e7c4b9d2a6f1_add_conversations_and_messages.py @@ -0,0 +1,90 @@ +"""add conversations and messages for server-side chat history + +Revision ID: e7c4b9d2a6f1 +Revises: d5e2c8a7f1b3 +Create Date: 2026-09-22 00:00:00.000000 + +Chats lived only in each browser's localStorage until now, so a thread vanished on +logout and never followed the account to another machine. These two tables move +them server-side. + +Purely additive: two new tables and their indexes. Nothing existing is altered, so +`alembic upgrade head` on boot cannot disturb a running deployment's documents, +chunks or users. `downgrade()` drops both tables, which is the only possible inverse +of creating them -- it therefore discards every saved conversation. That is honest +rather than safe, and it is the reason the upgrade direction was kept additive: the +way out of a bad conversations schema is another migration, not this downgrade. +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "e7c4b9d2a6f1" +down_revision = "d5e2c8a7f1b3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "conversations", + sa.Column("id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("title", sa.String(length=200), nullable=False, server_default="New Chat"), + sa.Column("mode", sa.String(length=16), nullable=False, server_default="docuquery"), + sa.Column("pinned", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + # Serves the sidebar's only query -- one user's threads, most recent first -- + # as an index-order scan rather than a filter plus sort. + op.create_index( + "ix_conversations_user_id_updated_at", + "conversations", + ["user_id", sa.text("updated_at DESC")], + ) + + op.create_table( + "messages", + sa.Column("id", sa.String(), nullable=False), + sa.Column("conversation_id", sa.String(), nullable=False), + sa.Column("role", sa.String(length=16), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + # complete | partial | error -- see app/database/models.py:Message. + # Deliberately a plain String rather than a native enum: adding a state to a + # Postgres ENUM needs its own migration, and these states are application + # vocabulary that is expected to grow. + sa.Column("status", sa.String(length=16), nullable=False, server_default="complete"), + sa.Column("mode", sa.String(length=16), nullable=True), + sa.Column("model", sa.String(length=120), nullable=True), + sa.Column("citations", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("steps", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("feedback", sa.String(length=8), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), + sa.ForeignKeyConstraint(["conversation_id"], ["conversations.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_messages_conversation_id_created_at", + "messages", + ["conversation_id", "created_at"], + ) + + +def downgrade() -> None: + """ + Drop both tables, discarding every stored conversation and message. + + `messages` goes first. Its FK to `conversations` is ON DELETE CASCADE, which + governs row deletes, not DROP TABLE -- Postgres refuses to drop a table another + table still references unless CASCADE is spelled out, and dropping the dependent + table first says the same thing without also silently dropping anything else that + happens to depend on `conversations`. + """ + op.drop_index("ix_messages_conversation_id_created_at", table_name="messages") + op.drop_table("messages") + op.drop_index("ix_conversations_user_id_updated_at", table_name="conversations") + op.drop_table("conversations") diff --git a/backend/scripts/persistence_check.sh b/backend/scripts/persistence_check.sh new file mode 100644 index 0000000..2041b53 --- /dev/null +++ b/backend/scripts/persistence_check.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Does a chat survive a different browser, and a logout/login round trip? +# +# Three cookie jars stand in for three browser sessions belonging to the SAME +# account. Nothing is shared between them but the credentials: no localStorage, +# no in-memory state, separate cookie stores on disk. +# +# jar A the browser the chat is created in +# jar B a second browser, signed in independently -- the "different browser" +# jar C jar B after logging out and back in +# +# If conversations were still client-side, B and C would both be empty. + +set -u +API=http://127.0.0.1:8011 +DIR="$(dirname "$0")/jars" +rm -rf "$DIR"; mkdir -p "$DIR" +A="$DIR/a.txt"; B="$DIR/b.txt"; C="$DIR/c.txt" + +EMAIL="persistence-$(date +%s)@example.invalid" +PASS="correct horse battery staple" + +hr() { printf '%s\n' "------------------------------------------------------------"; } +step(){ hr; printf ' %s\n' "$*"; hr; } + +step "1. Sign up in browser A" +curl -s -c "$A" -X POST "$API/auth/signup" -H 'Content-Type: application/json' \ + -d "{\"name\":\"Persistence Check\",\"email\":\"$EMAIL\",\"password\":\"$PASS\"}" \ + | python -m json.tool +echo "cookie jar A now holds: $(grep -c docuquery_session "$A" || true) session cookie" + +step "2. Sign in as the SAME user in browser B (a separate cookie jar)" +curl -s -c "$B" -X POST "$API/auth/login" -H 'Content-Type: application/json' \ + -d "{\"email\":\"$EMAIL\",\"password\":\"$PASS\"}" \ + | python -m json.tool +echo "jars A and B share no state. Both cookies encode the same user id, so only" +echo "the signature at the tail distinguishes them -- they are separate sessions:" +echo " A session cookie: ...$(awk '/docuquery_session/{print substr($7,length($7)-15)}' "$A")" +echo " B session cookie: ...$(awk '/docuquery_session/{print substr($7,length($7)-15)}' "$B")" + +step "3. Browser B sees no chats yet" +curl -s -b "$B" "$API/conversations" | python -m json.tool + +step "4. Browser A sends a message with no conversation_id" +echo "POST /chat {\"message\": \"What is a policy enforcement point?\", \"mode\": \"llm\"}" +echo +RAW=$(curl -s -N -b "$A" -X POST "$API/chat" -H 'Content-Type: application/json' \ + -d '{"message":"What is a policy enforcement point?","mode":"llm"}') +echo "--- first frames of the SSE stream ---" +printf '%s\n' "$RAW" | head -6 +echo " ..." +CONV=$(printf '%s\n' "$RAW" | grep -m1 -A1 '^event: conversation' | tail -1 \ + | sed 's/^data: //' | python -c "import json,sys; print(json.load(sys.stdin)['id'])") +echo +echo "server created conversation: $CONV" + +step "5. Browser B — never told about that conversation — lists its chats" +curl -s -b "$B" "$API/conversations" | python -m json.tool + +step "6. Browser B opens the conversation and reads the transcript" +curl -s -b "$B" "$API/conversations/$CONV" | python -c " +import json,sys +d=json.load(sys.stdin) +print(json.dumps({'id':d['id'],'title':d['title'],'mode':d['mode'],'pinned':d['pinned']},indent=2)) +print('messages:') +for m in d['messages']: + body = m['content'] if len(m['content'])<=70 else m['content'][:67]+'...' + print(f\" [{m['role']:9s} {m['status']:8s}] {body}\") +" + +step "7. Browser B logs out" +curl -s -o /dev/null -w " POST /auth/logout -> HTTP %{http_code}\n" -b "$B" -c "$B" -X POST "$API/auth/logout" +printf " GET /conversations with the logged-out jar -> HTTP %s\n" \ + "$(curl -s -o /dev/null -w '%{http_code}' -b "$B" "$API/conversations")" + +step "8. Log back in (jar C) and list again" +curl -s -c "$C" -X POST "$API/auth/login" -H 'Content-Type: application/json' \ + -d "{\"email\":\"$EMAIL\",\"password\":\"$PASS\"}" > /dev/null +echo " C session cookie: ...$(awk '/docuquery_session/{print substr($7,length($7)-15)}' "$C") (a newly minted session)" +echo +curl -s -b "$C" "$API/conversations" | python -m json.tool + +step "9. And the transcript is still there" +curl -s -b "$C" "$API/conversations/$CONV" | python -c " +import json,sys +d=json.load(sys.stdin) +print('messages:') +for m in d['messages']: + body = m['content'] if len(m['content'])<=70 else m['content'][:67]+'...' + print(f\" [{m['role']:9s} {m['status']:8s}] {body}\") +" + +step "10. A different account cannot see it" +OTHER="intruder-$(date +%s)@example.invalid" +curl -s -c "$DIR/x.txt" -X POST "$API/auth/signup" -H 'Content-Type: application/json' \ + -d "{\"name\":\"Someone Else\",\"email\":\"$OTHER\",\"password\":\"$PASS\"}" > /dev/null +OTHER_TOTAL=$(curl -s -b "$DIR/x.txt" "$API/conversations" \ + | python -c 'import json,sys; print(json.load(sys.stdin)["total"])') +OTHER_CODE=$(curl -s -o /dev/null -w '%{http_code}' -b "$DIR/x.txt" "$API/conversations/$CONV") +printf " their GET /conversations -> total=%s\n" "$OTHER_TOTAL" +printf " their GET /conversations/%s... -> HTTP %s (404, not 403)\n" "${CONV:0:8}" "$OTHER_CODE" +hr diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 56c086b..36fac86 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -10,16 +10,18 @@ integration test is skipped, so `pytest -q` on a laptop with no database behaves exactly as it did before. CI sets it to a pgvector/pgvector:pg16 service container. - # local - docker run -d --name docuquery-test-db -p 5433:5432 \ - -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=docuquery_test \ - pgvector/pgvector:pg16 - export TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5433/docuquery_test + # local -- port 5434, not 5433: the dev stack's docker compose already binds + # 5433, and the fixtures below TRUNCATE whatever they are pointed at. + docker run -d --name docuquery-test-db -p 5434:5432 \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=docuquery_test pgvector/pgvector:pg16 + export TEST_DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5434/docuquery_test pytest -q -m integration """ from __future__ import annotations +import asyncio import os import subprocess import sys @@ -31,15 +33,19 @@ import pytest_asyncio from httpx import ASGITransport, AsyncClient from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.ext.asyncio import ( + AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine, +) from sqlalchemy.pool import NullPool BACKEND_DIR = Path(__file__).resolve().parent.parent # Tables emptied between integration tests. Ordering is irrelevant because the -# TRUNCATE is a single statement with CASCADE, but document_chunks is listed first -# to make the dependency direction obvious to a reader. -_TRUNCATE_TABLES = ("document_chunks", "documents", "users") +# TRUNCATE is a single statement with CASCADE, but dependents are listed first to +# make the direction obvious to a reader. `messages` and `conversations` would be +# reached anyway through users' CASCADE; naming them keeps the list honest about +# what the fixture empties. +_TRUNCATE_TABLES = ("document_chunks", "documents", "messages", "conversations", "users") # Alembic's env.py reads the app Settings, which refuse to build without AUTH_SECRET # whenever DEBUG is false. The migration subprocess therefore needs one of its own; @@ -101,28 +107,53 @@ def migrated_database(database_url: str) -> str: @pytest_asyncio.fixture -async def db_session(migrated_database: str) -> AsyncGenerator[AsyncSession, None]: - """A session against the migrated test database, with the tables emptied first.""" +async def test_engine(migrated_database: str) -> AsyncGenerator[AsyncEngine, None]: + """ + An engine on the migrated test database, with the tables emptied first. + + Separate from `db_session` because application code that opens its own session + -- /chat saving an assistant message after the request session has closed -- + needs the engine itself to build a sessionmaker from. `db_session.get_bind()` + is not that: on an AsyncSession it returns the *sync* Engine proxy, which + `async_sessionmaker` rejects. + """ # NullPool: each test gets its own connections and disposes of them, rather than # leaving pooled ones open against a container that the next test truncates. engine = create_async_engine(migrated_database, poolclass=NullPool) - sessionmaker = async_sessionmaker(engine, expire_on_commit=False) async with engine.begin() as conn: await conn.execute(text(f"TRUNCATE {', '.join(_TRUNCATE_TABLES)} RESTART IDENTITY CASCADE")) try: - async with sessionmaker() as session: - yield session + yield engine finally: await engine.dispose() @pytest_asyncio.fixture -async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]: +async def db_session(test_engine: AsyncEngine) -> AsyncGenerator[AsyncSession, None]: + """A session against the migrated, emptied test database.""" + sessionmaker = async_sessionmaker(test_engine, expire_on_commit=False) + async with sessionmaker() as session: + yield session + + +@pytest_asyncio.fixture +async def client( + db_session: AsyncSession, test_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch +) -> AsyncGenerator[AsyncClient, None]: """ An HTTP client bound to the real test database. `get_db` is overridden to hand every request the one session the test also holds, so a row the test writes is visible to the request and vice versa. + + `app.api.chat.AsyncSessionLocal` is redirected at the same engine. Overriding + `get_db` is not enough on its own: /chat saves the assistant message from the + streaming generator's `finally`, where the request-scoped session is already + closed, so it opens one of its own from the sessionmaker. Left alone that + sessionmaker points at the *application's* DATABASE_URL, and every assistant + message an integration test provoked would be written to the developer's own + database instead of the disposable one -- invisible to the assertions here, + and quietly polluting a database this suite was never pointed at. """ from app.api.dependencies import get_db from app.main import app @@ -130,6 +161,11 @@ async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]: async def _override_get_db() -> AsyncGenerator[AsyncSession, None]: yield db_session + monkeypatch.setattr( + "app.api.chat.AsyncSessionLocal", + async_sessionmaker(test_engine, expire_on_commit=False), + ) + app.dependency_overrides[get_db] = _override_get_db try: async with AsyncClient( @@ -140,6 +176,44 @@ async def _override_get_db() -> AsyncGenerator[AsyncSession, None]: app.dependency_overrides.pop(get_db, None) +@pytest_asyncio.fixture +async def live_server(client: AsyncClient) -> AsyncGenerator[str, None]: + """ + The app on a real TCP port, for the one thing ASGITransport cannot do. + + httpx's ASGITransport buffers the entire response body before yielding a + single line, so under it a streaming endpoint always runs to completion and a + client cannot disconnect mid-answer. Testing the partial-save path needs a + socket that can actually be closed early. + + Depends on `client` for its side effects, not its value: that fixture installs + the `get_db` override and redirects `AsyncSessionLocal`, and uvicorn runs in + this same process and event loop, so both apply to requests it serves. + + `lifespan="off"` skips the app's startup hook, which eagerly loads the ONNX + embedding model. Nothing here embeds anything, and loading it would cost time + and a few hundred MB per test run. + """ + import uvicorn + + from app.main import app + + config = uvicorn.Config(app, host="127.0.0.1", port=0, log_level="warning", lifespan="off") + server = uvicorn.Server(config) + task = asyncio.create_task(server.serve()) + + # port=0 means the OS picks one; it is only knowable after the socket is bound. + while not server.started: + await asyncio.sleep(0.02) + port = server.servers[0].sockets[0].getsockname()[1] + + try: + yield f"http://127.0.0.1:{port}" + finally: + server.should_exit = True + await task + + @dataclass class SeededUser: """A user that exists in the test database, plus the cookie that authenticates it.""" @@ -182,6 +256,60 @@ async def _make(email: str, name: str = "Test User", password: str = default_pas return _make +@pytest_asyncio.fixture +async def make_conversation(db_session: AsyncSession): + """ + Factory for a conversation owned by `user_id`, optionally pre-filled with turns. + + `turns` is a list of (user_text, assistant_text) pairs, written in order with + increasing timestamps so that `ORDER BY created_at` reproduces the thread. The + explicit spacing matters: rows inserted in one transaction can otherwise share + a timestamp to the microsecond, and the history loader would then be asserting + against an arbitrary order. + """ + from datetime import datetime, timedelta, timezone + + from app.database.models import Conversation, Message + + async def _make( + user_id: str, + *, + title: str = "New Chat", + mode: str = "docuquery", + pinned: bool = False, + turns: list[tuple[str, str]] | None = None, + ) -> Conversation: + base = datetime.now(timezone.utc) - timedelta(hours=1) + conversation = Conversation( + user_id=user_id, title=title, mode=mode, pinned=pinned, + created_at=base, updated_at=base, + ) + db_session.add(conversation) + await db_session.flush() + + for index, (question, answer) in enumerate(turns or []): + db_session.add( + Message( + conversation_id=conversation.id, role="user", content=question, + status="complete", mode=mode, + created_at=base + timedelta(seconds=index * 2), + ) + ) + db_session.add( + Message( + conversation_id=conversation.id, role="assistant", content=answer, + status="complete", mode=mode, + created_at=base + timedelta(seconds=index * 2 + 1), + ) + ) + + await db_session.commit() + await db_session.refresh(conversation) + return conversation + + return _make + + @pytest_asyncio.fixture async def make_document(db_session: AsyncSession): """ diff --git a/backend/tests/test_conversation_memory.py b/backend/tests/test_conversation_memory.py new file mode 100644 index 0000000..e2db940 --- /dev/null +++ b/backend/tests/test_conversation_memory.py @@ -0,0 +1,307 @@ +""" +Unit tests for the conversation-memory logic that needs no database. + +The database-backed behaviour lives in tests/test_integration_conversations.py. +What is covered here is the shaping that happens either side of it: how turns are +paired, how each provider lays a history list onto its own wire format, and how +the query condenser behaves when the provider misbehaves. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from app.core.config import Settings +from app.llm.condenser import MAX_REWRITE_CHARS, _clean, condense_query +from app.llm.exceptions import ProviderUnavailableError, RateLimitError +from app.llm.models import HistoryMessage, LLMRequest, StreamCapture +from app.llm.providers.base import ( + build_anthropic_messages, + build_chat_messages, + build_user_turn, +) +from app.llm.providers.gemini_provider import GeminiProvider +from app.llm.stream import conversation_event, stream_with_citations +from app.services.conversation_service import _pair_into_turns, title_from_message + +# pytest.ini sets asyncio_mode = auto, so async tests need no marker. + + +def _msg(role: str, content: str): + return SimpleNamespace(role=role, content=content) + + +HISTORY = [ + HistoryMessage("user", "What are the RMF steps?"), + HistoryMessage("assistant", "Prepare, categorize, select."), +] + + +def _request(**overrides) -> LLMRequest: + return LLMRequest( + system_prompt=overrides.pop("system_prompt", "SYSTEM"), + user_message=overrides.pop("user_message", "Which comes after categorisation?"), + context=overrides.pop("context", "[1] passage text"), + model="test-model", + temperature=0.1, + max_tokens=256, + history=overrides.pop("history", list(HISTORY)), + ) + + +# --- Turn pairing ------------------------------------------------------------ + +def test_pairing_keeps_complete_exchanges_in_order(): + paired = _pair_into_turns( + [_msg("user", "q1"), _msg("assistant", "a1"), _msg("user", "q2"), _msg("assistant", "a2")] + ) + assert [(u.content, a.content) for u, a in paired] == [("q1", "a1"), ("q2", "a2")] + + +def test_pairing_drops_a_user_message_that_was_never_answered(): + """ + A question whose answer failed is not replayed on its own. + + Beyond being poor context, an unpaired message breaks role alternation, which + Anthropic's Messages API rejects outright -- so this would fail on one + provider and quietly degrade on the rest. + """ + paired = _pair_into_turns( + [_msg("user", "unanswered"), _msg("user", "q2"), _msg("assistant", "a2")] + ) + assert [(u.content, a.content) for u, a in paired] == [("q2", "a2")] + + +def test_pairing_drops_a_leading_assistant_message(): + paired = _pair_into_turns([_msg("assistant", "orphan"), _msg("user", "q"), _msg("assistant", "a")]) + assert [(u.content, a.content) for u, a in paired] == [("q", "a")] + + +def test_pairing_of_an_empty_thread_is_empty(): + assert _pair_into_turns([]) == [] + + +# --- Titles ------------------------------------------------------------------ + +def test_title_is_the_first_message_when_it_is_short(): + assert title_from_message("What is zero trust?") == "What is zero trust?" + + +def test_title_is_truncated_with_an_ellipsis(): + title = title_from_message("word " * 100) + assert len(title) <= 60 + assert title.endswith("…") + + +def test_title_collapses_whitespace_and_survives_an_empty_message(): + assert title_from_message("a\n\n b\tc") == "a b c" + assert title_from_message(" ") == "New Chat" + + +# --- Provider wire formats --------------------------------------------------- + +def test_openai_style_providers_replay_history_between_system_and_question(): + messages = build_chat_messages(_request()) + assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"] + assert messages[0]["content"] == "SYSTEM" + assert messages[1]["content"] == "What are the RMF steps?" + assert messages[-1]["content"].endswith("Question: Which comes after categorisation?") + + +def test_anthropic_omits_the_system_message_from_the_list(): + """Anthropic takes `system` as a top-level parameter, not a message.""" + messages = build_anthropic_messages(_request()) + assert [m["role"] for m in messages] == ["user", "assistant", "user"] + assert all(m["role"] != "system" for m in messages) + + +def test_anthropic_history_strictly_alternates(): + roles = [m["role"] for m in build_anthropic_messages(_request())] + assert all(a != b for a, b in zip(roles, roles[1:])) + + +def test_gemini_maps_the_assistant_role_to_model(): + contents = GeminiProvider._build_contents(_request()) + assert [c["role"] for c in contents] == ["user", "model", "user"] + assert contents[1]["parts"] == ["Prepare, categorize, select."] + + +def test_every_provider_sees_the_same_number_of_turns(): + request = _request() + chat = [m for m in build_chat_messages(request) if m["role"] != "system"] + anthropic = build_anthropic_messages(request) + gemini = GeminiProvider._build_contents(request) + assert len(chat) == len(anthropic) == len(gemini) == len(HISTORY) + 1 + + +def test_an_empty_context_sends_the_message_alone(): + """LLM mode and the condenser have no passages to prepend.""" + assert build_user_turn(_request(context="", user_message="hello")) == "hello" + + +def test_context_is_still_framed_as_a_question_when_present(): + turn = build_user_turn(_request(context="PASSAGE", user_message="why?")) + assert turn == "PASSAGE\n\nQuestion: why?" + + +# --- Condenser --------------------------------------------------------------- + +class _Provider: + def __init__(self, answer: str = "", raises: Exception | None = None, delay: float = 0.0): + self._answer = answer + self._raises = raises + self._delay = delay + self.calls = 0 + + async def generate(self, request): + self.calls += 1 + if self._delay: + await asyncio.sleep(self._delay) + if self._raises: + raise self._raises + return SimpleNamespace(answer=self._answer, citations=[], model=request.model) + + async def stream(self, request): + yield self._answer + + async def health_check(self) -> bool: + return True + + +@pytest.fixture +def condense_settings(monkeypatch): + def _apply(**overrides): + settings = Settings(debug=True, **overrides) + monkeypatch.setattr("app.llm.condenser.get_settings", lambda: settings) + return settings + + return _apply + + +async def test_condensing_rewrites_a_follow_up(condense_settings): + condense_settings() + provider = _Provider(answer="How does a policy enforcement point differ from a policy engine?") + + result = await condense_query(provider, "How does it differ?", HISTORY) + + assert result.rewritten is True + assert result.outcome == "ok" + assert result.query.startswith("How does a policy enforcement point") + + +async def test_condensing_is_skipped_without_history(condense_settings): + condense_settings() + provider = _Provider(answer="never used") + + result = await condense_query(provider, "a standalone question", []) + + assert result.query == "a standalone question" + assert result.outcome == "skipped" + # No history means no ambiguity to resolve, so no call is worth its latency. + assert provider.calls == 0 + + +async def test_condensing_can_be_turned_off(condense_settings): + condense_settings(chat_query_rewrite=False) + provider = _Provider(answer="rewrite") + + result = await condense_query(provider, "raw message", HISTORY) + + assert result.query == "raw message" + assert provider.calls == 0 + + +async def test_condensing_falls_back_to_the_raw_message_on_timeout(condense_settings): + condense_settings(chat_query_rewrite_timeout=0.05) + provider = _Provider(answer="too late", delay=5) + + result = await condense_query(provider, "How does it differ?", HISTORY) + + assert result.query == "How does it differ?" + assert result.rewritten is False + assert result.outcome == "timeout" + + +@pytest.mark.parametrize( + "error", [RateLimitError("429"), ProviderUnavailableError("503")] +) +async def test_condensing_falls_back_when_the_provider_errors(condense_settings, error): + condense_settings() + provider = _Provider(raises=error) + + result = await condense_query(provider, "How does it differ?", HISTORY) + + assert result.query == "How does it differ?" + assert result.outcome == "provider_error" + + +async def test_condensing_rejects_an_answer_instead_of_a_query(condense_settings): + """A reply long enough to be an answer means the model ignored the prompt.""" + condense_settings() + provider = _Provider(answer="x" * (MAX_REWRITE_CHARS + 1)) + + result = await condense_query(provider, "raw", HISTORY) + + assert result.query == "raw" + assert result.outcome == "rejected" + + +async def test_condensing_rejects_an_empty_reply(condense_settings): + condense_settings() + result = await condense_query(_Provider(answer=" "), "raw", HISTORY) + assert result.query == "raw" + assert result.outcome == "rejected" + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("plain rewrite", "plain rewrite"), + ('"quoted rewrite"', "quoted rewrite"), + ("Standalone search query: the rewrite", "the rewrite"), + ("Query: the rewrite", "the rewrite"), + ("\n\n the rewrite \n trailing junk", "the rewrite"), + ], +) +def test_clean_reduces_a_reply_to_one_query(raw, expected): + assert _clean(raw) == expected + + +# --- Stream capture ---------------------------------------------------------- + +async def test_stream_capture_records_tokens_for_persistence(): + async def tokens(): + for token in ["one ", "two ", "three"]: + yield token + + capture = StreamCapture() + frames = [frame async for frame in stream_with_citations(tokens(), [], capture=capture)] + + assert capture.text == "one two three" + assert frames[-1] == "data: [DONE]\n\n" + + +async def test_stream_capture_holds_a_partial_answer_when_the_consumer_leaves(): + """This is what makes a `partial` save possible: the text survives the abort.""" + async def tokens(): + for token in ["kept ", "also kept ", "never seen"]: + yield token + + capture = StreamCapture() + stream = stream_with_citations(tokens(), [], capture=capture) + async for _ in stream: + if len(capture.tokens) == 2: + break + await stream.aclose() + + assert capture.text == "kept also kept " + + +def test_conversation_event_is_well_formed_sse(): + frame = conversation_event("abc-123") + assert frame.startswith("event: conversation\ndata: ") + assert frame.endswith("\n\n") + assert '"id": "abc-123"' in frame diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py index 5543f7f..27847bc 100644 --- a/backend/tests/test_health.py +++ b/backend/tests/test_health.py @@ -65,9 +65,41 @@ async def override_db(): yield mock_db class FakeGenerator: - async def generate(self, message, retrieval_result, mode): + # `provider` is what the query condenser reaches for. LLM mode never + # condenses, but the attribute has to exist for the route to import cleanly. + provider = object() + + async def generate(self, message, retrieval_result, mode, history=None): return SimpleNamespace(answer="ok", citations=[], model="test-model") + # Conversation persistence is stubbed out rather than mocked through the + # AsyncMock session: this test is about the auth-and-routing path, and it runs + # in the unit tier where there is no database. The real persistence behaviour + # is covered against PostgreSQL in tests/test_integration_conversations.py. + conversation = SimpleNamespace(id="conv-1", title="New Chat", mode="llm") + + async def fake_create_conversation(db, user_id, **kwargs): + return conversation + + async def fake_append_message(db, conv, **kwargs): + return SimpleNamespace(id="msg-1") + + async def fake_load_history(db, conversation_id, **kwargs): + return [] + + async def fake_save_assistant(conversation_id, **kwargs): + return None + + monkeypatch.setattr( + "app.api.chat.conversation_service.create_conversation", fake_create_conversation + ) + monkeypatch.setattr( + "app.api.chat.conversation_service.append_message", fake_append_message + ) + monkeypatch.setattr( + "app.api.chat.conversation_service.load_history", fake_load_history + ) + monkeypatch.setattr("app.api.chat._save_assistant_message", fake_save_assistant) monkeypatch.setattr("app.api.chat.get_response_generator", lambda: FakeGenerator()) monkeypatch.setattr( "app.api.chat.get_settings", @@ -86,7 +118,11 @@ async def generate(self, message, retrieval_result, mode): app.dependency_overrides.pop(get_db, None) assert response.status_code == 200 - assert response.json()["answer"] == "ok" + body = response.json() + assert body["answer"] == "ok" + # The reply now names the conversation it belongs to, so the client can keep + # streaming into the same thread. + assert body["conversation_id"] == "conv-1" @pytest.mark.asyncio diff --git a/backend/tests/test_integration_conversations.py b/backend/tests/test_integration_conversations.py new file mode 100644 index 0000000..8903c7b --- /dev/null +++ b/backend/tests/test_integration_conversations.py @@ -0,0 +1,741 @@ +""" +Conversation persistence against real PostgreSQL. + +Four things are proved here, all of which are invisible to the mocked unit tier: + + 1. every conversation route is scoped to the caller, and another user's id is + indistinguishable from one that does not exist + 2. prior turns actually reach the provider -- asserted on the message list the + provider is handed, not on the fact that some history was loaded + 3. a condenser timeout degrades to the raw message instead of failing the request + 4. a client that disconnects mid-answer still leaves a `partial` message behind + +Skipped in full when TEST_DATABASE_URL is unset; see conftest.py. +""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import pytest +import pytest_asyncio +from httpx import AsyncClient +from sqlalchemy import select + +from app.core.config import Settings +from app.database.models import Conversation, Message + +# asyncio_mode = auto in pytest.ini handles the async marker. +pytestmark = pytest.mark.integration + + +# --- Ownership isolation ---------------------------------------------------- + +@pytest_asyncio.fixture +async def two_users(make_user): + owner = await make_user("owner@example.com", name="Owner") + intruder = await make_user("intruder@example.com", name="Intruder") + return owner, intruder + + +async def test_list_returns_only_the_callers_conversations( + client, two_users, make_conversation +): + owner, intruder = two_users + await make_conversation(owner.id, title="Owner thread") + await make_conversation(intruder.id, title="Intruder thread") + + response = await client.get("/conversations", cookies=owner.cookies) + + assert response.status_code == 200 + body = response.json() + assert body["total"] == 1 + assert [c["title"] for c in body["items"]] == ["Owner thread"] + + +@pytest.mark.parametrize( + "method,suffix,payload", + [ + ("get", "", None), + ("patch", "", {"title": "renamed"}), + ("delete", "", None), + ], +) +async def test_other_users_conversation_is_404_not_403( + client, two_users, make_conversation, method, suffix, payload +): + """ + A foreign id must be indistinguishable from a missing one. + + 403 would confirm that the id exists and belongs to somebody, which lets an + attacker enumerate real conversation ids without reading a single message. + """ + owner, intruder = two_users + conversation = await make_conversation(owner.id) + + call = getattr(client, method) + kwargs = {"cookies": intruder.cookies} + if payload is not None: + kwargs["json"] = payload + response = await call(f"/conversations/{conversation.id}{suffix}", **kwargs) + + assert response.status_code == 404 + assert response.json()["detail"] == "Conversation not found." + + +async def test_other_users_conversation_survives_a_foreign_delete( + client, db_session, two_users, make_conversation +): + """The 404 above must be a refusal, not a delete that happened to report oddly.""" + owner, intruder = two_users + conversation = await make_conversation(owner.id) + + await client.delete(f"/conversations/{conversation.id}", cookies=intruder.cookies) + + still_there = await db_session.scalar( + select(Conversation).where(Conversation.id == conversation.id) + ) + assert still_there is not None + + +async def test_feedback_on_another_users_message_is_404( + client, db_session, two_users, make_conversation +): + owner, intruder = two_users + conversation = await make_conversation(owner.id, turns=[("q", "a")]) + message = await db_session.scalar( + select(Message).where( + Message.conversation_id == conversation.id, Message.role == "assistant" + ) + ) + + response = await client.patch( + f"/conversations/{conversation.id}/messages/{message.id}/feedback", + json={"feedback": "up"}, + cookies=intruder.cookies, + ) + + assert response.status_code == 404 + await db_session.refresh(message) + assert message.feedback is None + + +async def test_owner_can_read_pin_rename_and_delete(client, two_users, make_conversation): + owner, _ = two_users + conversation = await make_conversation(owner.id, turns=[("first question", "first answer")]) + + detail = await client.get(f"/conversations/{conversation.id}", cookies=owner.cookies) + assert detail.status_code == 200 + assert [m["content"] for m in detail.json()["messages"]] == [ + "first question", "first answer", + ] + + patched = await client.patch( + f"/conversations/{conversation.id}", + json={"title": "Renamed", "pinned": True}, + cookies=owner.cookies, + ) + assert patched.status_code == 200 + assert patched.json()["title"] == "Renamed" + assert patched.json()["pinned"] is True + # A rename must not disturb the mode it did not mention. + assert patched.json()["mode"] == conversation.mode + + deleted = await client.delete(f"/conversations/{conversation.id}", cookies=owner.cookies) + assert deleted.status_code == 204 + + gone = await client.get(f"/conversations/{conversation.id}", cookies=owner.cookies) + assert gone.status_code == 404 + + +async def test_deleting_a_conversation_cascades_to_its_messages( + client, db_session, two_users, make_conversation +): + owner, _ = two_users + conversation = await make_conversation(owner.id, turns=[("q1", "a1"), ("q2", "a2")]) + + await client.delete(f"/conversations/{conversation.id}", cookies=owner.cookies) + + remaining = await db_session.scalars( + select(Message).where(Message.conversation_id == conversation.id) + ) + assert list(remaining) == [] + + +async def test_import_is_scoped_to_the_caller(client, db_session, two_users): + owner, _ = two_users + response = await client.post( + "/conversations/import", + json={ + "conversations": [ + { + "title": "From localStorage", + "mode": "docuquery", + "pinned": True, + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + }, + # Empty threads are the "New Chat" rows the old store created on + # click. They are skipped rather than restored. + {"title": "Never used", "messages": []}, + ] + }, + cookies=owner.cookies, + ) + + assert response.status_code == 200 + assert response.json() == {"imported": 1, "skipped": 1} + + rows = list(await db_session.scalars(select(Conversation))) + assert len(rows) == 1 + assert rows[0].user_id == owner.id + assert rows[0].pinned is True + + +async def test_import_rejects_more_than_two_hundred_conversations(client, two_users): + owner, _ = two_users + payload = { + "conversations": [ + {"title": f"c{i}", "messages": [{"role": "user", "content": "x"}]} + for i in range(201) + ] + } + + response = await client.post("/conversations/import", json=payload, cookies=owner.cookies) + + assert response.status_code == 422 + + +async def test_every_conversation_route_requires_authentication(client, two_users, make_conversation): + owner, _ = two_users + conversation = await make_conversation(owner.id) + + assert (await client.get("/conversations")).status_code == 401 + assert (await client.post("/conversations", json={})).status_code == 401 + assert (await client.get(f"/conversations/{conversation.id}")).status_code == 401 + assert ( + await client.patch(f"/conversations/{conversation.id}", json={"pinned": True}) + ).status_code == 401 + assert (await client.delete(f"/conversations/{conversation.id}")).status_code == 401 + assert ( + await client.post("/conversations/import", json={"conversations": []}) + ).status_code == 401 + + +# --- History reaches the provider ------------------------------------------- + +class RecordingProvider: + """ + A provider that records the LLMRequest it was handed and streams a fixed reply. + + Asserting on `seen` is the point: a test that only checks that history was + loaded would still pass if the provider silently dropped it, which is exactly + the bug this phase fixes. + """ + + def __init__( + self, + reply: str = "recorded answer", + tokens: list[str] | None = None, + token_delay: float = 0.0, + ): + self.seen: list = [] + # Kept apart because the two calls mean different things: `generate` during + # a streaming chat is the query condenser, `stream` is the answer. A single + # list makes "what did the answer prompt see?" ambiguous. + self.generated: list = [] + self.streamed: list = [] + self._reply = reply + self._tokens = tokens + self._token_delay = token_delay + + @property + def last(self): + return self.seen[-1] + + async def generate(self, request): + self.seen.append(request) + self.generated.append(request) + return SimpleNamespace( + answer=self._reply, citations=[], model=request.model, + prompt_tokens=None, completion_tokens=None, + ) + + async def stream(self, request): + self.seen.append(request) + self.streamed.append(request) + for index, token in enumerate(self._tokens or [self._reply]): + # A delay between tokens leaves the route's generator genuinely + # suspended at a yield, which is what a mid-stream disconnect needs. + if index and self._token_delay: + await asyncio.sleep(self._token_delay) + yield token + + async def health_check(self) -> bool: + return True + + +def fake_retrieval_returning(chunk_text: str, sink: list[str]): + """ + A stand-in for run_retrieval_pipeline that records the query it was given. + + It returns one real chunk on purpose. An empty result sends docuquery mode + into NoContextError before the provider is ever called, so a test that used + an empty one would be measuring the guard rather than the query. + """ + from app.retrieval.retrieval_pipeline import ChunkResult, RetrievalResult + + async def _fake(query, db, user_id, top_k, filters): + sink.append(query) + return RetrievalResult( + query=query, + chunks=[ + ChunkResult( + document_id="doc-1", + filename="NIST.SP.800-207.pdf", + page=7, + chunk_index=0, + similarity=0.71, + text=chunk_text, + ) + ], + context=chunk_text, + citations=[], + total_retrieved=1, + ) + + return _fake + + +@pytest.fixture +def stub_chat(monkeypatch): + """ + Point /chat at a recording provider and skip retrieval. + + Retrieval is stubbed to an empty result in `llm` mode, so these tests measure + the conversation plumbing rather than the embedding model -- which the + integration container does not have loaded. + """ + from app.llm.response_generator import ResponseGenerator + + def _install(provider, **setting_overrides): + generator = ResponseGenerator(provider=provider) + monkeypatch.setattr("app.api.chat.get_response_generator", lambda: generator) + settings = Settings(debug=True, **setting_overrides) + monkeypatch.setattr("app.api.chat.get_settings", lambda: settings) + monkeypatch.setattr("app.llm.response_generator.get_settings", lambda: settings) + monkeypatch.setattr("app.llm.condenser.get_settings", lambda: settings) + return generator + + return _install + + +async def test_history_is_replayed_to_the_provider( + client, two_users, make_conversation, stub_chat +): + owner, _ = two_users + conversation = await make_conversation( + owner.id, + mode="llm", + turns=[ + ("What are the RMF steps?", "Prepare, categorize, select, implement."), + ("Who publishes it?", "NIST."), + ], + ) + provider = RecordingProvider() + stub_chat(provider, llm_streaming_enabled=False) + + response = await client.post( + "/chat", + json={ + "message": "Which one comes first?", + "conversation_id": conversation.id, + "mode": "llm", + }, + cookies=owner.cookies, + ) + + assert response.status_code == 200 + request = provider.last + + # Oldest first, strictly alternating, and the live message is NOT duplicated + # into its own history. + assert [(h.role, h.content) for h in request.history] == [ + ("user", "What are the RMF steps?"), + ("assistant", "Prepare, categorize, select, implement."), + ("user", "Who publishes it?"), + ("assistant", "NIST."), + ] + assert request.user_message == "Which one comes first?" + + +async def test_history_is_capped_by_the_turn_limit( + client, two_users, make_conversation, stub_chat +): + owner, _ = two_users + conversation = await make_conversation( + owner.id, mode="llm", + turns=[(f"question {i}", f"answer {i}") for i in range(10)], + ) + provider = RecordingProvider() + stub_chat(provider, llm_streaming_enabled=False, chat_history_turns=2) + + await client.post( + "/chat", + json={"message": "and now?", "conversation_id": conversation.id, "mode": "llm"}, + cookies=owner.cookies, + ) + + # The two most recent turns, oldest of them first -- the oldest are dropped. + assert [h.content for h in provider.last.history] == [ + "question 8", "answer 8", "question 9", "answer 9", + ] + + +async def test_history_budget_drops_whole_turns_from_the_oldest_end( + client, two_users, make_conversation, stub_chat +): + """ + The budget must never cut inside a turn: a question without its answer, or an + answer without its question, is worse context than one turn fewer. + """ + owner, _ = two_users + long_answer = "x" * 2000 # ~500 estimated tokens + conversation = await make_conversation( + owner.id, mode="llm", + turns=[("old question", long_answer), ("recent question", "short answer")], + ) + provider = RecordingProvider() + stub_chat(provider, llm_streaming_enabled=False, chat_history_max_tokens=100) + + await client.post( + "/chat", + json={"message": "next", "conversation_id": conversation.id, "mode": "llm"}, + cookies=owner.cookies, + ) + + assert [h.content for h in provider.last.history] == ["recent question", "short answer"] + + +async def test_first_message_of_a_conversation_has_no_history( + client, two_users, stub_chat +): + owner, _ = two_users + provider = RecordingProvider() + stub_chat(provider, llm_streaming_enabled=False) + + response = await client.post( + "/chat", json={"message": "opening question", "mode": "llm"}, cookies=owner.cookies + ) + + assert response.status_code == 200 + assert provider.last.history == [] + assert response.json()["conversation_id"] + + +# --- /chat conversation lifecycle ------------------------------------------- + +async def test_new_chat_emits_the_conversation_event_first( + client, db_session, two_users, stub_chat +): + owner, _ = two_users + stub_chat(RecordingProvider(tokens=["one ", "two"]), llm_streaming_enabled=True) + + async with client.stream( + "POST", "/chat", json={"message": "hello", "mode": "llm"}, cookies=owner.cookies + ) as response: + assert response.status_code == 200 + frames = [line async for line in response.aiter_lines()] + + # First non-empty line of the stream, before any token. + head = [line for line in frames if line][:2] + assert head[0] == "event: conversation" + conversation_id = json.loads(head[1].removeprefix("data: "))["id"] + + stored = await db_session.scalar( + select(Conversation).where(Conversation.id == conversation_id) + ) + assert stored is not None + assert stored.user_id == owner.id + + +async def test_existing_conversation_does_not_re_emit_the_conversation_event( + client, two_users, make_conversation, stub_chat +): + owner, _ = two_users + conversation = await make_conversation(owner.id, mode="llm") + stub_chat(RecordingProvider(tokens=["hi"]), llm_streaming_enabled=True) + + async with client.stream( + "POST", "/chat", + json={"message": "hello", "conversation_id": conversation.id, "mode": "llm"}, + cookies=owner.cookies, + ) as response: + frames = [line async for line in response.aiter_lines()] + + assert "event: conversation" not in frames + + +async def test_unknown_conversation_id_is_404_before_streaming(client, two_users, stub_chat): + owner, _ = two_users + stub_chat(RecordingProvider(), llm_streaming_enabled=True) + + response = await client.post( + "/chat", + json={"message": "hello", "conversation_id": "does-not-exist", "mode": "llm"}, + cookies=owner.cookies, + ) + + # A real status code, not a 200 whose body happens to carry an SSE error frame. + assert response.status_code == 404 + assert "text/event-stream" not in response.headers.get("content-type", "") + + +async def test_foreign_conversation_id_is_404_before_streaming( + client, two_users, make_conversation, stub_chat +): + owner, intruder = two_users + conversation = await make_conversation(owner.id, mode="llm") + stub_chat(RecordingProvider(), llm_streaming_enabled=True) + + response = await client.post( + "/chat", + json={"message": "hello", "conversation_id": conversation.id, "mode": "llm"}, + cookies=intruder.cookies, + ) + + assert response.status_code == 404 + + +async def test_both_turns_are_persisted_and_the_title_comes_from_the_first_message( + client, db_session, two_users, stub_chat +): + owner, _ = two_users + stub_chat(RecordingProvider(reply="the answer"), llm_streaming_enabled=False) + + response = await client.post( + "/chat", + json={"message": "What is a policy enforcement point?", "mode": "llm"}, + cookies=owner.cookies, + ) + conversation_id = response.json()["conversation_id"] + + rows = list( + await db_session.scalars( + select(Message) + .where(Message.conversation_id == conversation_id) + .order_by(Message.created_at) + ) + ) + assert [(m.role, m.content, m.status) for m in rows] == [ + ("user", "What is a policy enforcement point?", "complete"), + ("assistant", "the answer", "complete"), + ] + + conversation = await db_session.scalar( + select(Conversation).where(Conversation.id == conversation_id) + ) + # Title is the first user message, truncated -- no extra LLM call was made. + assert conversation.title == "What is a policy enforcement point?" + + +async def test_regenerate_replaces_the_answer_instead_of_appending_a_turn( + client, db_session, two_users, make_conversation, stub_chat +): + owner, _ = two_users + conversation = await make_conversation( + owner.id, mode="llm", turns=[("the question", "the first answer")] + ) + stub_chat(RecordingProvider(reply="the second answer"), llm_streaming_enabled=False) + + response = await client.post( + "/chat", + json={"conversation_id": conversation.id, "mode": "llm", "regenerate": True}, + cookies=owner.cookies, + ) + + assert response.status_code == 200 + rows = list( + await db_session.scalars( + select(Message) + .where(Message.conversation_id == conversation.id) + .order_by(Message.created_at) + ) + ) + assert [(m.role, m.content) for m in rows] == [ + ("user", "the question"), + ("assistant", "the second answer"), + ] + + +# --- Condenser fallback ------------------------------------------------------ + +class HangingCondenserProvider(RecordingProvider): + """ + Times out on the condensing call, answers the real one. + + Condensing is the only `generate` during a streaming chat, so sleeping there + and streaming normally isolates the timeout to the step under test. + """ + + async def generate(self, request): + self.seen.append(request) + await asyncio.sleep(3600) + + +async def test_condenser_timeout_falls_back_to_the_raw_message( + client, two_users, make_conversation, stub_chat, monkeypatch, caplog +): + owner, _ = two_users + conversation = await make_conversation( + owner.id, + mode="docuquery", + turns=[("What is a policy enforcement point?", "It enforces access decisions.")], + ) + + provider = HangingCondenserProvider(tokens=["fallback answer"]) + stub_chat( + provider, + llm_streaming_enabled=False, + chat_query_rewrite_timeout=0.05, + chat_query_rewrite=True, + ) + + retrieved: list[str] = [] + monkeypatch.setattr( + "app.api.chat.run_retrieval_pipeline", + fake_retrieval_returning("A policy enforcement point enforces access decisions.", retrieved), + ) + + # The answer call must not also hang, so swap the provider's generate back + # once condensing has had its turn. + original_generate = provider.generate + + async def generate_once_then_answer(request): + if not provider.seen: + return await original_generate(request) + provider.seen.append(request) + return SimpleNamespace( + answer="fallback answer", citations=[], model=request.model, + prompt_tokens=None, completion_tokens=None, + ) + + provider.generate = generate_once_then_answer + + response = await client.post( + "/chat", + json={ + "message": "How does it differ from the policy engine?", + "conversation_id": conversation.id, + "mode": "docuquery", + }, + cookies=owner.cookies, + ) + + assert response.status_code == 200 + # Retrieval ran on the user's literal words, because the rewrite timed out. + assert retrieved == ["How does it differ from the policy engine?"] + assert response.json()["retrieval_query"] == "How does it differ from the policy engine?" + + +async def test_condensed_query_is_used_for_retrieval_but_not_for_the_answer( + client, two_users, make_conversation, stub_chat, monkeypatch +): + owner, _ = two_users + conversation = await make_conversation( + owner.id, + mode="docuquery", + turns=[("What is a policy enforcement point?", "It enforces access decisions.")], + ) + + rewrite = "How does a policy enforcement point differ from a policy engine?" + + # `reply` is what the condenser's generate() returns; `tokens` is the answer. + provider = RecordingProvider(reply=rewrite, tokens=["grounded answer"]) + stub_chat(provider, llm_streaming_enabled=True, chat_query_rewrite=True) + + retrieved: list[str] = [] + monkeypatch.setattr( + "app.api.chat.run_retrieval_pipeline", + fake_retrieval_returning("A policy enforcement point enforces access decisions.", retrieved), + ) + + original = "How does it differ from the policy engine?" + async with client.stream( + "POST", "/chat", + json={"message": original, "conversation_id": conversation.id, "mode": "docuquery"}, + cookies=owner.cookies, + ) as response: + [line async for line in response.aiter_lines()] + + # Retrieval saw the standalone rewrite ... + assert retrieved == [rewrite] + # ... while the answer prompt still saw what the user actually typed. + assert len(provider.streamed) == 1 + assert provider.streamed[0].user_message == original + # And the condenser was the one that got the transcript. + assert len(provider.generated) == 1 + assert original in provider.generated[0].user_message + + +# --- Partial save on disconnect --------------------------------------------- + +async def test_client_disconnect_saves_a_partial_message( + live_server, client, db_session, two_users, make_conversation, stub_chat +): + """ + Abandoning the stream must still leave behind the text the user saw. + + This one needs a real socket. httpx's ASGITransport buffers the whole + response body before handing back a single line, so the application always + runs to completion under it and a mid-stream disconnect cannot happen at all + -- the route would record an ordinary `complete` message and the test would + pass while proving nothing. + """ + owner, _ = two_users + conversation = await make_conversation(owner.id, mode="llm") + # Without a delay the whole answer is produced before the client can walk + # away, even over a real connection. + stub_chat( + RecordingProvider(tokens=["first ", "second ", "third"], token_delay=0.4), + llm_streaming_enabled=True, + ) + + async with AsyncClient(base_url=live_server, timeout=10.0) as http: + async with http.stream( + "POST", "/chat", + json={ + "message": "tell me a long story", + "conversation_id": conversation.id, + "mode": "llm", + }, + cookies=owner.cookies, + ) as response: + assert response.status_code == 200 + async for line in response.aiter_lines(): + if line.startswith("event: token"): + break # walk away mid-answer + # Leaving the context closes the socket; the server sees http.disconnect. + + stored = None + # The save runs on a task the disconnect does not cancel, from a session of + # its own. Poll rather than sleep a fixed amount. + for _ in range(100): + await asyncio.sleep(0.05) + stored = await db_session.scalar( + select(Message).where( + Message.conversation_id == conversation.id, Message.role == "assistant" + ) + ) + if stored is not None: + break + + assert stored is not None, "no assistant message was written after the disconnect" + assert stored.status == "partial" + # Only what had actually been generated when the client left. + assert stored.content == "first " + assert stored.content != "first second third" diff --git a/frontend/app/(dashboard)/agents/page.tsx b/frontend/app/(dashboard)/agents/page.tsx index 5d6b9df..c5c34e5 100644 --- a/frontend/app/(dashboard)/agents/page.tsx +++ b/frontend/app/(dashboard)/agents/page.tsx @@ -16,12 +16,13 @@ export default function AgentsPage() { description: (["Ask questions grounded in your uploaded documents.", "General AI conversations and open-ended questions.", "Combine document context with AI reasoning."] as const)[index], })) const { selectedAgent, setSelectedAgent } = useUIStore() - const setMode = useChatStore(s => s.setMode) + // Picking an agent chooses the mode the next chat starts in. + const setDraftMode = useChatStore(s => s.setDraftMode) const chooseAgent = (agent: typeof agents[number]) => { const selecting = selectedAgent?.id !== agent.id setSelectedAgent(selecting ? agent : null) - if (selecting && ["docuquery", "llm", "hybrid"].includes(agent.id)) setMode(agent.id as ChatMode) + if (selecting && ["docuquery", "llm", "hybrid"].includes(agent.id)) setDraftMode(agent.id as ChatMode) } return ( diff --git a/frontend/app/(dashboard)/chat/[id]/page.tsx b/frontend/app/(dashboard)/chat/[id]/page.tsx index 3127d17..445f8a1 100644 --- a/frontend/app/(dashboard)/chat/[id]/page.tsx +++ b/frontend/app/(dashboard)/chat/[id]/page.tsx @@ -1,27 +1,18 @@ "use client" -import { use, useEffect } from "react" -import { useRouter } from "next/navigation" -import { useChatStore } from "@/stores/chat.store" import { ChatWindow } from "@/components/chat/ChatWindow" -export default function ChatDetailPage({ params }: { params: Promise<{ id: string }> }) { - // Next.js 16 passes params as a Promise; client pages unwrap it with React.use(). - const { id } = use(params) - const router = useRouter() - const conversations = useChatStore(s => s.conversations) - const setActiveId = useChatStore(s => s.setActiveId) - const known = conversations.some(conversation => conversation.id === id) - - useEffect(() => { - // An id that isn't in this account's history (a stale link, or a chat deleted in - // another tab) falls back to /chat rather than showing an empty conversation. - if (!known) { - router.replace("/chat") - return - } - // setActiveId also switches activeMode to this conversation's mode. - setActiveId(id) - }, [id, known, router, setActiveId]) - +/** + * An existing conversation. + * + * Renders the same component as /chat and reads nothing from `params`. + * ChatWindow takes the conversation id from `usePathname()` instead, so that a + * new chat can adopt its server id mid-stream via `window.history.replaceState` + * -- which changes the URL without a route transition, and therefore without + * unmounting the answer being streamed. Both routes render the same tree, so it + * does not matter which one the browser arrived through. + * + * `params` is a Promise in Next.js 16; nothing here needs to await it. + */ +export default function ChatDetailPage() { return } diff --git a/frontend/app/(dashboard)/chat/page.tsx b/frontend/app/(dashboard)/chat/page.tsx index a98ffbc..8726e0d 100644 --- a/frontend/app/(dashboard)/chat/page.tsx +++ b/frontend/app/(dashboard)/chat/page.tsx @@ -1,5 +1,7 @@ +"use client" import { ChatWindow } from "@/components/chat/ChatWindow" +/** A new, unsaved chat. The server creates the conversation on the first message. */ export default function ChatPage() { return } diff --git a/frontend/app/(dashboard)/dashboard/page.tsx b/frontend/app/(dashboard)/dashboard/page.tsx index a114559..212ce3c 100644 --- a/frontend/app/(dashboard)/dashboard/page.tsx +++ b/frontend/app/(dashboard)/dashboard/page.tsx @@ -8,12 +8,13 @@ import { ArrowRight, FileText, MessageSquare, MessageSquarePlus, Upload, type Lu import { useAuthStore } from "@/stores/auth.store" import { useFileStore } from "@/stores/file.store" import { useChatStore } from "@/stores/chat.store" +import { useConversationList } from "@/hooks/useConversations" import { Button } from "@/components/ui/Button" import { Badge } from "@/components/ui/Badge" import { Skeleton } from "@/components/ui/Skeleton" import { FileStatus } from "@/components/file-upload/FileUploadZone" import { CHAT_MODE_ICONS } from "@/components/chat/modeIcons" -import { CHAT_MODE_META, type ChatMode, type Conversation } from "@/types" +import { CHAT_MODE_META, type ChatMode } from "@/types" import { cn, formatBytes, formatRelative } from "@/lib/utils" import { transition } from "@/lib/motion" @@ -29,9 +30,9 @@ export default function DashboardPage() { const user = useAuthStore(s => s.user) const files = useFileStore(s => s.files) const loadFiles = useFileStore(s => s.loadFiles) - const conversations = useChatStore(s => s.conversations) - const addConversation = useChatStore(s => s.addConversation) - const activeMode = useChatStore(s => s.activeMode) + const { conversations } = useConversationList() + const draftMode = useChatStore(s => s.draftMode) + const setDraftMode = useChatStore(s => s.setDraftMode) const router = useRouter() const [filesLoaded, setFilesLoaded] = useState(false) @@ -58,13 +59,10 @@ export default function DashboardPage() { .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)) .slice(0, RECENT_LIMIT) - const startChat = (mode: ChatMode = activeMode) => { - const now = new Date().toISOString() - const conversation: Conversation = { - id: crypto.randomUUID(), title: "New Chat", messages: [], mode, - pinned: false, createdAt: now, updatedAt: now, - } - addConversation(conversation) + // The conversation is created by the server on the first message, so starting + // a chat is just choosing the mode and opening an empty one. + const startChat = (mode: ChatMode = draftMode) => { + setDraftMode(mode) router.push("/chat") } @@ -189,7 +187,7 @@ export default function DashboardPage() { {MODES.map(mode => { const meta = CHAT_MODE_META[mode] const ModeIcon = CHAT_MODE_ICONS[mode] - const current = mode === activeMode + const current = mode === draftMode return (
  • diff --git a/frontend/components/chat/ChatInput.tsx b/frontend/components/chat/ChatInput.tsx index 1de155b..3386410 100644 --- a/frontend/components/chat/ChatInput.tsx +++ b/frontend/components/chat/ChatInput.tsx @@ -2,17 +2,16 @@ import { useState, useRef, useCallback } from "react" import { motion, AnimatePresence } from "framer-motion" import { ArrowUp, Loader2, Mic, MicOff, Paperclip, X } from "lucide-react" -import { useChatStore } from "@/stores/chat.store" +import { useChatStream } from "@/hooks/useChatStream" +import { useActiveConversationId } from "@/hooks/useConversations" import { useFileStore } from "@/stores/file.store" import { useVoice } from "@/hooks/useVoice" import { Button } from "@/components/ui/Button" import { Tooltip } from "@/components/ui/Tooltip" import { ModeIndicator } from "./ModeIndicator" -import { cn, formatBytes, generateId } from "@/lib/utils" +import { cn, formatBytes } from "@/lib/utils" import { transition } from "@/lib/motion" -import { CHAT_MODE_META } from "@/types" -import type { Conversation } from "@/types" -import { getDefaultChatMode } from "@/stores/settings.store" +import { CHAT_MODE_META, type ChatMode } from "@/types" const MODE_PLACEHOLDERS = { docuquery: "Ask about your uploaded documents…", @@ -20,13 +19,15 @@ const MODE_PLACEHOLDERS = { hybrid: "Ask anything — documents first, then AI knowledge…", } -export function ChatInput() { +/** `mode` comes from the open conversation, or the draft mode for a new one. */ +export function ChatInput({ mode }: { mode: ChatMode }) { const [text, setText] = useState("") const [pendingFiles, setPendingFiles] = useState([]) const textareaRef = useRef(null) const fileInputRef = useRef(null) - const { sendMessage, isStreaming, activeId, addConversation, activeMode } = useChatStore() + const activeId = useActiveConversationId() + const { send, isStreaming } = useChatStream() const { addFile } = useFileStore() const { voiceState, start: startVoice, stop: stopVoice } = useVoice((transcript) => { @@ -39,28 +40,16 @@ export function ChatInput() { const content = text.trim() if (!content || isStreaming) return - if (!activeId) { - const conv: Conversation = { - id: generateId(), - title: "New Chat", - messages: [], - mode: getDefaultChatMode(), - pinned: false, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - } - addConversation(conv) - } - setText("") setPendingFiles([]) for (const f of pendingFiles) addFile(f) - // Small delay to allow the store to register the new conversation - await new Promise(r => setTimeout(r, 0)) - await sendMessage(content) + // No conversation is created up front any more. Sending without an id makes + // the server create one and announce it on the stream, which is also what + // puts the id in the URL. + await send(content, mode, activeId) textareaRef.current?.style.setProperty("height", "auto") - }, [text, isStreaming, activeId, pendingFiles, addFile, addConversation, sendMessage]) + }, [text, isStreaming, activeId, mode, pendingFiles, addFile, send]) const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { @@ -87,9 +76,9 @@ export function ChatInput() {
    {/* Mode indicator row */}
    - + - {CHAT_MODE_META[activeMode].description} + {CHAT_MODE_META[mode].description}
    @@ -156,7 +145,7 @@ export function ChatInput() { value={text} onChange={handleTextChange} onKeyDown={handleKeyDown} - placeholder={MODE_PLACEHOLDERS[activeMode]} + placeholder={MODE_PLACEHOLDERS[mode]} rows={1} disabled={isStreaming} className="max-h-50 min-h-8 flex-1 resize-none bg-transparent px-1.5 py-1.5 text-body-lg text-fg placeholder:text-fg-subtle focus:outline-none disabled:opacity-50" diff --git a/frontend/components/chat/ChatWindow.tsx b/frontend/components/chat/ChatWindow.tsx index 78083e8..80d8e60 100644 --- a/frontend/components/chat/ChatWindow.tsx +++ b/frontend/components/chat/ChatWindow.tsx @@ -1,15 +1,17 @@ "use client" +import { useMemo } from "react" import { motion } from "framer-motion" import { useChatStore } from "@/stores/chat.store" +import { useActiveConversationId, useConversation } from "@/hooks/useConversations" +import { useChatStream } from "@/hooks/useChatStream" import { MessageBubble } from "./MessageBubble" import { ChatInput } from "./ChatInput" import { CHAT_MODE_ICONS } from "./modeIcons" import { useAutoScroll } from "@/hooks/useAutoScroll" import { CHAT_MODE_META } from "@/types" -import { generateId } from "@/lib/utils" +import type { ChatMode, Message } from "@/types" import { transition } from "@/lib/motion" -import type { Conversation } from "@/types" -import { getDefaultChatMode } from "@/stores/settings.store" +import { Skeleton } from "@/components/ui/Skeleton" const SUGGESTIONS_BY_MODE = { docuquery: [ @@ -38,48 +40,101 @@ const MODE_DETAILS = { hybrid: "Use your documents as context, then let AI reason through answers and next steps.", } +// Ids for the two bubbles that exist only while an answer is streaming. They are +// never sent anywhere: the server assigns real ids, which arrive with the refetch. +const PENDING_USER_ID = "pending-user" +const PENDING_ASSISTANT_ID = "pending-assistant" + export function ChatWindow() { - const { conversations, activeId, isStreaming, addConversation, sendMessage } = useChatStore() - const bottomRef = useAutoScroll() - const active = conversations.find(c => c.id === activeId) - - const handleSuggestion = async (text: string) => { - if (!activeId) { - const conv: Conversation = { - id: generateId(), - title: text.slice(0, 52).trim(), - messages: [], - mode: getDefaultChatMode(), - pinned: false, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - } - addConversation(conv) - // Let the store flush before sending - await new Promise(r => setTimeout(r, 0)) + const activeId = useActiveConversationId() + const { data: conversation, isLoading } = useConversation(activeId) + const { send } = useChatStream() + + const draftMode = useChatStore(s => s.draftMode) + const streamConversationId = useChatStore(s => s.streamConversationId) + const streamUserText = useChatStore(s => s.streamUserText) + const streamText = useChatStore(s => s.streamText) + const streamCitations = useChatStore(s => s.streamCitations) + const streamStatus = useChatStore(s => s.streamStatus) + const streamIsRetry = useChatStore(s => s.streamIsRetry) + + const mode: ChatMode = conversation?.mode ?? draftMode + + // The buffer belongs to this thread when the ids match, and also when neither + // has one yet -- a brand-new chat being answered in this very window, before + // the server has said what its id is. Requiring both to be null matters: if + // you navigate into another conversation while a new chat is still streaming, + // its tokens must not appear in the thread you just opened. + const streamingHere = + streamStatus !== "idle" && + (streamConversationId === activeId || + (streamConversationId === null && activeId === null)) + + const messages = useMemo(() => { + const stored = conversation?.messages ?? [] + if (!streamingHere) return stored + + const pending: Message[] = [] + // A retry re-answers a question already on screen, so only the reply is new. + if (!streamIsRetry && streamUserText) { + pending.push({ + id: PENDING_USER_ID, + role: "user", + content: streamUserText, + status: "done", + timestamp: new Date().toISOString(), + }) } - await sendMessage(text) + pending.push({ + id: PENDING_ASSISTANT_ID, + role: "assistant", + content: streamText, + status: streamStatus === "error" ? "error" : "streaming", + timestamp: new Date().toISOString(), + citations: streamCitations, + }) + + // On a retry the stored answer is about to be replaced server-side; drop it + // here too, or the old and new answers sit side by side while it streams. + const base = streamIsRetry + ? stored.slice(0, stored.findLastIndex(m => m.role === "assistant")) + : stored + return [...base, ...pending] + }, [conversation, streamingHere, streamIsRetry, streamUserText, streamText, streamStatus, streamCitations]) + + const bottomRef = useAutoScroll(messages.length, streamText) + + const handleSuggestion = (text: string) => { + void send(text, mode, activeId) } + const showWelcome = !activeId && messages.length === 0 + return (
    {/* Messages */}
    - {!active || active.messages.length === 0 ? ( - + {showWelcome ? ( + + ) : isLoading && messages.length === 0 ? ( + + ) : messages.length === 0 ? ( + ) : (
    - {active.messages.map((msg, i) => ( + {messages.map((msg, i) => ( ))}
    @@ -90,18 +145,28 @@ export function ChatWindow() { {/* Input */}
    - +
    ) } -function WelcomeScreen({ onSuggestion }: { onSuggestion: (t: string) => void }) { - const { activeMode } = useChatStore() - const meta = CHAT_MODE_META[activeMode] - const ModeIcon = CHAT_MODE_ICONS[activeMode] - const suggestions = SUGGESTIONS_BY_MODE[activeMode] +function LoadingTranscript() { + return ( +
    +

    Loading conversation…

    + + + +
    + ) +} + +function WelcomeScreen({ mode, onSuggestion }: { mode: ChatMode; onSuggestion: (t: string) => void }) { + const meta = CHAT_MODE_META[mode] + const ModeIcon = CHAT_MODE_ICONS[mode] + const suggestions = SUGGESTIONS_BY_MODE[mode] return (
    @@ -117,7 +182,7 @@ function WelcomeScreen({ onSuggestion }: { onSuggestion: (t: string) => void })

    {meta.label} mode

    {meta.description}

    -

    {MODE_DETAILS[activeMode]}

    +

    {MODE_DETAILS[mode]}

    {suggestions.map(s => ( diff --git a/frontend/components/chat/MessageBubble.tsx b/frontend/components/chat/MessageBubble.tsx index 6e11762..aff500d 100644 --- a/frontend/components/chat/MessageBubble.tsx +++ b/frontend/components/chat/MessageBubble.tsx @@ -3,30 +3,47 @@ import { motion } from "framer-motion" import ReactMarkdown from "react-markdown" import remarkGfm from "remark-gfm" import { AlertCircle, Check, Copy, FileSearch, RefreshCw, ThumbsDown, ThumbsUp } from "lucide-react" -import type { Message } from "@/types" +import type { ChatMode, Message } from "@/types" import { Tooltip } from "@/components/ui/Tooltip" import { useCopy } from "@/hooks/useCopy" -import { useChatStore } from "@/stores/chat.store" +import { useMessageFeedback } from "@/hooks/useConversations" +import { useChatStream } from "@/hooks/useChatStream" import { useSettingsStore } from "@/stores/settings.store" import { formatTime, cn } from "@/lib/utils" import { transition } from "@/lib/motion" import { ToolCallDisplay } from "./ToolCallDisplay" import { CitationList } from "./CitationCard" -export function MessageBubble({ message, isLast }: { message: Message; isLast: boolean }) { +export function MessageBubble({ message, isLast, conversationId, mode }: { + message: Message + isLast: boolean + /** Null only on a brand-new chat that has not been saved yet. */ + conversationId: string | null + mode: ChatMode +}) { const { copy, copied } = useCopy() - const retryLast = useChatStore(s => s.retryLast) - const setFeedback = useChatStore(s => s.setFeedback) - const activeId = useChatStore(s => s.activeId) + const { retry } = useChatStream() + const feedbackMutation = useMessageFeedback(conversationId) const showCitations = useSettingsStore(s => s.showCitations) const isUser = message.role === "user" const streaming = message.status === "streaming" + const partial = message.status === "partial" const hasCitations = !isUser && (message.citations?.length ?? 0) > 0 + // Actions need a row the server knows about. While an answer is still + // streaming it has no id yet, so they stay hidden rather than 404. + const persisted = !!conversationId && !message.id.startsWith("pending-") const handleFeedback = (fb: "up" | "down") => { - if (!activeId) return - const next = message.feedback === fb ? null : fb - setFeedback(activeId, message.id, next) + if (!persisted) return + feedbackMutation.mutate({ + messageId: message.id, + // Clicking the same thumb again clears it. + feedback: message.feedback === fb ? null : fb, + }) + } + + const handleRetry = () => { + if (conversationId) void retry(conversationId, mode) } if (isUser) { @@ -89,7 +106,7 @@ export function MessageBubble({ message, isLast }: { message: Message; isLast: b )} {/* Sources are set apart from the generated text in their own bordered list */} - {showCitations && hasCitations && message.status === "done" && ( + {showCitations && hasCitations && (message.status === "done" || partial) && ( )} @@ -100,7 +117,16 @@ export function MessageBubble({ message, isLast }: { message: Message; isLast: b

    )} - {message.status === "done" && ( + {/* The server marks an answer partial when the browser went away while it + was still being written. The text is real, just unfinished. */} + {partial && ( +

    +

    + )} + + {(message.status === "done" || partial) && persisted && (