diff --git a/.env.example b/.env.example index 66f49d8..a186e4d 100644 --- a/.env.example +++ b/.env.example @@ -47,10 +47,6 @@ RA_LLM__MODEL=auto RA_LLM__BASE_URL=http://localhost:11434/v1 RA_LLM__API_KEY=ollama -# Schema-enforced LLM output (native structured outputs). Disable only for -# backends that cannot honor response schemas; the legacy JSON-repair path -# is then used instead. -# RA_LLM__STRUCTURED_OUTPUTS=false # OLLAMA_API_KEY=ollama # Synthesis — see config/ollama_models.yaml for per-model hints diff --git a/config/default.yaml b/config/default.yaml index d9fe940..b11b0a9 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -4,7 +4,6 @@ llm: base_url: http://localhost:11434 temperature: 0.2 timeout_seconds: 120 - structured_outputs: true embedding: model: BAAI/bge-small-en-v1.5 @@ -145,7 +144,5 @@ memory: synthesis: llm_mode: auto max_llm_papers: 3 - extraction_max_retries: 0 - collective_max_retries: 0 concurrency: 2 circuit_breaker_failures: 2 diff --git a/docs/_analysis/config-inventory.md b/docs/_analysis/config-inventory.md index f1a220c..93b6f19 100644 --- a/docs/_analysis/config-inventory.md +++ b/docs/_analysis/config-inventory.md @@ -199,8 +199,6 @@ Source: `src/config/settings.py`, `config/*.yaml`, `.env.example`, `src/config/r | `llm_mode` | `"auto"` | | | `llm_enabled` | `False` | Resolved at pipeline start | | `max_llm_papers` | `3` | May be overridden by Ollama catalog hints | -| `extraction_max_retries` | `0` | | -| `collective_max_retries` | `0` | | | `concurrency` | `2` | | | `circuit_breaker_failures` | `2` | | diff --git a/docs/_analysis/llm-resolution-tree.md b/docs/_analysis/llm-resolution-tree.md index 4f84e88..ffb76a9 100644 --- a/docs/_analysis/llm-resolution-tree.md +++ b/docs/_analysis/llm-resolution-tree.md @@ -176,16 +176,16 @@ flowchart TD RESOLVED[ctx.config after resolve_effective_settings] RESOLVED --> QE{query_expansion.llm_enabled?} - QE -->|yes| QE_AGENT[AgentFactory EXPANSION stream_agent_text] + QE -->|yes| QE_AGENT[run_structured EXPANSION] QE -->|no| QE_SKIP[heuristics only] RESOLVED --> SY{synthesis.llm_enabled?} - SY -->|yes| SY_A[AgentFactory EXTRACTION up to max_llm_papers] - SY_A --> SY_B[AgentFactory SYNTHESIS collective] + SY -->|yes| SY_A[try_run_structured EXTRACTION up to max_llm_papers] + SY_A --> SY_B[try_run_structured SYNTHESIS collective] SY -->|no| SY_H[heuristic extraction + synthesis] RESOLVED --> GA{synthesis.llm_enabled?} - GA -->|yes| GA_AGENT[AgentFactory GAP_ANALYSIS structured response] + GA -->|yes| GA_AGENT[try_run_structured GAP_ANALYSIS] GA -->|no| GA_H[heuristic from synthesis fields] ``` diff --git a/docs/_analysis/test-behavior-index.md b/docs/_analysis/test-behavior-index.md index da41008..5a3cfba 100644 --- a/docs/_analysis/test-behavior-index.md +++ b/docs/_analysis/test-behavior-index.md @@ -1,6 +1,6 @@ # Test Behavior Index -Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/development/testing.md`. +Source: all files matching `tests/test_*.py`. Internal reference for `docs/development/testing.md`. ## Summary by domain @@ -12,7 +12,7 @@ Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/de | Retrieval providers | `test_providers.py` | | Embeddings | `test_embeddings.py` | | Reporting / export | `test_reporting.py`, `test_export.py` | -| LLM layer | `test_llm_providers.py`, `test_graceful_response_handling.py` | +| LLM layer | `test_llm_providers.py`, `test_structured_outputs.py` | | Orchestrator / degradation | `test_json_parsing_bug_exploration.py`, `test_json_parsing_preservation.py` | | CLI / interactive | `test_main_mode_detection.py`, `test_interactive_mode.py`, `test_complete_workflow.py`, `test_input_handler.py`, `test_message_formatting.py`, `test_signal_handling.py`, `test_interactive_filters.py` | | Memory | `test_memory.py` | @@ -216,7 +216,7 @@ Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/de | | | |---|---| | **Modules** | `src.analysis.synthesis`, `src.analysis.gap_analysis`, `src.core.stage_recovery` | -| **Mocks** | `MagicMock(EnhancedResponseHandler)`; `patch create_llm_agent` | +| **Mocks** | `patch src.models.structured.try_run_structured` (AsyncMock) | | Class | Behavior | |-------|----------| @@ -305,28 +305,6 @@ Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/de --- -### `test_graceful_response_handling.py` - -| | | -|---|---| -| **Modules** | `src.utils.{response_models,retry_manager,quality_monitor,enhanced_validation,content_quality,model_adaptation,fallback_processing}` | - -| Class | Behavior | -|-------|----------| -| `TestRetryManager` | Retry rules, prompt enhancement | -| `TestQualityMonitor` | Success/failure recording | -| `TestEnhancedValidation` | Retry strategy mapping | -| `TestContentQuality` | Empty/insufficient/incomplete analysis detection | -| `TestQueryAnalyzer` | Query broadening suggestions | -| `TestRelevanceScorer` | Paper relevance ordering | -| `TestJSONProcessing` | Extract, parse errors, validation | -| `TestModelAdaptation` | GPT/Claude detection, markdown stripping | -| `TestFallbackProcessing` | Unstructured text → structured fallback | - -**Note:** Does not exercise `EnhancedResponseHandler` end-to-end. - ---- - ### `test_json_parsing_bug_exploration.py` | | | @@ -490,7 +468,6 @@ Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/de |-----|--------| | Query understanding | No dedicated unit test file | | API routes | Only scaffold test in `test_phase3_extensibility.py` | -| `EnhancedResponseHandler` | Subcomponents tested, not end-to-end | | Live LLM integration | All LLM tests mock Pydantic AI | | Subprocess tests | `@pytest.mark.slow`; may skip in CI | diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index e80eda6..24a1fe4 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -30,7 +30,6 @@ Boolean env values accept standard truthy strings (`true`, `1`, `yes`). | Variable | Default | Description | |----------|---------|-------------| | `RA_LLM__PROVIDER` | `ollama` | `ollama`, `openai`, or `anthropic` | -| `RA_LLM__STRUCTURED_OUTPUTS` | `true` | Schema-enforced LLM output; disable to use the legacy JSON-repair path | | `RA_LLM__MODEL` | `auto` | Model name; `auto` selects from `config/ollama_models.yaml` (Ollama only) | | `RA_LLM__BASE_URL` | `http://localhost:11434` | API base URL (Ollama OpenAI-compatible endpoint) | diff --git a/docs/configuration/yaml-reference.md b/docs/configuration/yaml-reference.md index cc4bb1e..f60bcab 100644 --- a/docs/configuration/yaml-reference.md +++ b/docs/configuration/yaml-reference.md @@ -183,8 +183,6 @@ Set `cache_enabled: true` to reuse cached retrieval results keyed by query + ena synthesis: llm_mode: auto max_llm_papers: 3 - extraction_max_retries: 0 - collective_max_retries: 0 concurrency: 2 circuit_breaker_failures: 2 ``` diff --git a/docs/development/testing.md b/docs/development/testing.md index bca3865..ed2c1ca 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -42,9 +42,9 @@ All LLM integration tests **mock pydantic-ai** — no Ollama or cloud API requir | Pattern | Example location | |---------|------------------| -| `patch create_llm_agent` | `test_synthesis.py` | +| `patch src.models.structured.try_run_structured` | `test_synthesis.py`, `test_structured_outputs.py` | | `patch` OpenAI/Pydantic AI constructors | `test_llm_providers.py` | -| `MagicMock(EnhancedResponseHandler)` | synthesis workflow tests | +| `TestModel` / `FunctionModel` from pydantic-ai | `test_structured_outputs.py` | This keeps CI fast and deterministic. Manual LLM verification uses the CLI with real providers. @@ -128,7 +128,6 @@ Document these when adding tests: |-----|--------| | Query understanding | No dedicated unit test file | | API routes | Scaffold tests only — no HTTP integration tests | -| `EnhancedResponseHandler` | Subcomponents tested; not end-to-end | | Live LLM / API | All mocked in unit tests | | Subprocess tests | Marked `@pytest.mark.slow`; may skip in tight CI | diff --git a/docs/llm/heuristic-vs-llm.md b/docs/llm/heuristic-vs-llm.md index 57386af..6f85ee1 100644 --- a/docs/llm/heuristic-vs-llm.md +++ b/docs/llm/heuristic-vs-llm.md @@ -126,15 +126,13 @@ YAML equivalents under `synthesis:` and `query_expansion:` in `config/default.ya ## Structured outputs -When the LLM path runs, output schemas are enforced natively by default -(`RA_LLM__STRUCTURED_OUTPUTS=true`): the pydantic model is attached to the -agent (`output_type`), pydantic-ai validates the response and retries with -validation feedback on schema violations, and the caller receives a typed -object or falls back to heuristics. Disable the flag to route through the -legacy prose-JSON path with the repair/retry machinery in `src/utils/` — -useful only for backends that cannot honor response schemas. Identity -fields (paper ID, title) are always taken from retrieval metadata, never -from model output. +When the LLM path runs, output schemas are enforced natively: the pydantic +model is attached to the agent (`output_type`), pydantic-ai validates the +response and retries with validation feedback on schema violations, and the +caller receives a typed object or falls back to heuristics. This is the only +LLM call path — the legacy prose-JSON repair machinery has been retired. +Identity fields (paper ID, title) are always taken from retrieval metadata, +never from model output. ## Verify LLM is active diff --git a/docs/operations/logging-and-debug.md b/docs/operations/logging-and-debug.md index 9055ff1..836d1ae 100644 --- a/docs/operations/logging-and-debug.md +++ b/docs/operations/logging-and-debug.md @@ -104,10 +104,6 @@ Use these dumps to inspect why a stage was partial or what papers were retrieved 6. **Correlate with combined log** using timestamps and session ID in log lines. -## Quality metrics (optional) - -`src/utils/quality_monitor.py` can write additional JSON to `logs/quality_metrics_.json` when quality monitoring is active during LLM stages. - ## What debug does *not* include - Raw HTTP response bodies from retrieval providers (only normalized papers in artifacts) diff --git a/docs/operations/progress-streaming.md b/docs/operations/progress-streaming.md index afcdc4a..6a36d32 100644 --- a/docs/operations/progress-streaming.md +++ b/docs/operations/progress-streaming.md @@ -64,7 +64,7 @@ sequenceDiagram Bus->>Rep: _on_stage_complete ``` -Stages call `set_activity()` / `set_llm_preview()` during long operations (especially synthesis). LLM calls use `stream_agent_text()` which delegates to `stream_agent_response()` when a reporter is active. +Stages call `set_activity()` / `set_llm_preview()` during long operations (especially synthesis). Structured LLM calls report per-paper activity through `set_activity()`; `stream_agent_text()` remains available for streaming free-text agent runs and delegates to `stream_agent_response()` when a reporter is active. ## Context variable diff --git a/src/analysis/gap_analysis.py b/src/analysis/gap_analysis.py index 9d4744c..240bd1f 100644 --- a/src/analysis/gap_analysis.py +++ b/src/analysis/gap_analysis.py @@ -8,10 +8,8 @@ from typing import TYPE_CHECKING from ..core.context import PipelineContext, StageResult -from ..models import AgentFactory, AgentRole +from ..models import AgentRole from ..retrieval.models import GapAnalysisResult, PaperCluster, SynthesisResult -from ..utils.enhanced_response_handler import EnhancedResponseHandler -from ..utils.response_models import RequestContext, ResponseHandlerConfig if TYPE_CHECKING: from ..config.settings import LLMConfig @@ -98,8 +96,6 @@ async def analyze_gaps( clusters: list[PaperCluster] | None = None, *, llm_config: LLMConfig | None = None, - handler: EnhancedResponseHandler | None = None, - session_id: str = "", ) -> GapAnalysisResult: """Refine synthesis gaps into prioritized research opportunities.""" clusters = clusters or [] @@ -117,38 +113,16 @@ async def analyze_gaps( if reporter is not None: reporter.set_activity("Analyzing research gaps with AI…") - if llm_config.structured_outputs: - from ..models.structured import try_run_structured + from ..models.structured import try_run_structured - gap_result = await try_run_structured( - AgentRole.GAP_ANALYSIS, - prompt, - GapAnalysisResult, - llm_config, - ) - if gap_result is not None: - return gap_result - return _heuristic_gap_analysis(synthesis, query, clusters) - - agent = AgentFactory(llm_config).create_agent(AgentRole.GAP_ANALYSIS, config=llm_config) - response_handler = handler or EnhancedResponseHandler(ResponseHandlerConfig()) - context = RequestContext( - user_query=query, - model_name=llm_config.model, - session_id=session_id, - ) - - result = await response_handler.process_structured_response( - agent, + gap_result = await try_run_structured( + AgentRole.GAP_ANALYSIS, prompt, - context, GapAnalysisResult, - schema_description=( - '{"gaps": ["..."], "opportunities": ["..."], "underexplored_areas": ["..."]}' - ), + llm_config, ) - if result.success and isinstance(result.data, GapAnalysisResult): - return result.data + if gap_result is not None: + return gap_result return _heuristic_gap_analysis(synthesis, query, clusters) @@ -157,9 +131,6 @@ class GapAnalysisStage: name = "gap_analysis" - def __init__(self, handler: EnhancedResponseHandler | None = None) -> None: - self.handler = handler - async def run( self, ctx: PipelineContext, @@ -196,8 +167,6 @@ async def run( synthesis, clusters, llm_config=ctx.config.llm, - handler=self.handler, - session_id=ctx.session.id, ) except Exception as exc: warnings.append(f"Gap analysis failed, using heuristic fallback: {exc}") diff --git a/src/analysis/synthesis.py b/src/analysis/synthesis.py index 249f3e9..834c426 100644 --- a/src/analysis/synthesis.py +++ b/src/analysis/synthesis.py @@ -8,11 +8,9 @@ import time from typing import TYPE_CHECKING -from pydantic_ai import Agent - from ..core.context import PipelineContext, StageResult from ..core.paper_adapters import ensure_ranked_papers -from ..models import AgentFactory, AgentRole, create_llm_agent +from ..models import AgentRole from ..research.text_utils import extract_core_concepts from ..retrieval.models import ( PaperAnalysis, @@ -21,15 +19,13 @@ RankedPaper, SynthesisResult, ) -from ..utils.enhanced_response_handler import EnhancedResponseHandler from ..utils.logging_system import logger -from ..utils.response_models import RequestContext, ResponseHandlerConfig, RetryConfig if TYPE_CHECKING: from ..config.settings import LLMConfig, SynthesisConfig -__all__ = ["create_llm_agent", "SynthesisStage", "extract_papers", "synthesize_collective"] +__all__ = ["SynthesisStage", "extract_papers", "synthesize_collective"] HEURISTIC_DISAGREEMENT_PLACEHOLDER = ( "Cross-paper disagreement analysis limited in heuristic mode." @@ -287,12 +283,6 @@ def _extractions_to_paper_analyses( return analyses -def _synthesis_handler_config(max_retries: int) -> ResponseHandlerConfig: - return ResponseHandlerConfig( - retry_config=RetryConfig(max_retries=max_retries), - ) - - def resolve_synthesis_input(data: object, ctx: PipelineContext) -> SynthesisResult: """Resolve a synthesis result from stage output or pipeline artifacts.""" artifact = ctx.get_artifact("synthesis_result") @@ -343,14 +333,6 @@ def recover_synthesis_output( return synthesis -def _use_structured_outputs(llm_config: LLMConfig | None) -> bool: - if llm_config is not None: - return llm_config.structured_outputs - from ..config.settings import get_settings - - return get_settings().llm.structured_outputs - - async def _extract_single_paper_structured( paper: RankedPaper, query: str, @@ -380,44 +362,12 @@ async def _extract_single_paper_structured( return extraction.model_copy(update=updates), True -async def _extract_single_paper( - paper: RankedPaper, - query: str, - agent: Agent, - handler: EnhancedResponseHandler, - context: RequestContext, - passages: list[str] | None = None, -) -> tuple[PaperExtraction, bool]: - prompt = _build_extraction_prompt(paper, query, passages) - result = await handler.process_structured_response( - agent, - prompt, - context, - PaperExtraction, - schema_description=( - '{"paper_id": "...", "title": "...", "methodology": ["..."], ' - '"datasets": ["..."], "benchmarks": ["..."], ' - '"limitations": ["..."], "findings": ["..."], "evidence": ["..."]}' - ), - ) - if result.success and isinstance(result.data, PaperExtraction): - extraction = result.data - if not extraction.evidence and passages: - extraction = extraction.model_copy( - update={"evidence": _evidence_from_passages(passages)} - ) - return extraction, True - return _heuristic_extraction(paper, passages), False - - async def extract_papers( ranked_papers: list[RankedPaper], query: str, *, llm_config: LLMConfig | None = None, - handler: EnhancedResponseHandler | None = None, concurrency: int = 4, - session_id: str = "", synthesis_config: SynthesisConfig | None = None, passages: dict[str, list[str]] | None = None, ) -> list[PaperExtraction]: @@ -449,22 +399,6 @@ async def extract_papers( llm_targets = ranked_papers[:max_llm_papers] heuristic_targets = ranked_papers[max_llm_papers:] - use_structured = _use_structured_outputs(llm_config) - agent = None - response_handler = None - context = None - if not use_structured: - # Legacy prose-JSON path with the repair/retry machinery. - agent = AgentFactory(llm_config).create_agent(AgentRole.EXTRACTION, config=llm_config) - response_handler = handler or EnhancedResponseHandler( - _synthesis_handler_config(synthesis_config.extraction_max_retries) - ) - context = RequestContext( - user_query=query, - model_name=llm_config.model if llm_config else "default", - session_id=session_id, - ) - from ..utils.progress_reporter import get_progress_reporter semaphore = asyncio.Semaphore(max(concurrency, 1)) @@ -490,22 +424,12 @@ async def _extract_with_circuit(index: int, paper: RankedPaper) -> PaperExtracti f"Analyzing paper {index}/{len(llm_targets)}: {title_preview}…" ) - if use_structured: - extraction, llm_success = await _extract_single_paper_structured( - paper, - query, - llm_config, - passages=paper_passages, - ) - else: - extraction, llm_success = await _extract_single_paper( - paper, - query, - agent, - response_handler, - context, - passages=paper_passages, - ) + extraction, llm_success = await _extract_single_paper_structured( + paper, + query, + llm_config, + passages=paper_passages, + ) if llm_success: breaker["consecutive_failures"] = 0 @@ -548,8 +472,6 @@ async def synthesize_collective( *, ranked_papers: list[RankedPaper] | None = None, llm_config: LLMConfig | None = None, - handler: EnhancedResponseHandler | None = None, - session_id: str = "", synthesis_config: SynthesisConfig | None = None, ) -> SynthesisResult: """Pass B — collective cross-paper synthesis.""" @@ -581,42 +503,16 @@ async def synthesize_collective( f"Synthesizing insights across {len(extractions)} paper(s)…" ) - if _use_structured_outputs(llm_config): - from ..models.structured import try_run_structured - - synthesis = await try_run_structured( - AgentRole.SYNTHESIS, - prompt, - SynthesisResult, - llm_config, - ) - if synthesis is not None: - return synthesis - logger.warning("LLM collective synthesis failed; using heuristic fallback") - return _heuristic_synthesis(query, extractions, clusters, ranked_papers=ranked_papers) - - agent = AgentFactory(llm_config).create_agent(AgentRole.SYNTHESIS, config=llm_config) - response_handler = handler or EnhancedResponseHandler( - _synthesis_handler_config(synthesis_config.collective_max_retries) - ) - context = RequestContext( - user_query=query, - model_name=llm_config.model, - session_id=session_id, - ) + from ..models.structured import try_run_structured - result = await response_handler.process_structured_response( - agent, + synthesis = await try_run_structured( + AgentRole.SYNTHESIS, prompt, - context, SynthesisResult, - schema_description=( - '{"agreements": ["..."], "disagreements": ["..."], "trends": ["..."], ' - '"gaps": ["..."], "datasets": ["..."], "methodologies": ["..."]}' - ), + llm_config, ) - if result.success and isinstance(result.data, SynthesisResult): - return result.data + if synthesis is not None: + return synthesis logger.warning("LLM collective synthesis failed; using heuristic fallback") return _heuristic_synthesis(query, extractions, clusters, ranked_papers=ranked_papers) @@ -627,9 +523,7 @@ async def run_synthesis( clusters: list[PaperCluster], *, llm_config: LLMConfig | None = None, - handler: EnhancedResponseHandler | None = None, concurrency: int = 4, - session_id: str = "", synthesis_config: SynthesisConfig | None = None, passages: dict[str, list[str]] | None = None, ) -> tuple[SynthesisResult, list[PaperExtraction], list[PaperAnalysis]]: @@ -643,9 +537,7 @@ async def run_synthesis( ranked_papers, query, llm_config=llm_config, - handler=handler, concurrency=synthesis_config.concurrency or concurrency, - session_id=session_id, synthesis_config=synthesis_config, passages=passages, ) @@ -655,8 +547,6 @@ async def run_synthesis( clusters, ranked_papers=ranked_papers, llm_config=llm_config, - handler=handler, - session_id=session_id, synthesis_config=synthesis_config, ) paper_analyses = _extractions_to_paper_analyses(extractions, ranked_papers) @@ -668,12 +558,7 @@ class SynthesisStage: name = "synthesis" - def __init__( - self, - handler: EnhancedResponseHandler | None = None, - concurrency: int = 4, - ) -> None: - self.handler = handler + def __init__(self, concurrency: int = 4) -> None: self.concurrency = concurrency async def run( @@ -724,9 +609,7 @@ async def run( data, passages=ctx.get_artifact("fulltext_passages") or {}, llm_config=ctx.config.llm, - handler=self.handler, concurrency=ctx.config.synthesis.concurrency, - session_id=ctx.session.id, synthesis_config=ctx.config.synthesis, ) except (asyncio.TimeoutError, asyncio.CancelledError) as exc: diff --git a/src/analysis/verification.py b/src/analysis/verification.py index fdc5eb6..d53e96c 100644 --- a/src/analysis/verification.py +++ b/src/analysis/verification.py @@ -143,7 +143,7 @@ def _result(summary: VerificationSummary | None) -> StageResult[GapAnalysisResul } passages: dict[str, list[str]] = ctx.get_artifact("fulltext_passages") or {} - use_llm = ctx.config.synthesis.llm_enabled and ctx.config.llm.structured_outputs + use_llm = ctx.config.synthesis.llm_enabled method = "llm" if use_llm else "heuristic" checked = 0 diff --git a/src/config/settings.py b/src/config/settings.py index 2a8b0da..c602e71 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -66,7 +66,6 @@ class LLMConfig(BaseModel): api_key: str | None = None temperature: float = 0.2 timeout_seconds: int = 120 - structured_outputs: bool = True class EmbeddingConfig(BaseModel): @@ -237,8 +236,6 @@ class SynthesisConfig(BaseModel): llm_mode: LlmMode = "auto" llm_enabled: bool = False max_llm_papers: int = 3 - extraction_max_retries: int = 0 - collective_max_retries: int = 0 concurrency: int = 2 circuit_breaker_failures: int = 2 diff --git a/src/models/structured.py b/src/models/structured.py index fa5cffd..0a650b3 100644 --- a/src/models/structured.py +++ b/src/models/structured.py @@ -5,8 +5,8 @@ back, the schema is attached to the agent (``output_type``) and pydantic-ai enforces it natively — invalid output triggers the framework's own retry with validation feedback, and the caller receives a typed object or an -exception. This replaces the string-repair path for providers that support -structured outputs; heuristic fallbacks remain the safety net. +exception. This is the only LLM call path — the legacy prose-JSON repair +machinery has been retired; heuristic fallbacks remain the safety net. """ from __future__ import annotations diff --git a/src/research/query_expansion.py b/src/research/query_expansion.py index 89810e9..ebba086 100644 --- a/src/research/query_expansion.py +++ b/src/research/query_expansion.py @@ -277,51 +277,31 @@ async def expand_query_llm( try: from ..config.settings import get_settings - from ..models import AgentFactory, AgentRole + from ..models import AgentRole + from ..models.structured import run_structured + from ..utils.progress_reporter import get_progress_reporter settings = get_settings() prompt = ( f"Expand this research query into {config.max_variants} search variants " f"and {config.max_sub_questions} sub-questions: {query}" ) - from ..utils.progress_reporter import get_progress_reporter, stream_agent_text reporter = get_progress_reporter() if reporter is not None: reporter.set_activity("Expanding query with AI…") - if settings.llm.structured_outputs: - from ..models.structured import run_structured - - suggestions = await run_structured( - AgentRole.EXPANSION, - prompt, - _ExpansionSuggestions, - settings.llm, - ) - return ( - [str(item) for item in suggestions.variants][: config.max_variants], - [str(item) for item in suggestions.sub_questions][ - : config.max_sub_questions - ], - ) - - agent = AgentFactory(settings.llm).create_agent(AgentRole.EXPANSION, config=settings.llm) - raw_output = await stream_agent_text( - agent, + suggestions = await run_structured( + AgentRole.EXPANSION, prompt, - label="Expanding search queries…", + _ExpansionSuggestions, + settings.llm, ) - import json - - from ..retrieval.helpers_modules.json_extraction import extract_and_clean_json - - payload = json.loads(extract_and_clean_json(raw_output)) - variants = payload.get("variants", []) - sub_questions = payload.get("sub_questions", []) return ( - [str(item) for item in variants][: config.max_variants], - [str(item) for item in sub_questions][: config.max_sub_questions], + [str(item) for item in suggestions.variants][: config.max_variants], + [str(item) for item in suggestions.sub_questions][ + : config.max_sub_questions + ], ) except Exception: return [], [] diff --git a/src/research/research_loop.py b/src/research/research_loop.py index 4b10b9e..283ab78 100644 --- a/src/research/research_loop.py +++ b/src/research/research_loop.py @@ -95,7 +95,7 @@ async def assess_coverage( min_sufficient_papers=config.min_sufficient_papers, ) - if not (ctx.config.synthesis.llm_enabled and ctx.config.llm.structured_outputs): + if not ctx.config.synthesis.llm_enabled: return heuristic from ..models.structured import try_run_structured diff --git a/src/retrieval/helpers_modules/__init__.py b/src/retrieval/helpers_modules/__init__.py deleted file mode 100644 index ccfb7e8..0000000 --- a/src/retrieval/helpers_modules/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -"""Helper modules for orchestrator.""" - -from .json_extraction import extract_and_clean_json -from .validation import validate_json_structure, enhance_validation_with_retry_strategy -from .recovery import attempt_partial_recovery, enhanced_partial_recovery - -__all__ = [ - "extract_and_clean_json", - "validate_json_structure", - "enhance_validation_with_retry_strategy", - "attempt_partial_recovery", - "enhanced_partial_recovery", -] diff --git a/src/retrieval/helpers_modules/json_extraction.py b/src/retrieval/helpers_modules/json_extraction.py deleted file mode 100644 index 8084101..0000000 --- a/src/retrieval/helpers_modules/json_extraction.py +++ /dev/null @@ -1,242 +0,0 @@ -# -*- coding: utf-8 -*- -"""JSON extraction and cleaning utilities for orchestrator.""" - -import re -from typing import List - - -def extract_and_clean_json(raw_output: str) -> str: - """Extract and clean JSON from LLM response with enhanced pattern matching. - - Handles multiple code block formats, mixed content, and various LLM response patterns. - - Args: - raw_output: Raw LLM response string - - Returns: - str: Cleaned JSON string ready for parsing - - Raises: - ValueError: If no valid JSON can be extracted - """ - # Normalize whitespace and line endings - normalized = re.sub(r'\r\n|\r', '\n', raw_output.strip()) - - # Pattern 1: Try to extract JSON from code blocks with various formats - # Matches ```json, ```, and other code block variations - code_block_patterns = [ - r'```(?:json)?\s*\n?(.*?)\n?```', # ```json or ``` with optional newlines - r'`{3,}\s*(?:json)?\s*\n?(.*?)\n?`{3,}', # Multiple backticks - r'~~~(?:json)?\s*\n?(.*?)\n?~~~', # Alternative code block syntax - ] - - for pattern in code_block_patterns: - matches = re.findall(pattern, normalized, re.DOTALL | re.IGNORECASE) - for match in matches: - candidate = match.strip() - if _looks_like_json(candidate): - return _normalize_json_formatting(candidate) - - # Pattern 2: Try to extract JSON from mixed content - # Look for JSON-like structures in the text - json_patterns = [ - r'\{[^{}]*"query"[^{}]*"papers"[^{}]*\}', # Simple single-level JSON - r'\{(?:[^{}]|\{[^{}]*\})*"query"(?:[^{}]|\{[^{}]*\})*"papers"(?:[^{}]|\{[^{}]*\})*\}', # Nested JSON - ] - - for pattern in json_patterns: - matches = re.findall(pattern, normalized, re.DOTALL) - for match in matches: - candidate = match.strip() - if _looks_like_json(candidate): - return _normalize_json_formatting(candidate) - - # Pattern 3: Try to find JSON by looking for balanced braces - # Find the largest balanced JSON-like structure - json_candidates = _extract_balanced_json_candidates(normalized) - for candidate in json_candidates: - if _looks_like_json(candidate): - return _normalize_json_formatting(candidate) - - # Pattern 4: If no code blocks or clear JSON structure, try the whole response - # Remove common prefixes/suffixes that might interfere - cleaned_full = _remove_common_prefixes_suffixes(normalized) - if _looks_like_json(cleaned_full): - return _normalize_json_formatting(cleaned_full) - - # If all extraction attempts fail, raise an error - raise ValueError("No valid JSON structure found in LLM response") - - -def _looks_like_json(text: str) -> bool: - """Check if text looks like it could be valid JSON. - - Args: - text: Text to check - - Returns: - bool: True if text appears to be JSON-like - """ - text = text.strip() - if not text: - return False - - # Must start and end with braces for object - if not (text.startswith('{') and text.endswith('}')): - return False - - # Should contain both required fields - if '"query"' not in text or '"papers"' not in text: - return False - - # Basic structure check - should have reasonable JSON-like content - if text.count('{') < 1 or text.count('}') < 1: - return False - - return True - - -def _extract_balanced_json_candidates(text: str) -> List[str]: - """Extract potential JSON candidates by finding balanced brace structures. - - Args: - text: Text to search for JSON structures - - Returns: - List[str]: List of potential JSON candidates, ordered by size (largest first) - """ - candidates = [] - - # Find all opening braces and try to match them with closing braces - for i, char in enumerate(text): - if char == '{': - brace_count = 0 - for j in range(i, len(text)): - if text[j] == '{': - brace_count += 1 - elif text[j] == '}': - brace_count -= 1 - if brace_count == 0: - # Found a balanced structure - candidate = text[i:j+1].strip() - if len(candidate) > 10: # Minimum reasonable JSON size - candidates.append(candidate) - break - - # Sort by length (largest first) to prioritize more complete structures - return sorted(candidates, key=len, reverse=True) - - -def _remove_common_prefixes_suffixes(text: str) -> str: - """Remove common prefixes and suffixes that might interfere with JSON parsing. - - Args: - text: Text to clean - - Returns: - str: Cleaned text - """ - # Common prefixes to remove - prefixes = [ - r'^.*?(?=\{)', # Everything before the first opening brace - r'^[^{]*', # Non-brace characters at the start - ] - - cleaned = text - - # Apply prefix removal - for prefix in prefixes: - cleaned = re.sub(prefix, '', cleaned, count=1) - - # Apply suffix removal (but keep the closing brace) - # Find the last closing brace and remove everything after it - last_brace = cleaned.rfind('}') - if last_brace != -1: - cleaned = cleaned[:last_brace + 1] - - return cleaned.strip() - - -def _normalize_json_formatting(json_str: str) -> str: - """Normalize JSON formatting for consistent parsing. - - Args: - json_str: JSON string to normalize - - Returns: - str: Normalized JSON string - """ - # Remove extra whitespace while preserving string content - # This is a simple normalization - for more complex cases, - # we could parse and re-serialize, but that might fail on malformed JSON - - # Normalize line endings - normalized = re.sub(r'\r\n|\r', '\n', json_str) - - # CRITICAL FIX: Handle literal newlines within JSON string values - # This fixes the "Invalid control character" error when LLMs insert - # literal newlines within JSON strings instead of proper escaping - normalized = _fix_newlines_in_json_strings(normalized) - - # Remove excessive whitespace around structural elements - # But be careful not to modify string content - normalized = re.sub(r'\s*{\s*', '{', normalized) - normalized = re.sub(r'\s*}\s*', '}', normalized) - normalized = re.sub(r'\s*\[\s*', '[', normalized) - normalized = re.sub(r'\s*\]\s*', ']', normalized) - normalized = re.sub(r'\s*,\s*', ',', normalized) - normalized = re.sub(r'\s*:\s*', ':', normalized) - - # Remove leading/trailing whitespace - return normalized.strip() - - -def _fix_newlines_in_json_strings(json_str: str) -> str: - """Fix literal newlines within JSON string values. - - This function identifies JSON string values that contain literal newlines - and either removes them (joining the lines) or escapes them properly. - This fixes the "Invalid control character" JSON parsing error. - - Args: - json_str: JSON string that may contain literal newlines in string values - - Returns: - str: JSON string with fixed newlines in string values - """ - result = [] - i = 0 - in_string = False - escape_next = False - - while i < len(json_str): - char = json_str[i] - - if escape_next: - # Previous character was a backslash, so this character is escaped - result.append(char) - escape_next = False - elif char == '\\' and in_string: - # This is an escape character - result.append(char) - escape_next = True - elif char == '"' and not escape_next: - # This is a quote that starts or ends a string - result.append(char) - in_string = not in_string - elif char == '\n' and in_string: - # This is a literal newline within a JSON string - remove it - # and join with the next line (removing any leading whitespace) - # Skip the newline and any following whitespace - i += 1 - while i < len(json_str) and json_str[i] in ' \t': - i += 1 - # Continue without adding anything (effectively removes the newline and whitespace) - continue - else: - # Regular character - result.append(char) - - i += 1 - - return ''.join(result) diff --git a/src/retrieval/helpers_modules/recovery.py b/src/retrieval/helpers_modules/recovery.py deleted file mode 100644 index 72a8dea..0000000 --- a/src/retrieval/helpers_modules/recovery.py +++ /dev/null @@ -1,267 +0,0 @@ -# -*- coding: utf-8 -*- -"""Recovery utilities for orchestrator.""" - -import re -from typing import Optional, List, Dict - -from ...utils.message_formatter import MessageFormatter -from ...utils.response_models import RecoveryResult, RecoveryMethod, RecoveryConfig - - -def attempt_partial_recovery(raw_output: str, cleaned_output: str, parsed_data: dict = None) -> None: - """Attempt to recover partial data from malformed JSON responses. - - This function tries to extract useful information even when the JSON - structure is incomplete or malformed, providing fallback mechanisms - for partial data recovery. - - Args: - raw_output: The original raw LLM response - cleaned_output: The cleaned JSON string that failed parsing - parsed_data: Already parsed JSON data (if available) - """ - try: - print(MessageFormatter.partial_recovery_header()) - recovered_items = [] - - # If we have parsed data, use it directly - if parsed_data: - if "query" in parsed_data: - print(f"Recovered query: {parsed_data['query']}") - recovered_items.append("query") - - if "papers" in parsed_data and isinstance(parsed_data["papers"], list): - print(f"Found {len(parsed_data['papers'])} papers:") - for i, paper in enumerate(parsed_data["papers"][:5], 1): # Limit to first 5 - if isinstance(paper, dict) and "title" in paper: - print(f" {i}. {paper['title']}") - recovered_items.append(f"{len(parsed_data['papers'])} papers") - elif "papers" in parsed_data: - print(f"Papers field found but not in expected format: {type(parsed_data['papers'])}") - recovered_items.append("papers field (invalid format)") - else: - # Try to extract at least the query if it exists - query_match = re.search(r'"query"\s*:\s*"([^"]*)"', cleaned_output) - if query_match: - query = query_match.group(1) - print(f"Recovered query: {query}") - recovered_items.append("query") - - # Try to extract paper titles if they exist - title_matches = re.findall(r'"title"\s*:\s*"([^"]*)"', cleaned_output) - if title_matches: - print(f"Found {len(title_matches)} paper titles:") - for i, title in enumerate(title_matches[:5], 1): # Limit to first 5 - print(f" {i}. {title}") - recovered_items.append(f"{len(title_matches)} paper titles") - - if recovered_items: - print(MessageFormatter.recovery_success_message(recovered_items)) - else: - print(MessageFormatter.recovery_failure_message()) - print(MessageFormatter.raw_response_header()) - print(raw_output) - - except Exception: - # If partial recovery fails, just show the raw response - print(MessageFormatter.recovery_failure_message()) - print(MessageFormatter.raw_response_header()) - print(raw_output) - - -def enhanced_partial_recovery( - raw_output: str, - cleaned_output: str, - parsed_data: dict = None, - recovery_config: Optional[RecoveryConfig] = None -) -> RecoveryResult: - """Enhanced version of existing partial recovery with additional techniques. - - Extends the current attempt_partial_recovery with: - - Systematic recovery result tracking - - Additional extraction patterns - - Configurable recovery methods - - Structured return values - - Args: - raw_output: The original raw LLM response - cleaned_output: The cleaned JSON string that failed parsing - parsed_data: Already parsed JSON data (if available) - recovery_config: Configuration for recovery behavior - - Returns: - RecoveryResult: Structured recovery result with success status and data - """ - # Use default config if none provided - if recovery_config is None: - recovery_config = RecoveryConfig() - - recovery_result = RecoveryResult() - recovered_data = {} - warnings = [] - - try: - # First, call existing recovery function to maintain current behavior - attempt_partial_recovery(raw_output, cleaned_output, parsed_data) - - # Enhanced recovery techniques - if recovery_config.enable_enhanced_extraction: - enhanced_data = _extract_with_enhanced_patterns(raw_output) - if enhanced_data: - recovered_data.update(enhanced_data) - recovery_result.recovery_method = RecoveryMethod.ENHANCED_PATTERNS - - # Try to recover from parsed_data if available - if parsed_data: - if "query" in parsed_data and isinstance(parsed_data["query"], str): - recovered_data["query"] = parsed_data["query"] - - if "papers" in parsed_data: - if isinstance(parsed_data["papers"], list): - # Filter out invalid papers and keep valid ones - valid_papers = [] - for paper in parsed_data["papers"]: - if isinstance(paper, dict) and "title" in paper: - valid_papers.append(paper) - if valid_papers: - recovered_data["papers"] = valid_papers - else: - warnings.append("Papers field is not a list") - - # Pattern-based recovery from cleaned output - if not recovered_data.get("query"): - query_patterns = [ - r'"query"\s*:\s*"([^"]*)"', - r"'query'\s*:\s*'([^']*)'", - r'query:\s*"([^"]*)"', - r'Query:\s*([^\n,}]+)' - ] - for pattern in query_patterns: - match = re.search(pattern, cleaned_output, re.IGNORECASE) - if match: - recovered_data["query"] = match.group(1).strip() - break - - # Enhanced paper extraction - if not recovered_data.get("papers"): - papers = _extract_papers_with_patterns(cleaned_output) - if papers: - recovered_data["papers"] = papers - - # Set success status and confidence - if recovered_data: - recovery_result.success = True - recovery_result.recovered_data = recovered_data - recovery_result.warnings = warnings - - # Calculate confidence based on completeness - confidence = 0.0 - if "query" in recovered_data: - confidence += 0.3 - if "papers" in recovered_data: - confidence += 0.7 * min(len(recovered_data["papers"]) / 3, 1.0) - recovery_result.confidence_score = confidence - else: - recovery_result.success = False - recovery_result.error_message = "No recoverable data found" - - except Exception as e: - recovery_result.success = False - recovery_result.error_message = str(e) - - return recovery_result - - -def _extract_with_enhanced_patterns(text: str) -> Optional[Dict]: - """Additional extraction patterns beyond existing recovery. - - Args: - text: Raw text to extract from - - Returns: - dict: Extracted data or None if nothing found - """ - extracted = {} - - # Enhanced patterns for paper extraction from conversational text - paper_patterns = [ - # Pattern: "Title: Paper Name" - r'(?:Title|Paper):\s*([^\n]+)', - # Pattern: "1. Paper Name" - r'^\s*\d+\.\s*([^\n]+)', - # Pattern: "- Paper Name" - r'^\s*[-*]\s*([^\n]+)', - ] - - papers = [] - for pattern in paper_patterns: - matches = re.findall(pattern, text, re.MULTILINE | re.IGNORECASE) - for match in matches: - title = match.strip() - if len(title) > 10 and not title.lower().startswith(('the', 'a ', 'an ')): - papers.append({"title": title}) - - if papers: - extracted["papers"] = papers[:5] # Limit to 5 papers - - # Try to extract query from conversational context - query_patterns = [ - r'(?:looking for|searching for|find|about)\s+([^.!?]+)', - r'(?:research on|papers on|studies on)\s+([^.!?]+)', - r'(?:query|search):\s*([^.!?\n]+)' - ] - - for pattern in query_patterns: - match = re.search(pattern, text, re.IGNORECASE) - if match: - query = match.group(1).strip() - if len(query) > 5: - extracted["query"] = query - break - - return extracted if extracted else None - - -def _extract_papers_with_patterns(text: str) -> List[Dict]: - """Extract paper information using various patterns. - - Args: - text: Text to extract papers from - - Returns: - List[Dict]: List of extracted paper objects - """ - papers = [] - - # Try to find title patterns in the text - title_patterns = [ - r'"title"\s*:\s*"([^"]+)"', - r"'title'\s*:\s*'([^']+)'", - r'title:\s*"([^"]+)"', - r'Title:\s*([^\n,}]+)' - ] - - for pattern in title_patterns: - matches = re.findall(pattern, text, re.IGNORECASE) - for match in matches: - title = match.strip() - if title and len(title) > 5: - paper = {"title": title} - - # Try to find associated metadata near the title - title_pos = text.find(match) - context = text[max(0, title_pos-200):title_pos+200] - - # Look for year - year_match = re.search(r'"year"\s*:\s*(\d{4})', context) - if year_match: - paper["year"] = int(year_match.group(1)) - - # Look for venue - venue_match = re.search(r'"venue"\s*:\s*"([^"]+)"', context) - if venue_match: - paper["venue"] = venue_match.group(1) - - papers.append(paper) - - return papers[:5] # Limit to 5 papers diff --git a/src/retrieval/helpers_modules/validation.py b/src/retrieval/helpers_modules/validation.py deleted file mode 100644 index 67c8df4..0000000 --- a/src/retrieval/helpers_modules/validation.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- -"""Validation utilities for orchestrator.""" - -from dataclasses import dataclass -from typing import Optional - -from ...utils.message_formatter import MessageFormatter -from ...utils.response_models import RetryStrategy - - -@dataclass -class ValidationResult: - """Result of JSON structure validation.""" - is_valid: bool - error_message: str = "" - error_type: str = "" - show_raw_response: bool = True - retry_strategy: Optional[RetryStrategy] = None - - -def validate_json_structure(parsed_data: any) -> ValidationResult: - """Validate JSON structure with comprehensive checks and graceful degradation. - - Performs layered validation to ensure data integrity before model validation. - Provides specific error messages for each type of validation failure. - - Args: - parsed_data: The parsed JSON data to validate - - Returns: - ValidationResult: Validation result with error details if invalid - """ - # Check 1: Must be a dictionary (JSON object) - if not isinstance(parsed_data, dict): - actual_type = type(parsed_data).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.structure_error( - f"Response must be a JSON object, got {actual_type}" - ), - error_type="structure" - ) - - # Check 2: Must have required fields - required_fields = ["query", "papers"] - missing_fields = [field for field in required_fields if field not in parsed_data] - - if missing_fields: - available_fields = list(parsed_data.keys()) - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.schema_validation_error( - f"Missing required fields: {missing_fields}. " - f"Expected 'query' and 'papers', got: {available_fields}" - ), - error_type="schema" - ) - - # Check 3: Validate "query" field type - query_value = parsed_data["query"] - if not isinstance(query_value, str): - actual_type = type(query_value).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - "query", "string", actual_type - ), - error_type="field_type" - ) - - # Check 4: Validate "papers" field type - papers_value = parsed_data["papers"] - if not isinstance(papers_value, list): - actual_type = type(papers_value).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - "papers", "list", actual_type - ), - error_type="field_type" - ) - - # Check 5: Validate each paper object structure - for i, paper in enumerate(papers_value): - if not isinstance(paper, dict): - actual_type = type(paper).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.structure_error( - f"Paper at index {i} must be an object, got {actual_type}" - ), - error_type="structure" - ) - - # Check required paper fields - if "title" not in paper: - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.schema_validation_error( - f"Paper at index {i} missing required 'title' field" - ), - error_type="schema" - ) - - # Validate title is a string - if not isinstance(paper["title"], str): - actual_type = type(paper["title"]).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - f"papers[{i}].title", "string", actual_type - ), - error_type="field_type" - ) - - # Validate optional fields have correct types when present - optional_string_fields = ["venue", "url", "doi"] - for field in optional_string_fields: - if field in paper and paper[field] is not None and not isinstance(paper[field], str): - actual_type = type(paper[field]).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - f"papers[{i}].{field}", "string or null", actual_type - ), - error_type="field_type" - ) - - # Validate year is integer when present - if "year" in paper and paper["year"] is not None and not isinstance(paper["year"], int): - actual_type = type(paper["year"]).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - f"papers[{i}].year", "integer or null", actual_type - ), - error_type="field_type" - ) - - # Validate list fields have correct types when present - list_fields = ["key_points", "why_relevant"] - for field in list_fields: - if field in paper: - if not isinstance(paper[field], list): - actual_type = type(paper[field]).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - f"papers[{i}].{field}", "list", actual_type - ), - error_type="field_type" - ) - - # Validate all items in list are strings - for j, item in enumerate(paper[field]): - if not isinstance(item, str): - actual_type = type(item).__name__ - return ValidationResult( - is_valid=False, - error_message=MessageFormatter.field_type_error( - f"papers[{i}].{field}[{j}]", "string", actual_type - ), - error_type="field_type" - ) - - # All validation checks passed - return ValidationResult(is_valid=True) - - -def enhance_validation_with_retry_strategy(validation_result: ValidationResult) -> ValidationResult: - """Enhance existing validation result with retry strategy classification. - - This function adds retry strategy information to validation results without - changing the core validation logic, preserving backward compatibility. - - Args: - validation_result: Original validation result from validate_json_structure - - Returns: - ValidationResult: Enhanced validation result with retry strategy - """ - if not validation_result.is_valid: - # Classify error for retry strategy - if validation_result.error_type == "syntax": - validation_result.retry_strategy = RetryStrategy.SYNTAX_EMPHASIS - elif validation_result.error_type == "schema": - validation_result.retry_strategy = RetryStrategy.SCHEMA_EXAMPLE - elif validation_result.error_type == "field_type": - validation_result.retry_strategy = RetryStrategy.TYPE_CLARIFICATION - elif validation_result.error_type == "structure": - validation_result.retry_strategy = RetryStrategy.FORMAT_INSTRUCTION - else: - validation_result.retry_strategy = RetryStrategy.GENERIC_CLARIFICATION - - return validation_result diff --git a/src/utils/content_quality.py b/src/utils/content_quality.py deleted file mode 100644 index 76d18fe..0000000 --- a/src/utils/content_quality.py +++ /dev/null @@ -1,568 +0,0 @@ -# -*- coding: utf-8 -*- -"""Content quality validation and enhancement. - -This module provides content quality validation that goes beyond technical -parsing to assess the relevance, completeness, and usefulness of research -results returned by AI models. -""" - -import re -from typing import List, Dict, Optional, Set, TYPE_CHECKING -from collections import Counter - -if TYPE_CHECKING: - from pydantic_ai import Agent - - from ..retrieval.models import PaperAnalysis, ResearchReport - -from .response_models import ( - ContentQualityConfig, EnhancementConfig, ContentQualityResult, - ContentIssue, ContentIssueType, IssueSeverity, EnhancementResult, - EnhancementAttempt, EnhancementStrategy -) - - -class ContentQualityValidator: - """Validate content quality and relevance of successfully parsed responses.""" - - def __init__(self, config: ContentQualityConfig): - """Initialize content quality validator. - - Args: - config: Configuration for content quality validation - """ - self.config = config - self.query_analyzer = QueryAnalyzer() - self.relevance_scorer = RelevanceScorer() - - def validate_content_quality( - self, - research_report: 'ResearchReport', - original_query: str - ) -> ContentQualityResult: - """Validate content quality of successfully parsed research report. - - Args: - research_report: Parsed and validated research report - original_query: Original user query - - Returns: - ContentQualityResult: Analysis of content quality issues - """ - issues = [] - - # Check for empty results - if not research_report.papers: - issues.append(ContentIssue( - type=ContentIssueType.EMPTY_RESULTS, - severity=IssueSeverity.HIGH, - message="No papers found for this query", - suggestions=self.query_analyzer.suggest_broader_terms(original_query) - )) - - # Check for insufficient results - elif len(research_report.papers) < self.config.min_expected_papers: - issues.append(ContentIssue( - type=ContentIssueType.INSUFFICIENT_RESULTS, - severity=IssueSeverity.MEDIUM, - message=f"Only {len(research_report.papers)} papers found, expected at least {self.config.min_expected_papers}", - suggestions=self.query_analyzer.suggest_query_expansion(original_query) - )) - - # Check for incomplete paper analysis - incomplete_papers = self._find_incomplete_papers(research_report.papers) - if incomplete_papers: - issues.append(ContentIssue( - type=ContentIssueType.INCOMPLETE_ANALYSIS, - severity=IssueSeverity.LOW, - message=f"{len(incomplete_papers)} papers have incomplete analysis", - affected_papers=incomplete_papers - )) - - # Check for relevance issues - relevance_scores = self.relevance_scorer.score_papers(research_report.papers, original_query) - low_relevance_papers = [ - paper for paper, score in relevance_scores.items() - if score < self.config.min_relevance_threshold - ] - - if low_relevance_papers: - issues.append(ContentIssue( - type=ContentIssueType.LOW_RELEVANCE, - severity=IssueSeverity.MEDIUM, - message=f"{len(low_relevance_papers)} papers may not be relevant to your query", - affected_papers=low_relevance_papers, - suggestions=self.query_analyzer.suggest_more_specific_terms(original_query) - )) - - # Calculate overall quality score - overall_quality = self._calculate_overall_quality(research_report, relevance_scores) - - return ContentQualityResult( - has_issues=bool(issues), - issues=issues, - overall_quality_score=overall_quality - ) - - def _find_incomplete_papers(self, papers: List['PaperAnalysis']) -> List['PaperAnalysis']: - """Find papers with incomplete analysis. - - Args: - papers: List of paper analysis objects - - Returns: - List of papers with incomplete analysis - """ - incomplete = [] - - for paper in papers: - # Check for missing or empty key fields - missing_fields = [] - - if not paper.key_points or len(paper.key_points) == 0: - missing_fields.append("key_points") - - if not paper.why_relevant or len(paper.why_relevant) == 0: - missing_fields.append("why_relevant") - - # Check for very short or generic content - if paper.key_points: - avg_length = sum(len(point) for point in paper.key_points) / len(paper.key_points) - if avg_length < 20: # Very short key points - missing_fields.append("detailed_key_points") - - if paper.why_relevant: - avg_length = sum(len(reason) for reason in paper.why_relevant) / len(paper.why_relevant) - if avg_length < 15: # Very short relevance explanations - missing_fields.append("detailed_relevance") - - if missing_fields: - incomplete.append(paper) - - return incomplete - - def _calculate_overall_quality( - self, - research_report: 'ResearchReport', - relevance_scores: Dict['PaperAnalysis', float] - ) -> float: - """Calculate overall quality score for the research report. - - Args: - research_report: The research report - relevance_scores: Relevance scores for each paper - - Returns: - float: Overall quality score (0.0 to 1.0) - """ - if not research_report.papers: - return 0.0 - - # Factors contributing to quality score - factors = {} - - # 1. Number of papers (up to expected minimum) - paper_count_score = min(len(research_report.papers) / self.config.min_expected_papers, 1.0) - factors['paper_count'] = paper_count_score * 0.3 - - # 2. Average relevance score - if relevance_scores: - avg_relevance = sum(relevance_scores.values()) / len(relevance_scores) - factors['relevance'] = avg_relevance * 0.4 - else: - factors['relevance'] = 0.5 # Neutral if we can't score relevance - - # 3. Completeness of analysis - complete_papers = len(research_report.papers) - len(self._find_incomplete_papers(research_report.papers)) - completeness_score = complete_papers / len(research_report.papers) - factors['completeness'] = completeness_score * 0.3 - - # Calculate weighted average - total_score = sum(factors.values()) - return min(total_score, 1.0) - - -class QueryAnalyzer: - """Analyze queries to suggest improvements.""" - - def suggest_broader_terms(self, query: str) -> List[str]: - """Suggest broader search terms for queries that return no results. - - Args: - query: Original query - - Returns: - List of suggested broader terms - """ - suggestions = [] - - # Remove very specific terms - words = query.lower().split() - - # Remove years, specific numbers, very technical terms - broader_words = [] - for word in words: - if not re.match(r'^\d{4}$', word): # Remove years - if not re.match(r'^\d+$', word): # Remove pure numbers - if len(word) > 3: # Keep substantial words - broader_words.append(word) - - if broader_words: - suggestions.append(" ".join(broader_words[:3])) # Use first 3 substantial words - - # Suggest removing modifiers - modifiers = ['recent', 'latest', 'new', 'novel', 'advanced', 'modern', 'current'] - filtered_query = query.lower() - for modifier in modifiers: - filtered_query = filtered_query.replace(modifier, '').strip() - - if filtered_query != query.lower() and filtered_query: - suggestions.append(filtered_query) - - # Suggest core concepts - core_concepts = self._extract_core_concepts(query) - if core_concepts: - suggestions.append(" ".join(core_concepts)) - - return suggestions[:3] # Return top 3 suggestions - - def suggest_query_expansion(self, query: str) -> List[str]: - """Suggest query expansions for insufficient results. - - Args: - query: Original query - - Returns: - List of suggested expanded queries - """ - suggestions = [] - - # Add synonyms and related terms - expansions = { - 'machine learning': ['artificial intelligence', 'deep learning', 'neural networks'], - 'ai': ['artificial intelligence', 'machine learning', 'automation'], - 'nlp': ['natural language processing', 'text analysis', 'language models'], - 'computer vision': ['image processing', 'visual recognition', 'image analysis'], - 'data science': ['analytics', 'big data', 'data mining'], - 'algorithm': ['method', 'approach', 'technique'], - 'model': ['framework', 'system', 'approach'] - } - - query_lower = query.lower() - for term, synonyms in expansions.items(): - if term in query_lower: - for synonym in synonyms: - expanded = query + f" OR {synonym}" - suggestions.append(expanded) - break - - # Add broader research areas - if 'learning' in query_lower: - suggestions.append(query + " OR education OR training") - - if 'analysis' in query_lower: - suggestions.append(query + " OR evaluation OR assessment") - - return suggestions[:3] - - def suggest_more_specific_terms(self, query: str) -> List[str]: - """Suggest more specific terms for better relevance. - - Args: - query: Original query - - Returns: - List of suggested more specific queries - """ - suggestions = [] - - # Add specific modifiers - modifiers = ['recent', 'survey', 'review', 'comparative', 'empirical'] - for modifier in modifiers: - suggestions.append(f"{modifier} {query}") - - # Add specific contexts - contexts = ['applications', 'methods', 'techniques', 'approaches'] - for context in contexts: - suggestions.append(f"{query} {context}") - - return suggestions[:3] - - def _extract_core_concepts(self, query: str) -> List[str]: - """Extract core concepts from a query. - - Args: - query: Query to analyze - - Returns: - List of core concept words - """ - # Simple extraction - remove stop words and keep important terms - stop_words = { - 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', - 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being' - } - - words = re.findall(r'\b\w+\b', query.lower()) - core_words = [word for word in words if word not in stop_words and len(word) > 3] - - return core_words[:3] # Return top 3 core concepts - - -class RelevanceScorer: - """Score paper relevance to queries.""" - - def score_papers( - self, - papers: List['PaperAnalysis'], - query: str - ) -> Dict['PaperAnalysis', float]: - """Score relevance of papers to the original query. - - Args: - papers: List of paper analysis objects - query: Original user query - - Returns: - Dict mapping papers to relevance scores (0.0 to 1.0) - """ - scores = {} - query_terms = self._extract_query_terms(query) - - for paper in papers: - score = self._calculate_paper_relevance(paper, query_terms) - scores[paper] = score - - return scores - - def _extract_query_terms(self, query: str) -> Set[str]: - """Extract important terms from query for relevance scoring. - - Args: - query: User query - - Returns: - Set of important query terms - """ - # Extract words, convert to lowercase, remove stop words - stop_words = { - 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', - 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being', - 'paper', 'papers', 'research', 'study', 'studies' - } - - words = re.findall(r'\b\w+\b', query.lower()) - important_terms = {word for word in words if word not in stop_words and len(word) > 2} - - return important_terms - - def _calculate_paper_relevance( - self, - paper: 'PaperAnalysis', - query_terms: Set[str] - ) -> float: - """Calculate relevance score for a single paper. - - Args: - paper: Paper analysis object - query_terms: Set of important query terms - - Returns: - float: Relevance score (0.0 to 1.0) - """ - if not query_terms: - return 0.5 # Neutral score if no query terms - - # Collect all text from the paper - paper_text = [] - - if paper.title: - paper_text.append(paper.title.lower()) - - if hasattr(paper, 'venue') and paper.venue: - paper_text.append(paper.venue.lower()) - - if paper.key_points: - paper_text.extend([point.lower() for point in paper.key_points]) - - if paper.why_relevant: - paper_text.extend([reason.lower() for reason in paper.why_relevant]) - - # Count term matches - all_paper_text = " ".join(paper_text) - paper_words = set(re.findall(r'\b\w+\b', all_paper_text)) - - # Calculate overlap - matches = query_terms.intersection(paper_words) - - if not matches: - return 0.2 # Low but not zero score for no matches - - # Score based on match ratio and importance - match_ratio = len(matches) / len(query_terms) - - # Boost score if matches are in title or why_relevant - title_matches = 0 - relevance_matches = 0 - - if paper.title: - title_words = set(re.findall(r'\b\w+\b', paper.title.lower())) - title_matches = len(query_terms.intersection(title_words)) - - if paper.why_relevant: - relevance_text = " ".join(paper.why_relevant).lower() - relevance_words = set(re.findall(r'\b\w+\b', relevance_text)) - relevance_matches = len(query_terms.intersection(relevance_words)) - - # Calculate weighted score - base_score = match_ratio * 0.6 - title_boost = (title_matches / len(query_terms)) * 0.3 - relevance_boost = (relevance_matches / len(query_terms)) * 0.1 - - final_score = base_score + title_boost + relevance_boost - return min(final_score, 1.0) - - -class ResultEnhancementEngine: - """Enhance insufficient results through intelligent retry strategies.""" - - def __init__(self, config: EnhancementConfig): - """Initialize result enhancement engine. - - Args: - config: Enhancement configuration - """ - self.config = config - self.query_enhancer = QueryEnhancer() - self.prompt_optimizer = PromptOptimizer() - - async def enhance_insufficient_results( - self, - original_report: 'ResearchReport', - content_issues: List[ContentIssue], - analysis_agent: 'Agent', - original_prompt: str - ) -> EnhancementResult: - """Attempt to enhance results based on content quality issues. - - Args: - original_report: Original research report - content_issues: List of content quality issues - analysis_agent: AI agent for processing - original_prompt: Original prompt that was used - - Returns: - EnhancementResult: Result of enhancement attempts - """ - enhancement_attempts = [] - - for issue in content_issues: - if issue.type == ContentIssueType.EMPTY_RESULTS and self.config.enable_query_broadening: - # Try broader search terms - enhanced_prompt = self.query_enhancer.broaden_query(original_prompt) - enhancement_attempts.append( - EnhancementAttempt( - strategy=EnhancementStrategy.BROADER_QUERY, - enhanced_prompt=enhanced_prompt, - reason="Expanding search terms to find more papers" - ) - ) - - elif issue.type == ContentIssueType.INSUFFICIENT_RESULTS and self.config.enable_query_expansion: - # Try query expansion - enhanced_prompt = self.query_enhancer.expand_query(original_prompt) - enhancement_attempts.append( - EnhancementAttempt( - strategy=EnhancementStrategy.QUERY_EXPANSION, - enhanced_prompt=enhanced_prompt, - reason="Adding related terms to find more relevant papers" - ) - ) - - elif issue.type == ContentIssueType.INCOMPLETE_ANALYSIS and self.config.enable_analysis_enhancement: - # Request more detailed analysis - enhanced_prompt = self.prompt_optimizer.add_analysis_emphasis(original_prompt) - enhancement_attempts.append( - EnhancementAttempt( - strategy=EnhancementStrategy.DETAILED_ANALYSIS, - enhanced_prompt=enhanced_prompt, - reason="Requesting more complete paper analysis" - ) - ) - - # Execute enhancement attempts (limited to prevent loops) - best_result = original_report - improvement_score = 0.0 - - for attempt in enhancement_attempts[:self.config.max_enhancement_attempts]: - try: - # This would need to be implemented to actually retry with the enhanced prompt - # For now, we'll return the original result - pass - except Exception: - continue - - return EnhancementResult( - success=len(enhancement_attempts) > 0, - enhanced_report=best_result, - attempts_made=enhancement_attempts, - improvement_score=improvement_score - ) - - -class QueryEnhancer: - """Enhance queries for better results.""" - - def broaden_query(self, original_prompt: str) -> str: - """Broaden the query in the prompt for more results. - - Args: - original_prompt: Original prompt - - Returns: - str: Enhanced prompt with broader query - """ - # Add instruction to use broader terms - enhancement = ( - "\n\nIMPORTANT: The previous search found no results. " - "Please use broader, more general terms in your analysis. " - "Include related concepts and synonyms. " - "Focus on the core research area rather than specific techniques." - ) - return original_prompt + enhancement - - def expand_query(self, original_prompt: str) -> str: - """Expand the query for more comprehensive results. - - Args: - original_prompt: Original prompt - - Returns: - str: Enhanced prompt with expanded query - """ - enhancement = ( - "\n\nIMPORTANT: Please expand your search to include related topics, " - "alternative approaches, and broader research areas. " - "Look for papers that address similar problems or use related methods." - ) - return original_prompt + enhancement - - -class PromptOptimizer: - """Optimize prompts for better analysis quality.""" - - def add_analysis_emphasis(self, original_prompt: str) -> str: - """Add emphasis for more detailed analysis. - - Args: - original_prompt: Original prompt - - Returns: - str: Enhanced prompt with analysis emphasis - """ - enhancement = ( - "\n\nIMPORTANT: Please provide detailed analysis for each paper. " - "Include at least 3 specific key points and 2 clear reasons why each paper is relevant. " - "Make sure your analysis is comprehensive and informative." - ) - return original_prompt + enhancement \ No newline at end of file diff --git a/src/utils/enhanced_response_handler.py b/src/utils/enhanced_response_handler.py deleted file mode 100644 index bd85613..0000000 --- a/src/utils/enhanced_response_handler.py +++ /dev/null @@ -1,681 +0,0 @@ -# -*- coding: utf-8 -*- -"""Enhanced response handler for graceful model response processing. - -This module implements the main EnhancedResponseHandler that wraps existing -validation and recovery logic with retry capabilities, quality monitoring, -and content quality validation. -""" - -import asyncio -import json -import time -from typing import Optional, Any, Type - -from pydantic import BaseModel -from pydantic_ai import Agent - -from .response_models import ( - ResponseHandlerConfig, RequestContext, ProcessingResult, ProcessingPath, - ValidationResult, RecoveryResult, ContentQualityResult -) -from .retry_manager import RetryManager -from .quality_monitor import QualityMonitor -from .enhanced_validation import enhance_validation_result, should_attempt_retry, get_enhanced_prompt_for_retry, _enhanced_partial_recovery -from .message_formatter import MessageFormatter -from .logging_system import logger -from .model_adaptation import get_adaptation_engine -from .progress_reporter import get_progress_reporter - - -class EnhancedResponseHandler: - """Enhanced wrapper around existing response processing with retry capabilities.""" - - def __init__(self, config: Optional[ResponseHandlerConfig] = None): - """Initialize enhanced response handler. - - Args: - config: Configuration for response handling behavior - """ - self.config = config or ResponseHandlerConfig() - self.retry_manager = RetryManager(self.config.retry_config) - self.quality_monitor = QualityMonitor(self.config.quality_config) - self.adaptation_engine = get_adaptation_engine() - - # Set up logging integration - self.logger = logger - - # Integrate quality monitor with logging system - def log_quality_event(message: str, extra_data: dict) -> None: - """Log quality events through the logging system.""" - self.logger.log_event( - event_type='quality_monitoring', - message=message, - extra_data=extra_data - ) - - self.quality_monitor.set_external_logger(log_quality_event) - - # Import content quality components if enabled - self.content_validator = None - self.enhancement_engine = None - - if self.config.content_quality_config.enable_quality_validation: - try: - from .content_quality import ContentQualityValidator, ResultEnhancementEngine - self.content_validator = ContentQualityValidator(self.config.content_quality_config) - if self.config.content_quality_config.enable_result_enhancement: - self.enhancement_engine = ResultEnhancementEngine(self.config.enhancement_config) - except ImportError: - # Content quality components not yet implemented, continue without them - pass - - async def process_response_with_retries( - self, - analysis_agent: Agent, - prompt: str, - context: RequestContext - ) -> ProcessingResult: - """Process response with retry logic around existing validation. - - Args: - analysis_agent: The AI agent to use for processing - prompt: The prompt to send to the agent - context: Request context information - - Returns: - ProcessingResult: Result of processing with success/failure status - """ - start_time = time.time() - debug_info = { - "original_prompt": prompt, - "retry_attempts": [], - "validation_history": [], - "recovery_attempts": [], - "content_quality_issues": [] - } if self.config.debug_mode else None - - # Log the start of processing - self.logger.log_performance( - operation="response_processing_start", - duration_ms=0, - success=True, - extra_data={ - 'user_query_length': len(context.user_query), - 'model_name': context.model_name, - 'session_id': context.session_id - } - ) - - # Import necessary functions from retrieval helpers - from ..retrieval.helpers_modules.json_extraction import extract_and_clean_json - from ..retrieval.helpers_modules.validation import ( - enhance_validation_with_retry_strategy, - validate_json_structure, - ) - from ..retrieval.helpers_modules.recovery import enhanced_partial_recovery - from ..retrieval.models import ResearchReport - - current_prompt = prompt - - for attempt in range(self.retry_manager.config.max_retries + 1): - try: - # Update context for this attempt - attempt_context = RequestContext( - user_query=context.user_query, - model_name=context.model_name, - attempt_count=attempt, - previous_errors=context.previous_errors.copy(), - timestamp=context.timestamp, - session_id=context.session_id - ) - - # Add retry delay if this is a retry attempt - if attempt > 0: - delay = await self.retry_manager.get_retry_delay(attempt) - if delay > 0: - self.logger.info( - f"Applying retry delay of {delay}s before attempt {attempt + 1}", - extra={ - 'retry_attempt': attempt + 1, - 'delay_seconds': delay, - 'session_id': context.session_id - } - ) - await asyncio.sleep(delay) - - # Enhance prompt with model-specific adjustments on first attempt - if attempt == 0: - current_prompt = self.adaptation_engine.enhance_prompt_for_model(current_prompt, attempt_context) - - # Use existing agent execution - result = await analysis_agent.run(current_prompt) - raw_output = result.output - - if debug_info: - debug_info["retry_attempts"].append({ - "attempt": attempt, - "prompt": current_prompt, - "raw_response": raw_output[:500] + "..." if len(raw_output) > 500 else raw_output - }) - - # Apply model adaptation preprocessing - adaptation_result = self.adaptation_engine.preprocess_response(raw_output, attempt_context) - if adaptation_result.success: - raw_output = adaptation_result.adapted_response - self.logger.debug( - f"Applied model adaptation for {adaptation_result.model_type.value}", - extra={ - 'model_type': adaptation_result.model_type.value, - 'adaptations': [a.value for a in adaptation_result.adaptations_applied], - 'confidence': adaptation_result.confidence_score - } - ) - - # Use existing extraction and validation - try: - clean_json = extract_and_clean_json(raw_output) - parsed = json.loads(clean_json) - - # Use existing validation with enhancement - validation_result = validate_json_structure(parsed) - enhanced_validation = enhance_validation_with_retry_strategy(validation_result) - - if debug_info: - debug_info["validation_history"].append({ - "attempt": attempt, - "is_valid": enhanced_validation.is_valid, - "error_type": enhanced_validation.error_type, - "retry_strategy": enhanced_validation.retry_strategy.value if enhanced_validation.retry_strategy else None - }) - - if enhanced_validation.is_valid: - # Success path - use existing ResearchReport validation - try: - report = ResearchReport.model_validate(parsed) - - # Check content quality if enabled - content_quality_result = None - if self.content_validator: - content_quality_result = self.content_validator.validate_content_quality( - report, context.user_query - ) - - if debug_info and content_quality_result.has_issues: - debug_info["content_quality_issues"].extend([ - {"type": issue.type.value, "message": issue.message} - for issue in content_quality_result.issues - ]) - - # Record content quality issues - if content_quality_result.has_issues: - self.quality_monitor.record_content_quality_issue( - attempt_context, content_quality_result.issues - ) - - # Attempt enhancement if needed and enabled - if (content_quality_result.has_issues and - self.enhancement_engine and - attempt < self.retry_manager.config.max_retries): - - enhancement_result = await self.enhancement_engine.enhance_insufficient_results( - report, content_quality_result.issues, analysis_agent, current_prompt - ) - - self.quality_monitor.record_enhancement_attempt( - attempt_context, enhancement_result - ) - - if enhancement_result.success: - report = enhancement_result.enhanced_report - self.quality_monitor.record_success( - attempt_context, ProcessingPath.CONTENT_ENHANCED_SUCCESS - ) - - return ProcessingResult( - success=True, - data=report, - processing_path=ProcessingPath.CONTENT_ENHANCED_SUCCESS, - quality_score=content_quality_result.overall_quality_score, - debug_info=debug_info - ) - - # Record success - processing_path = ProcessingPath.RETRY_SUCCESS if attempt > 0 else ProcessingPath.DIRECT_SUCCESS - self.quality_monitor.record_success(attempt_context, processing_path) - - # Log successful completion - duration_ms = (time.time() - start_time) * 1000 - self.logger.log_performance( - operation="response_processing_complete", - duration_ms=duration_ms, - success=True, - extra_data={ - 'processing_path': processing_path.value, - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - - return ProcessingResult( - success=True, - data=report, - processing_path=processing_path, - quality_score=content_quality_result.overall_quality_score if content_quality_result else 1.0, - debug_info=debug_info - ) - - except Exception as pydantic_error: - # Pydantic validation failed - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "pydantic_validation", False, True - ) - print(MessageFormatter.validation_error(str(pydantic_error))) - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info - ) - - # Try to enhance prompt for pydantic validation error - current_prompt = self.retry_manager.enhance_prompt_generic( - current_prompt, str(pydantic_error) - ) - continue - - else: - # Validation failed - check if we should retry - if should_attempt_retry(enhanced_validation, attempt, self.retry_manager): - # Record retry attempt - self.quality_monitor.record_retry(attempt_context, enhanced_validation.error_type) - - # Log retry decision - self.logger.warning( - f"Validation failed, attempting retry {attempt + 1}/{self.retry_manager.config.max_retries}", - extra={ - 'error_type': enhanced_validation.error_type, - 'retry_strategy': enhanced_validation.retry_strategy.value if enhanced_validation.retry_strategy else None, - 'attempt': attempt + 1, - 'session_id': context.session_id - } - ) - - # Enhance prompt for retry - current_prompt = get_enhanced_prompt_for_retry( - prompt, enhanced_validation, self.retry_manager, raw_output - ) - - # Add to previous errors - attempt_context.previous_errors.append(enhanced_validation.error_type) - continue - else: - # No more retries - attempt recovery - self.logger.warning( - "Maximum retries reached, attempting recovery", - extra={ - 'error_type': enhanced_validation.error_type, - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - - recovery_result = enhanced_partial_recovery( - raw_output, clean_json, parsed, self.config.recovery_config, context.user_query - ) - - if debug_info: - debug_info["recovery_attempts"].append({ - "success": recovery_result.success, - "method": recovery_result.recovery_method.value if recovery_result.recovery_method else None, - "confidence": recovery_result.confidence_score - }) - - processing_path = ProcessingPath.RECOVERY_SUCCESS if recovery_result.success else ProcessingPath.COMPLETE_FAILURE - - if recovery_result.success: - self.quality_monitor.record_success(attempt_context, processing_path) - self.logger.info( - f"Recovery successful using {recovery_result.recovery_method.value if recovery_result.recovery_method else 'unknown'} method", - extra={ - 'recovery_method': recovery_result.recovery_method.value if recovery_result.recovery_method else None, - 'confidence_score': recovery_result.confidence_score, - 'session_id': context.session_id - } - ) - else: - self.quality_monitor.record_failure( - attempt_context, enhanced_validation.error_type, True, True - ) - self.logger.error( - f"Recovery failed after {attempt + 1} attempts", - extra={ - 'final_error_type': enhanced_validation.error_type, - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - - return ProcessingResult( - success=recovery_result.success, - data=recovery_result.recovered_data, - processing_path=processing_path, - quality_score=recovery_result.confidence_score, - warnings=recovery_result.warnings, - debug_info=debug_info - ) - - except json.JSONDecodeError as e: - # JSON parsing failed - if attempt == self.retry_manager.config.max_retries: - # Final attempt - use existing error handling - self.quality_monitor.record_failure( - attempt_context, "json_syntax", False, True - ) - self.logger.error( - f"JSON parsing failed on final attempt: {str(e)}", - extra={ - 'error_line': e.lineno, - 'error_column': e.colno, - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - print(MessageFormatter.json_syntax_error(str(e), e.lineno, e.colno)) - print(MessageFormatter.debugging_suggestion("syntax")) - print(MessageFormatter.raw_response_header()) - print(raw_output) - - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info - ) - - # Record retry and enhance prompt for syntax retry - self.quality_monitor.record_retry(attempt_context, "json_syntax") - self.logger.warning( - f"JSON syntax error, retrying with enhanced prompt: {str(e)}", - extra={ - 'error_line': e.lineno, - 'error_column': e.colno, - 'attempt': attempt + 1, - 'session_id': context.session_id - } - ) - current_prompt = self.retry_manager.enhance_prompt_for_syntax_error( - current_prompt, str(e) - ) - continue - - except ValueError as extraction_error: - # JSON extraction failed - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "extraction", False, True - ) - self.logger.error( - f"JSON extraction failed on final attempt: {str(extraction_error)}", - extra={ - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - print(MessageFormatter.extraction_error(str(extraction_error))) - print(MessageFormatter.debugging_suggestion("extraction")) - print(MessageFormatter.raw_response_header()) - print(raw_output) - - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info - ) - - # Record retry and enhance prompt for extraction retry - self.quality_monitor.record_retry(attempt_context, "extraction") - self.logger.warning( - f"JSON extraction failed, retrying with enhanced prompt: {str(extraction_error)}", - extra={ - 'attempt': attempt + 1, - 'session_id': context.session_id - } - ) - current_prompt = self.retry_manager.enhance_prompt( - current_prompt, "extraction", raw_output - ) - continue - - except Exception as e: - # Unexpected error - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "unexpected", False, True - ) - self.logger.error( - f"Unexpected error on final attempt: {str(e)}", - extra={ - 'error_type': type(e).__name__, - 'total_attempts': attempt + 1, - 'session_id': context.session_id - } - ) - print(MessageFormatter.parsing_error(str(e))) - print(MessageFormatter.debugging_suggestion("validation")) - - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info - ) - - # Record retry and enhance prompt generically - self.quality_monitor.record_retry(attempt_context, "unexpected") - self.logger.warning( - f"Unexpected error, retrying with enhanced prompt: {str(e)}", - extra={ - 'error_type': type(e).__name__, - 'attempt': attempt + 1, - 'session_id': context.session_id - } - ) - current_prompt = self.retry_manager.enhance_prompt_generic( - current_prompt, str(e) - ) - continue - - # Should not reach here, but handle gracefully - self.logger.error( - "Response processing completed without returning result", - extra={'session_id': context.session_id} - ) - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info - ) - - async def process_structured_response( - self, - agent: Agent, - prompt: str, - context: RequestContext, - response_model: Type[BaseModel], - schema_description: str | None = None, - ) -> ProcessingResult: - """Process an LLM response and validate it against an arbitrary Pydantic model. - - Uses the same retry, recovery, and quality monitoring paths as - :meth:`process_response_with_retries`, but skips ResearchReport-specific - structure validation and content-quality checks. - """ - start_time = time.time() - debug_info = { - "original_prompt": prompt, - "retry_attempts": [], - "validation_history": [], - "recovery_attempts": [], - "response_model": response_model.__name__, - } if self.config.debug_mode else None - - from ..retrieval.helpers_modules.json_extraction import extract_and_clean_json - - schema_hint = schema_description or json.dumps( - { - field: "..." - for field in response_model.model_fields - }, - indent=2, - ) - current_prompt = ( - f"{prompt}\n\nRespond with ONLY valid JSON matching this schema:\n{schema_hint}" - ) - - for attempt in range(self.retry_manager.config.max_retries + 1): - try: - attempt_context = RequestContext( - user_query=context.user_query, - model_name=context.model_name, - attempt_count=attempt, - previous_errors=context.previous_errors.copy(), - timestamp=context.timestamp, - session_id=context.session_id, - ) - - if attempt > 0: - delay = await self.retry_manager.get_retry_delay(attempt) - if delay > 0: - await asyncio.sleep(delay) - - if attempt == 0: - current_prompt = self.adaptation_engine.enhance_prompt_for_model( - current_prompt, attempt_context - ) - - reporter = get_progress_reporter() - llm_label = f"Analyzing ({response_model.__name__})…" - if reporter is not None and reporter.enabled and attempt == 0: - raw_output = await reporter.stream_agent_response( - agent, - current_prompt, - label=llm_label, - ) - else: - result = await agent.run(current_prompt) - raw_output = result.output - - if debug_info is not None: - debug_info["retry_attempts"].append({ - "attempt": attempt, - "prompt": current_prompt, - "raw_response": raw_output[:500] + "..." if len(raw_output) > 500 else raw_output, - }) - - adaptation_result = self.adaptation_engine.preprocess_response( - raw_output, attempt_context - ) - if adaptation_result.success: - raw_output = adaptation_result.adapted_response - - try: - clean_json = extract_and_clean_json(raw_output) - parsed = json.loads(clean_json) - validated = response_model.model_validate(parsed) - - processing_path = ( - ProcessingPath.RETRY_SUCCESS if attempt > 0 else ProcessingPath.DIRECT_SUCCESS - ) - self.quality_monitor.record_success(attempt_context, processing_path) - - duration_ms = (time.time() - start_time) * 1000 - self.logger.log_performance( - operation="structured_response_processing_complete", - duration_ms=duration_ms, - success=True, - extra_data={ - "processing_path": processing_path.value, - "total_attempts": attempt + 1, - "response_model": response_model.__name__, - "session_id": context.session_id, - }, - ) - - return ProcessingResult( - success=True, - data=validated, - processing_path=processing_path, - quality_score=1.0, - debug_info=debug_info, - ) - - except json.JSONDecodeError as exc: - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "json_syntax", False, True - ) - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info, - ) - - self.quality_monitor.record_retry(attempt_context, "json_syntax") - current_prompt = self.retry_manager.enhance_prompt_for_syntax_error( - current_prompt, str(exc) - ) - continue - - except Exception as validation_error: - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "schema", False, True - ) - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info, - ) - - self.quality_monitor.record_retry(attempt_context, "schema") - current_prompt = self.retry_manager.enhance_prompt_generic( - current_prompt, str(validation_error) - ) - continue - - except Exception as exc: - if attempt == self.retry_manager.config.max_retries: - self.quality_monitor.record_failure( - attempt_context, "unexpected", False, True - ) - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info, - ) - - self.quality_monitor.record_retry(attempt_context, "unexpected") - current_prompt = self.retry_manager.enhance_prompt_generic( - current_prompt, str(exc) - ) - continue - - return ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - debug_info=debug_info, - ) - - def get_quality_report(self): - """Get quality monitoring report.""" - return self.quality_monitor.get_quality_report() - - def check_quality_degradation(self) -> bool: - """Check if quality has degraded.""" - return self.quality_monitor.check_quality_degradation() - - def export_metrics(self, filepath=None): - """Export quality metrics.""" - return self.quality_monitor.export_metrics(filepath) - - def is_monitoring_enabled(self) -> bool: - """Check if monitoring is enabled.""" - return self.quality_monitor.is_monitoring_enabled() \ No newline at end of file diff --git a/src/utils/enhanced_validation.py b/src/utils/enhanced_validation.py deleted file mode 100644 index 9936e0b..0000000 --- a/src/utils/enhanced_validation.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- -"""Enhanced validation with retry strategy classification. - -This module extends the existing validation system with retry strategy -classification while preserving all existing validation logic. -""" - -from typing import Optional - -from .response_models import ValidationResult, RetryStrategy, RecoveryResult, RecoveryConfig -from .retry_manager import RetryManager - - -def _classify_error_for_retry(validation_result: ValidationResult) -> Optional[RetryStrategy]: - """Classify validation errors to determine optimal retry strategy. - - Extends existing validation with retry-specific classification. - - Args: - validation_result: Result from existing _validate_json_structure - - Returns: - RetryStrategy: Recommended retry strategy, or None if no retry recommended - """ - if not validation_result.is_valid: - if validation_result.error_type == "syntax": - return RetryStrategy.SYNTAX_EMPHASIS - elif validation_result.error_type == "schema": - return RetryStrategy.SCHEMA_EXAMPLE - elif validation_result.error_type == "field_type": - return RetryStrategy.TYPE_CLARIFICATION - elif validation_result.error_type == "structure": - return RetryStrategy.FORMAT_INSTRUCTION - else: - return RetryStrategy.GENERIC_CLARIFICATION - - return None - - -def enhance_validation_result(validation_result: ValidationResult) -> ValidationResult: - """Enhance existing validation result with retry strategy classification. - - This function takes the result from the existing _validate_json_structure - function and adds retry strategy information without changing the core - validation logic. - - Args: - validation_result: Original validation result - - Returns: - ValidationResult: Enhanced validation result with retry strategy - """ - # Create enhanced result with retry strategy - enhanced_result = ValidationResult( - is_valid=validation_result.is_valid, - error_message=validation_result.error_message, - error_type=validation_result.error_type, - show_raw_response=validation_result.show_raw_response, - retry_strategy=_classify_error_for_retry(validation_result) - ) - - return enhanced_result - - -def should_attempt_retry(validation_result: ValidationResult, attempt_count: int, retry_manager: RetryManager) -> bool: - """Determine if retry should be attempted based on validation result. - - Args: - validation_result: Result from validation - attempt_count: Current attempt number - retry_manager: Retry manager instance - - Returns: - bool: True if retry should be attempted - """ - if validation_result.is_valid: - return False - - return retry_manager.should_retry(validation_result.error_type, attempt_count) - - -def get_enhanced_prompt_for_retry( - original_prompt: str, - validation_result: ValidationResult, - retry_manager: RetryManager, - previous_response: str = "" -) -> str: - """Get enhanced prompt for retry based on validation result. - - Args: - original_prompt: The original prompt that failed - validation_result: Validation result with error details - retry_manager: Retry manager instance - previous_response: Previous response that failed - - Returns: - str: Enhanced prompt for retry - """ - return retry_manager.enhance_prompt( - original_prompt, - validation_result.error_type, - previous_response - ) - - -def _enhanced_partial_recovery( - raw_output: str, - clean_json: str, - parsed_data, - config: RecoveryConfig, - user_query: str = "" -) -> RecoveryResult: - """Enhanced partial recovery with fallback processing. - - This function wraps the existing _attempt_partial_recovery with enhanced - fallback processing capabilities. - - Args: - raw_output: Original model output - clean_json: Cleaned JSON string - parsed_data: Partially parsed data - config: Recovery configuration - user_query: Original user query for context - - Returns: - RecoveryResult: Enhanced recovery result - """ - try: - from .fallback_processing import enhance_partial_recovery_with_fallback - return enhance_partial_recovery_with_fallback( - raw_output, clean_json, parsed_data, config, user_query - ) - except ImportError: - # Fallback processing not available, use existing recovery - from ..retrieval.helpers_modules.recovery import enhanced_partial_recovery - return enhanced_partial_recovery(raw_output, clean_json, parsed_data) \ No newline at end of file diff --git a/src/utils/fallback_processing.py b/src/utils/fallback_processing.py deleted file mode 100644 index 1b530f0..0000000 --- a/src/utils/fallback_processing.py +++ /dev/null @@ -1,590 +0,0 @@ -# -*- coding: utf-8 -*- -"""Fallback processing for complete response failures. - -This module provides enhanced fallback mechanisms when structured JSON parsing -completely fails. It attempts to extract useful information from unstructured -text responses and present it in a helpful format to users. -""" - -import re -import json -from typing import List, Dict, Any, Optional, Tuple -from dataclasses import dataclass -from enum import Enum - -from .response_models import RecoveryConfig, RecoveryMethod, RecoveryResult -from .logging_system import logger - - -class FallbackMethod(Enum): - """Methods for fallback processing.""" - TEXT_EXTRACTION = "text_extraction" - PATTERN_MATCHING = "pattern_matching" - KEYWORD_ANALYSIS = "keyword_analysis" - STRUCTURED_SUMMARY = "structured_summary" - - -@dataclass -class FallbackResult: - """Result of fallback processing.""" - success: bool - method: FallbackMethod - extracted_content: Dict[str, Any] - confidence_score: float - warnings: List[str] - raw_sections: List[str] - - -class TextFallbackProcessor: - """Process unstructured text responses as fallback.""" - - def __init__(self, config: Optional[RecoveryConfig] = None): - """Initialize text fallback processor. - - Args: - config: Recovery configuration - """ - self.config = config or RecoveryConfig() - self.logger = logger - - def process_unstructured_response( - self, - raw_response: str, - user_query: str - ) -> FallbackResult: - """Process unstructured response text to extract useful information. - - Args: - raw_response: The raw model response - user_query: Original user query for context - - Returns: - FallbackResult: Extracted information and metadata - """ - self.logger.info( - "Attempting fallback processing for unstructured response", - extra={ - 'response_length': len(raw_response), - 'query_length': len(user_query) - } - ) - - # Try different fallback methods in order of preference - methods = [ - self._extract_paper_mentions, - self._extract_key_findings, - self._extract_research_themes, - self._create_structured_summary - ] - - best_result = None - best_confidence = 0.0 - - for method in methods: - try: - result = method(raw_response, user_query) - if result.success and result.confidence_score > best_confidence: - best_result = result - best_confidence = result.confidence_score - except Exception as e: - self.logger.warning( - f"Fallback method {method.__name__} failed: {str(e)}", - extra={'method': method.__name__} - ) - continue - - if best_result is None: - # Create minimal fallback result - best_result = FallbackResult( - success=False, - method=FallbackMethod.TEXT_EXTRACTION, - extracted_content={ - "summary": "Unable to extract structured information from response", - "raw_text": raw_response[:1000] + "..." if len(raw_response) > 1000 else raw_response - }, - confidence_score=0.1, - warnings=["Could not extract structured information"], - raw_sections=[raw_response] - ) - - return best_result - - def _extract_paper_mentions(self, response: str, query: str) -> FallbackResult: - """Extract paper mentions from unstructured text. - - Args: - response: Raw response text - query: User query - - Returns: - FallbackResult: Extracted paper information - """ - papers = [] - warnings = [] - - # Look for paper title patterns - title_patterns = [ - r'"([^"]{10,100})"', # Quoted titles - r'titled?\s+"([^"]{10,100})"', # "titled X" - r'paper\s+"([^"]{10,100})"', # "paper X" - r'study\s+"([^"]{10,100})"', # "study X" - r'article\s+"([^"]{10,100})"', # "article X" - ] - - found_titles = set() - for pattern in title_patterns: - matches = re.finditer(pattern, response, re.IGNORECASE) - for match in matches: - title = match.group(1).strip() - if len(title) > 10 and title not in found_titles: - found_titles.add(title) - - # Look for author patterns - author_patterns = [ - r'by\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*(?:\s+et\s+al\.?)?)', - r'authors?\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*(?:\s+et\s+al\.?)?)', - ] - - found_authors = set() - for pattern in author_patterns: - matches = re.finditer(pattern, response, re.IGNORECASE) - for match in matches: - author = match.group(1).strip() - if len(author) > 3 and author not in found_authors: - found_authors.add(author) - - # Look for year patterns - year_pattern = r'\b(19|20)\d{2}\b' - found_years = set() - for match in re.finditer(year_pattern, response): - year = int(match.group(0)) - if 1990 <= year <= 2024: - found_years.add(year) - - # Create paper entries from found information - for i, title in enumerate(list(found_titles)[:5]): # Limit to 5 papers - paper = { - "title": title, - "authors": list(found_authors)[i] if i < len(found_authors) else "Unknown", - "year": list(found_years)[i] if i < len(found_years) else None, - "key_points": self._extract_key_points_for_paper(response, title), - "why_relevant": f"Mentioned in response to query about {query}", - "confidence": "low", - "source": "text_extraction" - } - papers.append(paper) - - if not papers: - warnings.append("No clear paper mentions found in response") - - confidence = min(0.7, len(papers) * 0.2) if papers else 0.1 - - return FallbackResult( - success=len(papers) > 0, - method=FallbackMethod.PATTERN_MATCHING, - extracted_content={ - "papers": papers, - "extraction_method": "pattern_matching", - "total_found": len(papers) - }, - confidence_score=confidence, - warnings=warnings, - raw_sections=[response] - ) - - def _extract_key_points_for_paper(self, response: str, title: str) -> List[str]: - """Extract key points related to a specific paper. - - Args: - response: Full response text - title: Paper title to find context for - - Returns: - List[str]: Key points related to the paper - """ - # Find sentences containing the title or nearby - sentences = re.split(r'[.!?]+', response) - relevant_sentences = [] - - for i, sentence in enumerate(sentences): - if title.lower() in sentence.lower(): - # Include this sentence and nearby ones - start = max(0, i - 1) - end = min(len(sentences), i + 2) - relevant_sentences.extend(sentences[start:end]) - - # Extract key points from relevant sentences - key_points = [] - for sentence in relevant_sentences: - sentence = sentence.strip() - if len(sentence) > 20 and any(word in sentence.lower() for word in - ['shows', 'demonstrates', 'finds', 'concludes', 'reports', 'indicates']): - key_points.append(sentence) - - return key_points[:3] # Limit to 3 key points - - def _extract_key_findings(self, response: str, query: str) -> FallbackResult: - """Extract key findings from unstructured text. - - Args: - response: Raw response text - query: User query - - Returns: - FallbackResult: Extracted findings - """ - findings = [] - warnings = [] - - # Look for finding indicators - finding_patterns = [ - r'(found that [^.!?]{20,200})', - r'(shows that [^.!?]{20,200})', - r'(demonstrates [^.!?]{20,200})', - r'(indicates [^.!?]{20,200})', - r'(suggests [^.!?]{20,200})', - r'(concludes [^.!?]{20,200})', - r'(reveals [^.!?]{20,200})', - ] - - for pattern in finding_patterns: - matches = re.finditer(pattern, response, re.IGNORECASE) - for match in matches: - finding = match.group(1).strip() - if len(finding) > 30: - findings.append(finding) - - # Remove duplicates and limit - unique_findings = list(dict.fromkeys(findings))[:10] - - if not unique_findings: - warnings.append("No clear research findings identified") - - confidence = min(0.6, len(unique_findings) * 0.1) if unique_findings else 0.1 - - return FallbackResult( - success=len(unique_findings) > 0, - method=FallbackMethod.KEYWORD_ANALYSIS, - extracted_content={ - "key_findings": unique_findings, - "total_findings": len(unique_findings), - "extraction_method": "keyword_analysis" - }, - confidence_score=confidence, - warnings=warnings, - raw_sections=[response] - ) - - def _extract_research_themes(self, response: str, query: str) -> FallbackResult: - """Extract research themes and topics from text. - - Args: - response: Raw response text - query: User query - - Returns: - FallbackResult: Extracted themes - """ - warnings = [] - - # Look for research-related keywords - research_keywords = [ - 'machine learning', 'artificial intelligence', 'deep learning', - 'neural networks', 'natural language processing', 'computer vision', - 'data mining', 'big data', 'algorithm', 'model', 'training', - 'classification', 'regression', 'clustering', 'optimization', - 'performance', 'accuracy', 'evaluation', 'dataset', 'experiment' - ] - - found_themes = {} - for keyword in research_keywords: - count = len(re.findall(keyword, response, re.IGNORECASE)) - if count > 0: - found_themes[keyword] = count - - # Sort by frequency - sorted_themes = sorted(found_themes.items(), key=lambda x: x[1], reverse=True) - top_themes = [theme for theme, count in sorted_themes[:8]] - - # Extract sentences containing top themes - theme_contexts = {} - for theme in top_themes[:3]: # Top 3 themes - sentences = [] - for sentence in re.split(r'[.!?]+', response): - if theme.lower() in sentence.lower() and len(sentence.strip()) > 20: - sentences.append(sentence.strip()) - theme_contexts[theme] = sentences[:2] # Max 2 sentences per theme - - if not top_themes: - warnings.append("No clear research themes identified") - - confidence = min(0.5, len(top_themes) * 0.08) if top_themes else 0.1 - - return FallbackResult( - success=len(top_themes) > 0, - method=FallbackMethod.KEYWORD_ANALYSIS, - extracted_content={ - "research_themes": top_themes, - "theme_contexts": theme_contexts, - "theme_frequencies": dict(sorted_themes[:5]), - "extraction_method": "theme_analysis" - }, - confidence_score=confidence, - warnings=warnings, - raw_sections=[response] - ) - - def _create_structured_summary(self, response: str, query: str) -> FallbackResult: - """Create a structured summary of the response. - - Args: - response: Raw response text - query: User query - - Returns: - FallbackResult: Structured summary - """ - warnings = [] - - # Split into paragraphs - paragraphs = [p.strip() for p in response.split('\n\n') if p.strip()] - if not paragraphs: - paragraphs = [p.strip() for p in response.split('\n') if p.strip()] - - # Identify different sections - summary_sections = { - "introduction": [], - "main_content": [], - "conclusions": [] - } - - for i, paragraph in enumerate(paragraphs): - if len(paragraph) < 20: - continue - - # Classify paragraph - if i == 0 or any(word in paragraph.lower() for word in ['introduction', 'overview', 'background']): - summary_sections["introduction"].append(paragraph) - elif i >= len(paragraphs) - 2 or any(word in paragraph.lower() for word in ['conclusion', 'summary', 'in summary']): - summary_sections["conclusions"].append(paragraph) - else: - summary_sections["main_content"].append(paragraph) - - # Create summary - structured_summary = {} - - if summary_sections["introduction"]: - structured_summary["overview"] = summary_sections["introduction"][0][:300] + "..." - - if summary_sections["main_content"]: - structured_summary["key_content"] = [ - p[:200] + "..." if len(p) > 200 else p - for p in summary_sections["main_content"][:3] - ] - - if summary_sections["conclusions"]: - structured_summary["conclusions"] = summary_sections["conclusions"][0][:300] + "..." - - # Add metadata - structured_summary["word_count"] = len(response.split()) - structured_summary["paragraph_count"] = len(paragraphs) - structured_summary["query_context"] = query - - if not any(summary_sections.values()): - warnings.append("Could not identify clear document structure") - - confidence = 0.3 if any(summary_sections.values()) else 0.1 - - return FallbackResult( - success=True, # Always succeeds as a last resort - method=FallbackMethod.STRUCTURED_SUMMARY, - extracted_content={ - "structured_summary": structured_summary, - "extraction_method": "structured_summary", - "sections_found": {k: len(v) for k, v in summary_sections.items()} - }, - confidence_score=confidence, - warnings=warnings, - raw_sections=paragraphs - ) - - -def create_fallback_research_report(fallback_result: FallbackResult, user_query: str) -> Dict[str, Any]: - """Create a research report structure from fallback processing results. - - Args: - fallback_result: Result from fallback processing - user_query: Original user query - - Returns: - Dict[str, Any]: Research report structure with fallback data - """ - report = { - "query": user_query, - "papers": [], - "summary": "", - "metadata": { - "processing_method": "fallback", - "fallback_method": fallback_result.method.value, - "confidence": fallback_result.confidence_score, - "warnings": fallback_result.warnings, - "data_completeness": "partial" - } - } - - content = fallback_result.extracted_content - - # Handle different types of extracted content - if "papers" in content: - report["papers"] = content["papers"] - report["summary"] = f"Extracted {len(content['papers'])} paper references from unstructured response." - - elif "key_findings" in content: - # Convert findings to paper-like structure - for i, finding in enumerate(content["key_findings"][:5]): - paper = { - "title": f"Finding {i+1}", - "key_points": [finding], - "why_relevant": "Extracted from model response", - "confidence": "low", - "source": "text_extraction" - } - report["papers"].append(paper) - report["summary"] = f"Extracted {len(content['key_findings'])} key findings from response." - - elif "research_themes" in content: - # Convert themes to paper-like structure - for theme in content["research_themes"][:5]: - contexts = content.get("theme_contexts", {}).get(theme, []) - paper = { - "title": f"Research Theme: {theme.title()}", - "key_points": contexts, - "why_relevant": f"Identified as relevant theme for query about {user_query}", - "confidence": "low", - "source": "theme_extraction" - } - report["papers"].append(paper) - report["summary"] = f"Identified {len(content['research_themes'])} research themes." - - elif "structured_summary" in content: - summary_data = content["structured_summary"] - - # Create a single "paper" from the summary - paper = { - "title": "Response Summary", - "key_points": summary_data.get("key_content", []), - "why_relevant": "Summary of model response", - "confidence": "low", - "source": "summary_extraction" - } - - if "overview" in summary_data: - paper["abstract"] = summary_data["overview"] - - if "conclusions" in summary_data: - paper["conclusions"] = summary_data["conclusions"] - - report["papers"] = [paper] - report["summary"] = "Created structured summary from unstructured response." - - # Add fallback indicators - if report["papers"]: - for paper in report["papers"]: - paper["_fallback_processed"] = True - paper["_original_confidence"] = fallback_result.confidence_score - - return report - - -def enhance_partial_recovery_with_fallback( - raw_output: str, - clean_json: str, - parsed_data: Any, - config: RecoveryConfig, - user_query: str -) -> RecoveryResult: - """Enhance existing partial recovery with fallback processing. - - Args: - raw_output: Original model output - clean_json: Cleaned JSON string (may be invalid) - parsed_data: Partially parsed data (may be incomplete) - config: Recovery configuration - user_query: Original user query for context - - Returns: - RecoveryResult: Enhanced recovery result with fallback data - """ - # First try existing recovery methods - from ..retrieval.helpers_modules.recovery import enhanced_partial_recovery - - try: - existing_result = enhanced_partial_recovery(raw_output, clean_json, parsed_data) - if existing_result.success and existing_result.confidence_score > 0.5: - logger.info("Existing recovery method succeeded, using that result") - return existing_result - except Exception as e: - logger.warning(f"Existing recovery method failed: {str(e)}") - - # If existing recovery failed or has low confidence, try fallback processing - logger.info("Attempting enhanced fallback processing") - - fallback_processor = TextFallbackProcessor(config) - fallback_result = fallback_processor.process_unstructured_response(raw_output, user_query) - - if fallback_result.success: - # Create research report from fallback data - fallback_report = create_fallback_research_report(fallback_result, user_query) - - # Convert to expected format (ResearchReport-like structure) - try: - from ..retrieval.models import ResearchReport, PaperAnalysis - - # Convert fallback papers to PaperAnalysis objects - papers = [] - for paper_data in fallback_report["papers"]: - paper = PaperAnalysis( - title=paper_data.get("title", "Unknown Title"), - authors=paper_data.get("authors", "Unknown"), - year=paper_data.get("year"), - venue=paper_data.get("venue"), - url=paper_data.get("url"), - doi=paper_data.get("doi"), - key_points=paper_data.get("key_points", []), - why_relevant=paper_data.get("why_relevant", "Extracted from response") - ) - papers.append(paper) - - recovered_report = ResearchReport( - query=user_query, - papers=papers - ) - - return RecoveryResult( - success=True, - recovered_data=recovered_report, - recovery_method=RecoveryMethod.FALLBACK_PROCESSING, - confidence_score=fallback_result.confidence_score, - warnings=fallback_result.warnings + ["Data extracted using fallback processing - completeness not guaranteed"] - ) - - except Exception as e: - logger.warning(f"Failed to create ResearchReport from fallback data: {str(e)}") - - # Return raw fallback data if ResearchReport creation fails - return RecoveryResult( - success=True, - recovered_data=fallback_report, - recovery_method=RecoveryMethod.FALLBACK_PROCESSING, - confidence_score=fallback_result.confidence_score * 0.8, # Reduce confidence for raw data - warnings=fallback_result.warnings + ["Returning raw extracted data - structure may be incomplete"] - ) - - # If all recovery methods fail - return RecoveryResult( - success=False, - recovered_data=None, - recovery_method=RecoveryMethod.FALLBACK_PROCESSING, - confidence_score=0.0, - warnings=["All recovery methods failed, including fallback processing"] - ) \ No newline at end of file diff --git a/src/utils/message_formatter.py b/src/utils/message_formatter.py index 9496595..060f6b5 100644 --- a/src/utils/message_formatter.py +++ b/src/utils/message_formatter.py @@ -302,84 +302,6 @@ def api_max_retries_message(cls, service: str) -> str: """ return f"{service}: Max retries exceeded. Skipping this source." - @classmethod - def retry_attempt_message(cls, attempt: int, max_retries: int, error_type: str) -> str: - """Format retry attempt messages. - - Args: - attempt: Current attempt number. - max_retries: Maximum number of retries. - error_type: Type of error that triggered retry. - - Returns: - str: Formatted retry message. - """ - return ( - f"⟳ Retry attempt {attempt}/{max_retries}\n" - f" Error: {error_type}\n" - f" Adjusting request and trying again..." - ) - - @classmethod - def recovery_attempt_message(cls) -> str: - """Format recovery attempt messages. - - Returns: - str: Formatted recovery message. - """ - return ( - "⚠ Unable to parse complete response.\n" - " Attempting partial recovery from available data..." - ) - - @classmethod - def content_quality_warning(cls, issues: list[str]) -> str: - """Format content quality warning messages. - - Args: - issues: List of content quality issues. - - Returns: - str: Formatted warning message. - """ - issues_text = "\n • ".join(issues[:3]) # Show top 3 issues - return ( - f"⚠ Content quality issues detected:\n" - f" • {issues_text}\n" - f" Results may be incomplete or require refinement." - ) - - @classmethod - def enhancement_attempt_message(cls, strategy: str) -> str: - """Format result enhancement attempt messages. - - Args: - strategy: Enhancement strategy being used. - - Returns: - str: Formatted enhancement message. - """ - return ( - f"✓ Attempting to enhance results using: {strategy}\n" - f" Refining query and requesting additional analysis..." - ) - - @classmethod - def enhancement_success_message(cls, improvement: float) -> str: - """Format successful enhancement messages. - - Args: - improvement: Improvement score (0.0-1.0). - - Returns: - str: Formatted success message. - """ - percentage = int(improvement * 100) - return ( - f"✓ Results enhanced successfully\n" - f" Quality improvement: +{percentage}%" - ) - @classmethod def partial_success_message(cls, complete_count: int, total_count: int) -> str: """Format partial success messages. @@ -414,36 +336,6 @@ def query_suggestion_message(cls, suggestions: list[str]) -> str: f" Try refining your query with these terms for more relevant results." ) - @classmethod - def model_adaptation_message(cls, model_type: str) -> str: - """Format model adaptation messages. - - Args: - model_type: Type of model being adapted for. - - Returns: - str: Formatted adaptation message. - """ - return f"🔧 Adapting response format for {model_type} model..." - - @classmethod - def fallback_processing_message(cls, method: str, confidence: float) -> str: - """Format fallback processing messages. - - Args: - method: Fallback processing method used. - confidence: Confidence score (0.0-1.0). - - Returns: - str: Formatted fallback message. - """ - confidence_pct = int(confidence * 100) - return ( - f"⚠ Using fallback processing: {method}\n" - f" Confidence: {confidence_pct}%\n" - f" Results extracted from unstructured response." - ) - @classmethod def debug_info_header(cls) -> str: """Format debug information header. diff --git a/src/utils/model_adaptation.py b/src/utils/model_adaptation.py deleted file mode 100644 index ec4352a..0000000 --- a/src/utils/model_adaptation.py +++ /dev/null @@ -1,521 +0,0 @@ -# -*- coding: utf-8 -*- -"""Model adaptation for handling different AI model response patterns. - -This module provides capabilities to detect and adapt to different AI model -response patterns, preprocessing quirks, and output formats to improve -parsing success rates across different models. -""" - -import re -import json -from typing import Dict, List, Optional, Any, Tuple -from dataclasses import dataclass -from enum import Enum - -from .response_models import RequestContext -from .logging_system import logger - - -class ModelType(Enum): - """Known model types with specific adaptation patterns.""" - OPENAI_GPT = "openai_gpt" - ANTHROPIC_CLAUDE = "anthropic_claude" - GOOGLE_GEMINI = "google_gemini" - OLLAMA_LOCAL = "ollama_local" - GENERIC = "generic" - - -class AdaptationStrategy(Enum): - """Strategies for model adaptation.""" - RESPONSE_CLEANING = "response_cleaning" - FORMAT_NORMALIZATION = "format_normalization" - SCHEMA_ADJUSTMENT = "schema_adjustment" - PROMPT_MODIFICATION = "prompt_modification" - - -@dataclass -class ModelPattern: - """Pattern definition for a specific model.""" - model_type: ModelType - name_patterns: List[str] # Regex patterns to match model names - response_quirks: List[str] # Known response formatting issues - cleaning_rules: List[Tuple[str, str]] # (pattern, replacement) pairs - schema_preferences: Dict[str, Any] # Preferred schema formats - prompt_adjustments: Dict[str, str] # Prompt modifications - - -@dataclass -class AdaptationResult: - """Result of model adaptation processing.""" - success: bool - adapted_response: str - adaptations_applied: List[AdaptationStrategy] - model_type: ModelType - confidence_score: float - warnings: List[str] - - -class ModelAdaptationEngine: - """Engine for adapting responses based on model-specific patterns.""" - - def __init__(self): - """Initialize model adaptation engine.""" - self.logger = logger - self.model_patterns = self._initialize_model_patterns() - self.adaptation_cache: Dict[str, ModelType] = {} - - def _initialize_model_patterns(self) -> Dict[ModelType, ModelPattern]: - """Initialize known model patterns and adaptations. - - Returns: - Dict[ModelType, ModelPattern]: Model patterns by type - """ - patterns = {} - - # OpenAI GPT patterns - patterns[ModelType.OPENAI_GPT] = ModelPattern( - model_type=ModelType.OPENAI_GPT, - name_patterns=[ - r"gpt-[34]\.?5?-?turbo", - r"gpt-[34]", - r"openai", - r"chatgpt" - ], - response_quirks=[ - "Sometimes adds explanatory text before JSON", - "May include markdown code blocks", - "Occasionally adds trailing comments" - ], - cleaning_rules=[ - (r"```json\s*", ""), # Remove markdown JSON blocks - (r"```\s*$", ""), # Remove closing markdown - (r"^[^{]*({.*})[^}]*$", r"\1"), # Extract JSON from surrounding text - (r"//.*$", ""), # Remove trailing comments - ], - schema_preferences={ - "strict_types": True, - "required_fields": True, - "array_format": "standard" - }, - prompt_adjustments={ - "json_emphasis": "Please respond with ONLY valid JSON, no additional text or formatting.", - "schema_reminder": "Ensure your response follows the exact JSON schema provided." - } - ) - - # Anthropic Claude patterns - patterns[ModelType.ANTHROPIC_CLAUDE] = ModelPattern( - model_type=ModelType.ANTHROPIC_CLAUDE, - name_patterns=[ - r"claude-[123]", - r"claude-instant", - r"anthropic", - r"claude" - ], - response_quirks=[ - "Often provides detailed explanations", - "May structure responses with headers", - "Sometimes uses XML-like tags" - ], - cleaning_rules=[ - (r".*?", ""), # Remove thinking tags - (r"\s*", ""), # Remove XML-style JSON tags - (r"\s*", ""), - (r"^.*?Here's the JSON.*?:\s*", ""), # Remove explanatory prefixes - (r"^.*?(?=\{)", ""), # Remove text before first { - ], - schema_preferences={ - "strict_types": True, - "detailed_descriptions": True, - "nested_objects": True - }, - prompt_adjustments={ - "json_emphasis": "Respond with only the JSON object, without any explanatory text or tags.", - "format_instruction": "Do not include tags or explanations, just the JSON." - } - ) - - # Google Gemini patterns - patterns[ModelType.GOOGLE_GEMINI] = ModelPattern( - model_type=ModelType.GOOGLE_GEMINI, - name_patterns=[ - r"gemini-pro", - r"gemini-[0-9]", - r"google", - r"bard" - ], - response_quirks=[ - "May include safety disclaimers", - "Sometimes formats with bullet points", - "Can include multiple JSON objects" - ], - cleaning_rules=[ - (r"^\*\*.*?\*\*\s*", ""), # Remove bold headers - (r"^- .*?:\s*", ""), # Remove bullet points - (r"I cannot.*?However,?\s*", ""), # Remove safety disclaimers - (r"(?<=})\s*{", "},{"), # Fix multiple JSON objects - ], - schema_preferences={ - "flexible_types": True, - "optional_fields": True, - "simple_structure": True - }, - prompt_adjustments={ - "json_emphasis": "Return only a single valid JSON object with no additional formatting.", - "safety_note": "This is for research purposes. Please provide the requested JSON format." - } - ) - - # Ollama/Local model patterns - patterns[ModelType.OLLAMA_LOCAL] = ModelPattern( - model_type=ModelType.OLLAMA_LOCAL, - name_patterns=[ - r"llama", - r"mistral", - r"codellama", - r"ollama", - r"local" - ], - response_quirks=[ - "May have inconsistent formatting", - "Sometimes includes model artifacts", - "Can have encoding issues" - ], - cleaning_rules=[ - (r"<\|.*?\|>", ""), # Remove special tokens - (r"###.*?###", ""), # Remove section headers - (r"^\s*\d+\.\s*", ""), # Remove numbered lists - (r"[^\x00-\x7F]+", ""), # Remove non-ASCII characters - ], - schema_preferences={ - "simple_types": True, - "minimal_nesting": True, - "string_fallbacks": True - }, - prompt_adjustments={ - "json_emphasis": "Output valid JSON only. No explanations or additional text.", - "format_strict": "Use double quotes for all strings. Ensure proper JSON syntax." - } - ) - - return patterns - - def detect_model_type(self, context: RequestContext) -> ModelType: - """Detect model type from context information. - - Args: - context: Request context with model information - - Returns: - ModelType: Detected model type - """ - model_name = context.model_name.lower() - - # Check cache first - if model_name in self.adaptation_cache: - return self.adaptation_cache[model_name] - - # Try to match against known patterns - for model_type, pattern in self.model_patterns.items(): - for name_pattern in pattern.name_patterns: - if re.search(name_pattern, model_name, re.IGNORECASE): - self.adaptation_cache[model_name] = model_type - self.logger.info( - f"Detected model type {model_type.value} for {model_name}", - extra={'model_name': model_name, 'detected_type': model_type.value} - ) - return model_type - - # Default to generic if no match - self.adaptation_cache[model_name] = ModelType.GENERIC - self.logger.info( - f"Using generic adaptation for unknown model {model_name}", - extra={'model_name': model_name} - ) - return ModelType.GENERIC - - def preprocess_response( - self, - raw_response: str, - context: RequestContext - ) -> AdaptationResult: - """Preprocess response based on detected model type. - - Args: - raw_response: Raw model response - context: Request context - - Returns: - AdaptationResult: Preprocessed response with adaptations applied - """ - model_type = self.detect_model_type(context) - adaptations_applied = [] - warnings = [] - - # Start with original response - adapted_response = raw_response - - # Apply model-specific cleaning rules - if model_type in self.model_patterns: - pattern = self.model_patterns[model_type] - - self.logger.debug( - f"Applying {len(pattern.cleaning_rules)} cleaning rules for {model_type.value}", - extra={'model_type': model_type.value, 'rules_count': len(pattern.cleaning_rules)} - ) - - for rule_pattern, replacement in pattern.cleaning_rules: - try: - before_length = len(adapted_response) - adapted_response = re.sub(rule_pattern, replacement, adapted_response, flags=re.DOTALL | re.MULTILINE) - after_length = len(adapted_response) - - if before_length != after_length: - adaptations_applied.append(AdaptationStrategy.RESPONSE_CLEANING) - self.logger.debug( - f"Applied cleaning rule: {rule_pattern[:50]}...", - extra={ - 'rule_pattern': rule_pattern, - 'length_change': after_length - before_length - } - ) - except re.error as e: - warnings.append(f"Failed to apply cleaning rule {rule_pattern}: {str(e)}") - self.logger.warning( - f"Regex error in cleaning rule: {str(e)}", - extra={'rule_pattern': rule_pattern} - ) - - # Apply generic cleaning if no specific patterns or if generic type - if model_type == ModelType.GENERIC or not adaptations_applied: - adapted_response = self._apply_generic_cleaning(adapted_response) - if adapted_response != raw_response: - adaptations_applied.append(AdaptationStrategy.RESPONSE_CLEANING) - - # Calculate confidence based on adaptations and response quality - confidence_score = self._calculate_adaptation_confidence( - raw_response, adapted_response, adaptations_applied - ) - - return AdaptationResult( - success=len(adaptations_applied) > 0 or adapted_response != raw_response, - adapted_response=adapted_response, - adaptations_applied=adaptations_applied, - model_type=model_type, - confidence_score=confidence_score, - warnings=warnings - ) - - def _apply_generic_cleaning(self, response: str) -> str: - """Apply generic cleaning rules for unknown models. - - Args: - response: Raw response text - - Returns: - str: Cleaned response - """ - # Generic cleaning rules that work for most models - generic_rules = [ - (r"```(?:json)?\s*", ""), # Remove code blocks - (r"```\s*$", ""), # Remove closing code blocks - (r"^[^{\[]*", ""), # Remove text before JSON/array - (r"[^}\]]*$", ""), # Remove text after JSON/array - (r"\n\s*\n", "\n"), # Collapse multiple newlines - (r"^\s+|\s+$", ""), # Trim whitespace - ] - - cleaned = response - for pattern, replacement in generic_rules: - try: - cleaned = re.sub(pattern, replacement, cleaned, flags=re.DOTALL | re.MULTILINE) - except re.error: - continue # Skip problematic patterns - - return cleaned - - def _calculate_adaptation_confidence( - self, - original: str, - adapted: str, - adaptations: List[AdaptationStrategy] - ) -> float: - """Calculate confidence score for adaptation result. - - Args: - original: Original response - adapted: Adapted response - adaptations: List of adaptations applied - - Returns: - float: Confidence score between 0.0 and 1.0 - """ - base_confidence = 0.5 - - # Increase confidence if adaptations were applied - if adaptations: - base_confidence += 0.2 * len(set(adaptations)) - - # Increase confidence if response looks more JSON-like after adaptation - if self._looks_like_json(adapted) and not self._looks_like_json(original): - base_confidence += 0.3 - - # Decrease confidence if response became much shorter (might have removed too much) - length_ratio = len(adapted) / max(len(original), 1) - if length_ratio < 0.3: - base_confidence -= 0.2 - - return min(1.0, max(0.0, base_confidence)) - - def _looks_like_json(self, text: str) -> bool: - """Check if text looks like JSON. - - Args: - text: Text to check - - Returns: - bool: True if text appears to be JSON - """ - text = text.strip() - return ( - (text.startswith('{') and text.endswith('}')) or - (text.startswith('[') and text.endswith(']')) - ) - - def get_model_prompt_adjustments(self, context: RequestContext) -> Dict[str, str]: - """Get prompt adjustments for specific model type. - - Args: - context: Request context - - Returns: - Dict[str, str]: Prompt adjustments by category - """ - model_type = self.detect_model_type(context) - - if model_type in self.model_patterns: - return self.model_patterns[model_type].prompt_adjustments.copy() - - # Generic adjustments - return { - "json_emphasis": "Please respond with valid JSON only.", - "format_instruction": "Ensure proper JSON formatting with double quotes." - } - - def enhance_prompt_for_model(self, prompt: str, context: RequestContext) -> str: - """Enhance prompt with model-specific adjustments. - - Args: - prompt: Original prompt - context: Request context - - Returns: - str: Enhanced prompt with model-specific adjustments - """ - adjustments = self.get_model_prompt_adjustments(context) - - if not adjustments: - return prompt - - # Add model-specific instructions - enhanced_prompt = prompt - - # Add JSON emphasis if available - if "json_emphasis" in adjustments: - enhanced_prompt += f"\n\nIMPORTANT: {adjustments['json_emphasis']}" - - # Add format instructions if available - if "format_instruction" in adjustments: - enhanced_prompt += f"\n{adjustments['format_instruction']}" - - self.logger.debug( - f"Enhanced prompt for model type {self.detect_model_type(context).value}", - extra={ - 'model_type': self.detect_model_type(context).value, - 'adjustments_applied': list(adjustments.keys()) - } - ) - - return enhanced_prompt - - def get_model_schema_preferences(self, context: RequestContext) -> Dict[str, Any]: - """Get schema preferences for specific model type. - - Args: - context: Request context - - Returns: - Dict[str, Any]: Schema preferences - """ - model_type = self.detect_model_type(context) - - if model_type in self.model_patterns: - return self.model_patterns[model_type].schema_preferences.copy() - - # Generic preferences - return { - "strict_types": False, - "flexible_format": True, - "simple_structure": True - } - - def get_adaptation_statistics(self) -> Dict[str, Any]: - """Get statistics about model adaptations. - - Returns: - Dict[str, Any]: Adaptation statistics - """ - model_counts = {} - for model_name, model_type in self.adaptation_cache.items(): - model_counts[model_type.value] = model_counts.get(model_type.value, 0) + 1 - - return { - "total_models_seen": len(self.adaptation_cache), - "model_type_distribution": model_counts, - "supported_model_types": [mt.value for mt in self.model_patterns.keys()], - "adaptation_strategies": [strategy.value for strategy in AdaptationStrategy] - } - - -# Global adaptation engine instance -_adaptation_engine: Optional[ModelAdaptationEngine] = None - - -def get_adaptation_engine() -> ModelAdaptationEngine: - """Get the global model adaptation engine. - - Returns: - ModelAdaptationEngine: Global adaptation engine instance - """ - global _adaptation_engine - if _adaptation_engine is None: - _adaptation_engine = ModelAdaptationEngine() - return _adaptation_engine - - -def preprocess_model_response(raw_response: str, context: RequestContext) -> AdaptationResult: - """Preprocess model response with adaptation. - - Args: - raw_response: Raw model response - context: Request context - - Returns: - AdaptationResult: Preprocessed response - """ - engine = get_adaptation_engine() - return engine.preprocess_response(raw_response, context) - - -def enhance_prompt_for_model(prompt: str, context: RequestContext) -> str: - """Enhance prompt for specific model. - - Args: - prompt: Original prompt - context: Request context - - Returns: - str: Enhanced prompt - """ - engine = get_adaptation_engine() - return engine.enhance_prompt_for_model(prompt, context) \ No newline at end of file diff --git a/src/utils/quality_monitor.py b/src/utils/quality_monitor.py deleted file mode 100644 index 65288fc..0000000 --- a/src/utils/quality_monitor.py +++ /dev/null @@ -1,394 +0,0 @@ -# -*- coding: utf-8 -*- -"""Quality monitoring for response processing. - -This module provides quality monitoring capabilities that track response -processing patterns, success rates, and failure modes. It focuses purely -on metrics collection and analysis, with optional integration with external -logging systems. -""" - -import json -from datetime import datetime, timedelta -from pathlib import Path -from typing import Dict, List, Optional, Callable, Any - -from .response_models import ( - QualityConfig, RequestContext, ResponseMetrics, QualityReport, - ContentIssue, EnhancementResult, ProcessingPath -) - - -class QualityMonitor: - """Monitor response quality patterns and collect metrics.""" - - def __init__(self, config: Optional[QualityConfig] = None): - """Initialize quality monitor with configuration. - - Args: - config: Quality monitoring configuration - """ - self.config = config or QualityConfig() - self.metrics = ResponseMetrics() - - # Optional external logger integration - self._external_logger: Optional[Callable[[str, Dict[str, Any]], None]] = None - - # Event listeners for external integration - self._event_listeners: List[Callable[[str, Dict[str, Any]], None]] = [] - - def set_external_logger(self, logger_func: Callable[[str, Dict[str, Any]], None]) -> None: - """Set external logger function for integration. - - Args: - logger_func: Function that takes (message, extra_data) and logs it - """ - self._external_logger = logger_func - - def add_event_listener(self, listener: Callable[[str, Dict[str, Any]], None]) -> None: - """Add event listener for quality events. - - Args: - listener: Function that receives quality events - """ - self._event_listeners.append(listener) - - def _emit_event(self, event_type: str, data: Dict[str, Any]) -> None: - """Emit quality event to listeners and logger. - - Args: - event_type: Type of quality event - data: Event data - """ - event_data = { - 'event_type': event_type, - 'timestamp': datetime.now().isoformat(), - **data - } - - # Send to external logger if configured - if self._external_logger: - message = f"Quality event: {event_type}" - self._external_logger(message, event_data) - - # Send to event listeners - for listener in self._event_listeners: - try: - listener(event_type, event_data) - except Exception: - # Don't let listener failures affect monitoring - pass - - def record_success(self, context: RequestContext, processing_path: ProcessingPath = ProcessingPath.DIRECT_SUCCESS) -> None: - """Record successful response processing. - - Args: - context: Request context information - processing_path: How the success was achieved - """ - if not self.config.enable_monitoring: - return - - self.metrics.total_attempts += 1 - self.metrics.successful_attempts += 1 - self.metrics.success_by_model[context.model_name] += 1 - - # Emit success event - self._emit_event('success', { - 'model': context.model_name, - 'processing_path': processing_path.value, - 'attempt_count': context.attempt_count + 1, - 'query_length': len(context.user_query), - 'session_id': context.session_id - }) - - def record_failure( - self, - context: RequestContext, - error_type: str, - recovery_attempted: bool = False, - final_failure: bool = True - ) -> None: - """Record failed response processing. - - Args: - context: Request context information - error_type: Type of error that occurred - recovery_attempted: Whether recovery was attempted - final_failure: Whether this is the final failure (after all retries) - """ - if not self.config.enable_monitoring: - return - - if final_failure: - self.metrics.total_attempts += 1 - self.metrics.failed_attempts += 1 - self.metrics.failures_by_model[context.model_name] += 1 - - self.metrics.failures_by_type[error_type] += 1 - - if recovery_attempted: - self.metrics.recovery_attempts += 1 - - # Emit failure event - self._emit_event('failure', { - 'model': context.model_name, - 'error_type': error_type, - 'recovery_attempted': recovery_attempted, - 'final_failure': final_failure, - 'attempt_count': context.attempt_count + 1, - 'session_id': context.session_id - }) - - def record_retry(self, context: RequestContext, error_type: str) -> None: - """Record retry attempt. - - Args: - context: Request context information - error_type: Type of error that triggered retry - """ - if not self.config.enable_monitoring: - return - - self.metrics.retry_attempts += 1 - self.metrics.retries_by_error_type[error_type] += 1 - - # Emit retry event - self._emit_event('retry', { - 'model': context.model_name, - 'error_type': error_type, - 'attempt_count': context.attempt_count + 1, - 'session_id': context.session_id - }) - - def record_content_quality_issue( - self, - context: RequestContext, - issues: List[ContentIssue] - ) -> None: - """Record content quality issues. - - Args: - context: Request context information - issues: List of content quality issues detected - """ - if not self.config.enable_monitoring or not self.config.track_content_quality: - return - - self.metrics.content_quality_issues += len(issues) - - for issue in issues: - self.metrics.content_issues_by_type[issue.type.value] += 1 - - # Emit content quality event - issue_types = [issue.type.value for issue in issues] - self._emit_event('content_quality_issues', { - 'model': context.model_name, - 'issue_types': issue_types, - 'issue_count': len(issues), - 'session_id': context.session_id - }) - - def record_enhancement_attempt( - self, - context: RequestContext, - enhancement_result: EnhancementResult - ) -> None: - """Record result enhancement attempt. - - Args: - context: Request context information - enhancement_result: Result of enhancement attempt - """ - if not self.config.enable_monitoring: - return - - self.metrics.enhancement_attempts += 1 - - for attempt in enhancement_result.attempts_made: - strategy = attempt.strategy.value - if enhancement_result.success: - self.metrics.enhancement_success_by_strategy[strategy] += 1 - - # Emit enhancement event - strategies = [a.strategy.value for a in enhancement_result.attempts_made] - self._emit_event('enhancement_attempt', { - 'model': context.model_name, - 'success': enhancement_result.success, - 'strategies': strategies, - 'improvement_score': enhancement_result.improvement_score, - 'session_id': context.session_id - }) - - def get_quality_report(self) -> QualityReport: - """Generate quality report for monitoring. - - Returns: - QualityReport: Current quality metrics and analysis - """ - success_rate = ( - self.metrics.successful_attempts / max(self.metrics.total_attempts, 1) - ) - - content_quality_metrics = {} - if self.config.track_content_quality: - total_content_issues = sum(self.metrics.content_issues_by_type.values()) - content_quality_metrics = { - "total_content_issues": total_content_issues, - "content_issue_rate": total_content_issues / max(self.metrics.total_attempts, 1), - "issues_by_type": dict(self.metrics.content_issues_by_type) - } - - enhancement_metrics = { - "total_enhancement_attempts": self.metrics.enhancement_attempts, - "enhancement_rate": self.metrics.enhancement_attempts / max(self.metrics.total_attempts, 1), - "success_by_strategy": dict(self.metrics.enhancement_success_by_strategy) - } - - return QualityReport( - success_rate=success_rate, - total_attempts=self.metrics.total_attempts, - failure_breakdown=dict(self.metrics.failures_by_type), - model_performance=dict(self.metrics.success_by_model), - content_quality_metrics=content_quality_metrics, - enhancement_metrics=enhancement_metrics - ) - - def check_quality_degradation(self) -> bool: - """Check if quality has degraded below threshold. - - Returns: - bool: True if quality degradation detected - """ - if not self.config.alert_on_degradation: - return False - - report = self.get_quality_report() - - # Check overall success rate - if report.success_rate < (1.0 - self.config.failure_threshold): - self._emit_event('quality_degradation', { - 'success_rate': report.success_rate, - 'threshold': 1.0 - self.config.failure_threshold, - 'total_attempts': report.total_attempts - }) - return True - - return False - - def get_model_performance_summary(self) -> Dict[str, Dict[str, float]]: - """Get performance summary by model. - - Returns: - Dict: Performance metrics by model name - """ - summary = {} - - for model_name in self.metrics.success_by_model.keys(): - successes = self.metrics.success_by_model[model_name] - failures = self.metrics.failures_by_model.get(model_name, 0) - total = successes + failures - - if total > 0: - summary[model_name] = { - "success_rate": successes / total, - "total_attempts": total, - "successes": successes, - "failures": failures - } - - return summary - - def export_metrics(self, filepath: Optional[Path] = None) -> Path: - """Export metrics to JSON file. - - Args: - filepath: Optional path for export file - - Returns: - Path: Path to exported file - """ - if filepath is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filepath = Path(f"logs/quality_metrics_{timestamp}.json") - - # Ensure directory exists - filepath.parent.mkdir(parents=True, exist_ok=True) - - # Prepare export data - export_data = { - "timestamp": datetime.now().isoformat(), - "metrics": { - "total_attempts": self.metrics.total_attempts, - "successful_attempts": self.metrics.successful_attempts, - "failed_attempts": self.metrics.failed_attempts, - "retry_attempts": self.metrics.retry_attempts, - "recovery_attempts": self.metrics.recovery_attempts, - "content_quality_issues": self.metrics.content_quality_issues, - "enhancement_attempts": self.metrics.enhancement_attempts, - "failures_by_type": dict(self.metrics.failures_by_type), - "failures_by_model": dict(self.metrics.failures_by_model), - "success_by_model": dict(self.metrics.success_by_model), - "retries_by_error_type": dict(self.metrics.retries_by_error_type), - "content_issues_by_type": dict(self.metrics.content_issues_by_type), - "enhancement_success_by_strategy": dict(self.metrics.enhancement_success_by_strategy) - }, - "quality_report": self.get_quality_report().__dict__, - "model_performance": self.get_model_performance_summary() - } - - # Write to file - with open(filepath, 'w') as f: - json.dump(export_data, f, indent=2, default=str) - - # Emit export event - self._emit_event('metrics_exported', { - 'filepath': str(filepath), - 'total_attempts': self.metrics.total_attempts - }) - - return filepath - - def reset_metrics(self) -> None: - """Reset all metrics to initial state.""" - self.metrics = ResponseMetrics() - self._emit_event('metrics_reset', {}) - - def is_monitoring_enabled(self) -> bool: - """Check if monitoring is enabled. - - Returns: - bool: True if monitoring is enabled - """ - return self.config.enable_monitoring - - def get_current_metrics(self) -> ResponseMetrics: - """Get current metrics object. - - Returns: - ResponseMetrics: Current metrics - """ - return self.metrics - - def get_metrics_summary(self) -> Dict[str, Any]: - """Get a summary of current metrics. - - Returns: - Dict: Summary of key metrics - """ - total = max(self.metrics.total_attempts, 1) - - return { - 'total_attempts': self.metrics.total_attempts, - 'success_rate': self.metrics.successful_attempts / total, - 'failure_rate': self.metrics.failed_attempts / total, - 'retry_rate': self.metrics.retry_attempts / total, - 'recovery_rate': self.metrics.recovery_attempts / total, - 'content_issue_rate': self.metrics.content_quality_issues / total, - 'enhancement_rate': self.metrics.enhancement_attempts / total, - 'top_failure_types': dict(sorted( - self.metrics.failures_by_type.items(), - key=lambda x: x[1], - reverse=True - )[:5]), - 'model_count': len(self.metrics.success_by_model) - } \ No newline at end of file diff --git a/src/utils/response_models.py b/src/utils/response_models.py deleted file mode 100644 index cf91f2a..0000000 --- a/src/utils/response_models.py +++ /dev/null @@ -1,267 +0,0 @@ -# -*- coding: utf-8 -*- -"""Data models for graceful response handling. - -This module defines the core data structures used throughout the graceful -model response handling system, including configuration models, result types, -and error classifications. -""" - -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from typing import Dict, List, Optional, Set, Any -from collections import defaultdict - - -# Enums for error types and processing paths -class ErrorType(Enum): - """Types of errors that can occur during response processing.""" - SYNTAX_ERROR = "syntax_error" - SCHEMA_VALIDATION = "schema_validation" - MISSING_FIELDS = "missing_fields" - WRONG_TYPES = "wrong_types" - EXTRACTION_FAILURE = "extraction_failure" - NETWORK_ERROR = "network_error" - TIMEOUT_ERROR = "timeout_error" - UNKNOWN_ERROR = "unknown_error" - - -class ProcessingPath(Enum): - """Different paths through the response processing pipeline.""" - DIRECT_SUCCESS = "direct_success" - RECOVERY_SUCCESS = "recovery_success" - RETRY_SUCCESS = "retry_success" - FALLBACK_SUCCESS = "fallback_success" - CONTENT_ENHANCED_SUCCESS = "content_enhanced_success" - COMPLETE_FAILURE = "complete_failure" - - -class RetryStrategy(Enum): - """Retry strategies based on error classification.""" - SYNTAX_EMPHASIS = "syntax_emphasis" - SCHEMA_EXAMPLE = "schema_example" - TYPE_CLARIFICATION = "type_clarification" - FORMAT_INSTRUCTION = "format_instruction" - GENERIC_CLARIFICATION = "generic_clarification" - - -class RecoveryMethod(Enum): - """Methods used for response recovery.""" - SYNTAX_REPAIR = "syntax_repair" - PARTIAL_EXTRACTION = "partial_extraction" - PATTERN_MATCHING = "pattern_matching" - TYPE_COERCION = "type_coercion" - ENHANCED_PATTERNS = "enhanced_patterns" - FALLBACK_PROCESSING = "fallback_processing" - - -class ContentIssueType(Enum): - """Types of content quality issues.""" - EMPTY_RESULTS = "empty_results" - INSUFFICIENT_RESULTS = "insufficient_results" - INCOMPLETE_ANALYSIS = "incomplete_analysis" - LOW_RELEVANCE = "low_relevance" - CONVERSATIONAL_RESPONSE = "conversational_response" - - -class IssueSeverity(Enum): - """Severity levels for content issues.""" - LOW = "low" - MEDIUM = "medium" - HIGH = "high" - - -class EnhancementStrategy(Enum): - """Strategies for enhancing insufficient results.""" - BROADER_QUERY = "broader_query" - QUERY_EXPANSION = "query_expansion" - DETAILED_ANALYSIS = "detailed_analysis" - RELEVANCE_FOCUS = "relevance_focus" - - -# Configuration models -@dataclass -class RetryConfig: - """Configuration for retry behavior.""" - max_retries: int = 3 - base_delay: float = 1.0 - max_delay: float = 10.0 - timeout_per_attempt: float = 30.0 - enabled_for_errors: Set[str] = field(default_factory=lambda: { - "syntax", "schema", "field_type", "structure" - }) - - -@dataclass -class RecoveryConfig: - """Configuration for recovery strategies.""" - enable_syntax_repair: bool = True - enable_partial_extraction: bool = True - enable_type_coercion: bool = True - enable_pattern_matching: bool = True - enable_enhanced_extraction: bool = True - min_confidence_threshold: float = 0.6 - - -@dataclass -class ContentQualityConfig: - """Configuration for content quality validation.""" - min_expected_papers: int = 3 - min_relevance_threshold: float = 0.7 - enable_quality_validation: bool = True - enable_result_enhancement: bool = True - - -@dataclass -class EnhancementConfig: - """Configuration for result enhancement.""" - max_enhancement_attempts: int = 2 - enable_query_broadening: bool = True - enable_query_expansion: bool = True - enable_analysis_enhancement: bool = True - - -@dataclass -class QualityConfig: - """Configuration for quality monitoring.""" - enable_monitoring: bool = True - failure_threshold: float = 0.3 - alert_on_degradation: bool = True - track_model_patterns: bool = True - track_content_quality: bool = True - - -@dataclass -class ResponseHandlerConfig: - """Main configuration for response handler behavior.""" - retry_config: RetryConfig = field(default_factory=RetryConfig) - recovery_config: RecoveryConfig = field(default_factory=RecoveryConfig) - content_quality_config: ContentQualityConfig = field(default_factory=ContentQualityConfig) - enhancement_config: EnhancementConfig = field(default_factory=EnhancementConfig) - quality_config: QualityConfig = field(default_factory=QualityConfig) - debug_mode: bool = False - - -# Request context and result models -@dataclass -class RequestContext: - """Context information for request processing.""" - user_query: str - model_name: str - attempt_count: int = 0 - previous_errors: List[str] = field(default_factory=list) - timestamp: datetime = field(default_factory=datetime.now) - session_id: str = "" - - -@dataclass -class ValidationResult: - """Extended validation result with retry classification.""" - is_valid: bool - error_message: str = "" - error_type: str = "" - show_raw_response: bool = True - retry_strategy: Optional[RetryStrategy] = None - - -@dataclass -class RecoveryResult: - """Result of recovery attempt.""" - success: bool - recovered_data: Optional[Dict[str, Any]] = None - recovery_method: Optional[RecoveryMethod] = None - confidence_score: float = 0.0 - warnings: List[str] = field(default_factory=list) - error_message: str = "" - - -@dataclass -class ProcessingResult: - """Result of complete response processing.""" - success: bool - data: Optional[Any] = None - processing_path: ProcessingPath = ProcessingPath.COMPLETE_FAILURE - quality_score: float = 0.0 - warnings: List[str] = field(default_factory=list) - debug_info: Optional[Dict[str, Any]] = None - - -# Content quality models -@dataclass -class ContentIssue: - """Represents a content quality issue.""" - type: ContentIssueType - severity: IssueSeverity - message: str - suggestions: List[str] = field(default_factory=list) - affected_papers: List[Any] = field(default_factory=list) - - -@dataclass -class ContentQualityResult: - """Result of content quality validation.""" - has_issues: bool - issues: List[ContentIssue] = field(default_factory=list) - overall_quality_score: float = 1.0 - - -@dataclass -class EnhancementAttempt: - """Represents an enhancement attempt.""" - strategy: EnhancementStrategy - enhanced_prompt: str - reason: str - - -@dataclass -class EnhancementResult: - """Result of enhancement attempts.""" - success: bool - enhanced_report: Optional[Any] = None - attempts_made: List[EnhancementAttempt] = field(default_factory=list) - improvement_score: float = 0.0 - - -# Quality monitoring models -@dataclass -class ResponseMetrics: - """Metrics for response processing quality.""" - total_attempts: int = 0 - successful_attempts: int = 0 - failed_attempts: int = 0 - retry_attempts: int = 0 - recovery_attempts: int = 0 - content_quality_issues: int = 0 - enhancement_attempts: int = 0 - failures_by_type: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - failures_by_model: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - success_by_model: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - retries_by_error_type: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - content_issues_by_type: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - enhancement_success_by_strategy: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - - -@dataclass -class QualityReport: - """Quality report for monitoring.""" - success_rate: float - total_attempts: int - failure_breakdown: Dict[str, int] - model_performance: Dict[str, int] - content_quality_metrics: Dict[str, int] = field(default_factory=dict) - enhancement_metrics: Dict[str, int] = field(default_factory=dict) - timestamp: datetime = field(default_factory=datetime.now) - - -# Debug information model -@dataclass -class DebugInfo: - """Debug information for troubleshooting.""" - original_response: str - cleaned_response: str - validation_details: Dict[str, Any] - recovery_attempts: List[Dict[str, Any]] - retry_history: List[Dict[str, Any]] - content_quality_analysis: Dict[str, Any] - enhancement_history: List[Dict[str, Any]] - processing_time_ms: float \ No newline at end of file diff --git a/src/utils/retry_manager.py b/src/utils/retry_manager.py deleted file mode 100644 index b820e24..0000000 --- a/src/utils/retry_manager.py +++ /dev/null @@ -1,235 +0,0 @@ -# -*- coding: utf-8 -*- -"""Retry management for graceful response handling. - -This module implements intelligent retry strategies that enhance prompts based -on specific error types detected during response processing. -""" - -import asyncio -import time -from typing import Optional - -from .response_models import ( - RetryConfig, RetryStrategy, RequestContext, ValidationResult -) - - -class RetryManager: - """Progressive retry mechanism that enhances prompts based on error analysis.""" - - def __init__(self, config: RetryConfig): - """Initialize retry manager with configuration. - - Args: - config: Retry configuration settings - """ - self.config = config - - def should_retry(self, error_type: str, attempt_count: int) -> bool: - """Determine if retry is appropriate for the given error type and attempt count. - - Args: - error_type: The type of error that occurred - attempt_count: Current attempt number (0-based) - - Returns: - bool: True if retry should be attempted - """ - if attempt_count >= self.config.max_retries: - return False - - return error_type in self.config.enabled_for_errors - - def enhance_prompt( - self, - original_prompt: str, - error_type: str, - previous_response: str = "" - ) -> str: - """Enhance prompt based on specific error type from existing validation. - - Args: - original_prompt: The original prompt that failed - error_type: Type of error detected (from existing validation) - previous_response: The previous response that failed (for context) - - Returns: - str: Enhanced prompt with specific instructions - """ - if error_type == "syntax": - return self._add_syntax_emphasis(original_prompt) - elif error_type == "schema": - return self._add_schema_example(original_prompt) - elif error_type == "field_type": - return self._add_type_clarification(original_prompt) - elif error_type == "structure": - return self._add_format_instruction(original_prompt) - else: - return self._add_generic_clarification(original_prompt) - - def enhance_prompt_for_syntax_error(self, original_prompt: str, error_details: str) -> str: - """Enhance prompt specifically for JSON syntax errors. - - Args: - original_prompt: The original prompt - error_details: Details about the syntax error - - Returns: - str: Enhanced prompt with syntax emphasis - """ - syntax_instruction = ( - "\n\nCRITICAL: The previous response had JSON syntax errors. " - "You MUST respond with ONLY valid JSON. " - "Common issues to avoid:\n" - "- Missing commas between fields\n" - "- Missing quotes around strings\n" - "- Unescaped quotes within strings\n" - "- Missing closing brackets or braces\n" - "- No text before or after the JSON object\n" - f"Previous error: {error_details[:100]}..." - ) - return original_prompt + syntax_instruction - - def enhance_prompt_generic(self, original_prompt: str, error_details: str) -> str: - """Generic prompt enhancement for unknown errors. - - Args: - original_prompt: The original prompt - error_details: Details about the error - - Returns: - str: Enhanced prompt with generic clarification - """ - generic_instruction = ( - "\n\nIMPORTANT: The previous response could not be processed correctly. " - "Please ensure your response is valid JSON with the exact structure requested. " - "Do not include any conversational text or explanations outside the JSON." - ) - return original_prompt + generic_instruction - - def _add_syntax_emphasis(self, prompt: str) -> str: - """Add JSON syntax emphasis to prompt.""" - syntax_instruction = ( - "\n\nIMPORTANT: Respond with ONLY valid JSON. " - "Ensure proper commas, brackets, and quotes. " - "Do not include any text before or after the JSON object. " - "Example format: {\"query\": \"...\", \"papers\": [...]}" - ) - return prompt + syntax_instruction - - def _add_schema_example(self, prompt: str) -> str: - """Add schema example to prompt.""" - schema_example = ( - '\n\nRequired JSON format:\n' - '{\n' - ' "query": "user query string",\n' - ' "papers": [\n' - ' {\n' - ' "title": "paper title",\n' - ' "year": 2023,\n' - ' "venue": "venue name",\n' - ' "url": "paper url",\n' - ' "doi": "paper doi",\n' - ' "key_points": ["point 1", "point 2"],\n' - ' "why_relevant": ["reason 1", "reason 2"]\n' - ' }\n' - ' ]\n' - '}\n' - 'You MUST include both "query" and "papers" fields.' - ) - return prompt + schema_example - - def _add_type_clarification(self, prompt: str) -> str: - """Add type clarification to prompt.""" - type_instruction = ( - "\n\nField type requirements:\n" - "- query: must be a string\n" - "- papers: must be an array/list\n" - "- title: must be a string\n" - "- year: must be a number (integer) or null\n" - "- venue, url, doi: must be strings or null\n" - "- key_points, why_relevant: must be arrays of strings\n" - "Ensure all fields have the correct data types." - ) - return prompt + type_instruction - - def _add_format_instruction(self, prompt: str) -> str: - """Add format instruction to prompt.""" - format_instruction = ( - "\n\nFormat requirements:\n" - "- Response must be a JSON object (starts with { and ends with })\n" - "- Must contain exactly two top-level fields: 'query' and 'papers'\n" - "- No additional fields at the top level\n" - "- No conversational text outside the JSON\n" - "- No code block markers (```json or ```)" - ) - return prompt + format_instruction - - def _add_generic_clarification(self, prompt: str) -> str: - """Add generic clarification to prompt.""" - generic_instruction = ( - "\n\nPlease ensure your response:\n" - "- Is valid JSON format\n" - "- Contains the required 'query' and 'papers' fields\n" - "- Has no text outside the JSON object\n" - "- Uses proper JSON syntax (commas, quotes, brackets)" - ) - return prompt + generic_instruction - - async def get_retry_delay(self, attempt_count: int) -> float: - """Calculate exponential backoff delay for retry attempts. - - Args: - attempt_count: Current attempt number (0-based) - - Returns: - float: Delay in seconds - """ - if attempt_count == 0: - return 0.0 - - # Exponential backoff: base_delay * 2^(attempt_count - 1) - delay = self.config.base_delay * (2 ** (attempt_count - 1)) - return min(delay, self.config.max_delay) - - def classify_error_for_retry(self, validation_result: ValidationResult) -> RetryStrategy: - """Classify validation errors to determine optimal retry strategy. - - Extends existing validation with retry-specific classification. - - Args: - validation_result: Result from existing validation - - Returns: - RetryStrategy: Recommended retry strategy - """ - if validation_result.error_type == "syntax": - return RetryStrategy.SYNTAX_EMPHASIS - elif validation_result.error_type == "schema": - return RetryStrategy.SCHEMA_EXAMPLE - elif validation_result.error_type == "field_type": - return RetryStrategy.TYPE_CLARIFICATION - elif validation_result.error_type == "structure": - return RetryStrategy.FORMAT_INSTRUCTION - else: - return RetryStrategy.GENERIC_CLARIFICATION - - def get_retry_context(self, context: RequestContext, error_type: str) -> RequestContext: - """Create updated context for retry attempt. - - Args: - context: Original request context - error_type: Type of error that triggered retry - - Returns: - RequestContext: Updated context for retry - """ - new_context = RequestContext( - user_query=context.user_query, - model_name=context.model_name, - attempt_count=context.attempt_count + 1, - previous_errors=context.previous_errors + [error_type], - timestamp=context.timestamp, - session_id=context.session_id - ) - return new_context \ No newline at end of file diff --git a/tests/test_graceful_response_handling.py b/tests/test_graceful_response_handling.py deleted file mode 100644 index 3670022..0000000 --- a/tests/test_graceful_response_handling.py +++ /dev/null @@ -1,674 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests for graceful response handling implementation. - -This module contains basic tests to verify the graceful response handling -components work correctly and integrate properly with the existing system. -""" - -import pytest -from unittest.mock import Mock, AsyncMock -from dataclasses import dataclass -from typing import List, Optional - -from src.utils.response_models import ( - ResponseHandlerConfig, RequestContext, ProcessingResult, ProcessingPath, - ValidationResult, RetryStrategy, ContentIssue, ContentIssueType, IssueSeverity -) -from src.utils.retry_manager import RetryManager -from src.utils.quality_monitor import QualityMonitor -from src.utils.enhanced_validation import enhance_validation_result -from src.utils.content_quality import ContentQualityValidator, QueryAnalyzer, RelevanceScorer - - -# Mock classes for testing -@dataclass(frozen=True) # Make it hashable -class MockPaperAnalysis: - """Mock paper analysis for testing.""" - title: str - year: Optional[int] = None - venue: Optional[str] = None - url: Optional[str] = None - doi: Optional[str] = None - key_points: tuple = () # Use tuple instead of list for hashability - why_relevant: tuple = () # Use tuple instead of list for hashability - - def __post_init__(self): - # Convert lists to tuples if needed - if isinstance(self.key_points, list): - object.__setattr__(self, 'key_points', tuple(self.key_points)) - if isinstance(self.why_relevant, list): - object.__setattr__(self, 'why_relevant', tuple(self.why_relevant)) - - -@dataclass -class MockResearchReport: - """Mock research report for testing.""" - query: str - papers: List[MockPaperAnalysis] - - -class TestRetryManager: - """Test retry manager functionality.""" - - def test_retry_manager_initialization(self): - """Test retry manager initializes correctly.""" - from src.utils.response_models import RetryConfig - - config = RetryConfig(max_retries=2, base_delay=0.5) - retry_manager = RetryManager(config) - - assert retry_manager.config.max_retries == 2 - assert retry_manager.config.base_delay == 0.5 - - def test_should_retry_logic(self): - """Test retry decision logic.""" - from src.utils.response_models import RetryConfig - - config = RetryConfig(max_retries=3, enabled_for_errors={"syntax", "schema"}) - retry_manager = RetryManager(config) - - # Should retry for enabled error types within limit - assert retry_manager.should_retry("syntax", 0) == True - assert retry_manager.should_retry("syntax", 2) == True - assert retry_manager.should_retry("schema", 1) == True - - # Should not retry beyond limit - assert retry_manager.should_retry("syntax", 3) == False - assert retry_manager.should_retry("syntax", 5) == False - - # Should not retry for disabled error types - assert retry_manager.should_retry("unknown", 0) == False - - def test_prompt_enhancement(self): - """Test prompt enhancement for different error types.""" - from src.utils.response_models import RetryConfig - - retry_manager = RetryManager(RetryConfig()) - original_prompt = "Analyze these papers" - - # Test syntax enhancement - enhanced = retry_manager.enhance_prompt(original_prompt, "syntax") - assert "valid JSON" in enhanced - assert original_prompt in enhanced - - # Test schema enhancement - enhanced = retry_manager.enhance_prompt(original_prompt, "schema") - assert "Required JSON format" in enhanced - assert original_prompt in enhanced - - # Test generic enhancement - enhanced = retry_manager.enhance_prompt(original_prompt, "unknown") - assert "ensure your response" in enhanced - assert original_prompt in enhanced - - -class TestQualityMonitor: - """Test quality monitoring functionality.""" - - def test_quality_monitor_initialization(self): - """Test quality monitor initializes correctly.""" - from src.utils.response_models import QualityConfig - - config = QualityConfig(enable_monitoring=True) - monitor = QualityMonitor(config) - - assert monitor.config.enable_monitoring == True - assert monitor.metrics.total_attempts == 0 - - def test_success_recording(self): - """Test recording successful operations.""" - from src.utils.response_models import QualityConfig - - monitor = QualityMonitor(QualityConfig(enable_monitoring=True)) - context = RequestContext( - user_query="test query", - model_name="test_model", - attempt_count=0 - ) - - monitor.record_success(context) - - assert monitor.metrics.total_attempts == 1 - assert monitor.metrics.successful_attempts == 1 - assert monitor.metrics.success_by_model["test_model"] == 1 - - def test_failure_recording(self): - """Test recording failed operations.""" - from src.utils.response_models import QualityConfig - - monitor = QualityMonitor(QualityConfig(enable_monitoring=True)) - context = RequestContext( - user_query="test query", - model_name="test_model", - attempt_count=0 - ) - - monitor.record_failure(context, "syntax_error", recovery_attempted=True, final_failure=True) - - assert monitor.metrics.total_attempts == 1 - assert monitor.metrics.failed_attempts == 1 - assert monitor.metrics.failures_by_type["syntax_error"] == 1 - assert monitor.metrics.recovery_attempts == 1 - - def test_quality_report_generation(self): - """Test quality report generation.""" - from src.utils.response_models import QualityConfig - - monitor = QualityMonitor(QualityConfig(enable_monitoring=True)) - context = RequestContext( - user_query="test query", - model_name="test_model", - attempt_count=0 - ) - - # Record some operations - monitor.record_success(context) - monitor.record_success(context) - monitor.record_failure(context, "syntax_error", final_failure=True) - - report = monitor.get_quality_report() - - assert report.total_attempts == 3 - assert report.success_rate == 2/3 # 2 successes out of 3 attempts - assert "syntax_error" in report.failure_breakdown - - -class TestEnhancedValidation: - """Test enhanced validation functionality.""" - - def test_validation_enhancement(self): - """Test validation result enhancement with retry strategy.""" - # Create a validation result - validation_result = ValidationResult( - is_valid=False, - error_message="Test error", - error_type="syntax" - ) - - # Enhance it - enhanced = enhance_validation_result(validation_result) - - assert enhanced.is_valid == False - assert enhanced.error_type == "syntax" - assert enhanced.retry_strategy == RetryStrategy.SYNTAX_EMPHASIS - - def test_valid_response_enhancement(self): - """Test enhancement of valid responses.""" - validation_result = ValidationResult(is_valid=True) - enhanced = enhance_validation_result(validation_result) - - assert enhanced.is_valid == True - assert enhanced.retry_strategy is None - - -class TestContentQuality: - """Test content quality validation.""" - - def test_content_quality_validator_initialization(self): - """Test content quality validator initializes correctly.""" - from src.utils.response_models import ContentQualityConfig - - config = ContentQualityConfig(min_expected_papers=3) - validator = ContentQualityValidator(config) - - assert validator.config.min_expected_papers == 3 - assert validator.query_analyzer is not None - assert validator.relevance_scorer is not None - - def test_empty_results_detection(self): - """Test detection of empty results.""" - from src.utils.response_models import ContentQualityConfig - - validator = ContentQualityValidator(ContentQualityConfig()) - report = MockResearchReport(query="test query", papers=[]) - - result = validator.validate_content_quality(report, "test query") - - assert result.has_issues == True - assert len(result.issues) == 1 - assert result.issues[0].type == ContentIssueType.EMPTY_RESULTS - assert result.issues[0].severity == IssueSeverity.HIGH - - def test_insufficient_results_detection(self): - """Test detection of insufficient results.""" - from src.utils.response_models import ContentQualityConfig - - config = ContentQualityConfig(min_expected_papers=3) - validator = ContentQualityValidator(config) - - # Create report with only 1 paper (less than minimum of 3) - papers = [MockPaperAnalysis(title="Test Paper", key_points=(), why_relevant=())] - report = MockResearchReport(query="test query", papers=papers) - - result = validator.validate_content_quality(report, "test query") - - assert result.has_issues == True - insufficient_issues = [issue for issue in result.issues if issue.type == ContentIssueType.INSUFFICIENT_RESULTS] - assert len(insufficient_issues) == 1 - assert insufficient_issues[0].severity == IssueSeverity.MEDIUM - - def test_incomplete_analysis_detection(self): - """Test detection of incomplete paper analysis.""" - from src.utils.response_models import ContentQualityConfig - - validator = ContentQualityValidator(ContentQualityConfig()) - - # Create paper with missing key_points and why_relevant - papers = [MockPaperAnalysis(title="Test Paper", key_points=(), why_relevant=())] - report = MockResearchReport(query="test query", papers=papers) - - result = validator.validate_content_quality(report, "test query") - - assert result.has_issues == True - incomplete_issues = [issue for issue in result.issues if issue.type == ContentIssueType.INCOMPLETE_ANALYSIS] - assert len(incomplete_issues) == 1 - assert incomplete_issues[0].severity == IssueSeverity.LOW - - def test_quality_with_complete_papers(self): - """Test quality validation with complete, relevant papers.""" - from src.utils.response_models import ContentQualityConfig - - config = ContentQualityConfig(min_expected_papers=2) - validator = ContentQualityValidator(config) - - # Create complete papers - papers = [ - MockPaperAnalysis( - title="Machine Learning Paper", - key_points=("Uses neural networks", "Achieves high accuracy", "Novel architecture"), - why_relevant=("Addresses the research question", "Uses similar methodology") - ), - MockPaperAnalysis( - title="Deep Learning Study", - key_points=("Compares different models", "Extensive evaluation", "Open source code"), - why_relevant=("Relevant to the domain", "Provides baseline comparisons") - ) - ] - report = MockResearchReport(query="machine learning", papers=papers) - - result = validator.validate_content_quality(report, "machine learning") - - # Should have no major issues (might have low relevance but that's expected) - high_severity_issues = [issue for issue in result.issues if issue.severity == IssueSeverity.HIGH] - assert len(high_severity_issues) == 0 - - -class TestQueryAnalyzer: - """Test query analysis functionality.""" - - def test_broader_terms_suggestion(self): - """Test suggestion of broader terms.""" - analyzer = QueryAnalyzer() - - # Test with specific query - suggestions = analyzer.suggest_broader_terms("advanced neural networks 2023") - - assert len(suggestions) > 0 - # Should remove year and specific modifiers in at least some suggestions - has_year_removed = any("2023" not in suggestion for suggestion in suggestions) - has_modifier_removed = any("advanced" not in suggestion.lower() for suggestion in suggestions) - assert has_year_removed or has_modifier_removed - - def test_query_expansion_suggestion(self): - """Test query expansion suggestions.""" - analyzer = QueryAnalyzer() - - suggestions = analyzer.suggest_query_expansion("machine learning") - - assert len(suggestions) > 0 - # Should include related terms - assert any("artificial intelligence" in suggestion.lower() for suggestion in suggestions) - - def test_specific_terms_suggestion(self): - """Test more specific terms suggestion.""" - analyzer = QueryAnalyzer() - - suggestions = analyzer.suggest_more_specific_terms("learning") - - assert len(suggestions) > 0 - # Should add modifiers or contexts - assert any(len(suggestion.split()) > 1 for suggestion in suggestions) - - -class TestRelevanceScorer: - """Test relevance scoring functionality.""" - - def test_relevance_scoring(self): - """Test paper relevance scoring.""" - scorer = RelevanceScorer() - - # Create papers with different relevance levels - papers = [ - MockPaperAnalysis( - title="Machine Learning Classification", - key_points=("machine learning", "classification algorithms"), - why_relevant=("directly addresses machine learning",) - ), - MockPaperAnalysis( - title="Database Systems", - key_points=("database design", "query optimization"), - why_relevant=("not directly related",) - ) - ] - - scores = scorer.score_papers(papers, "machine learning classification") - - assert len(scores) == 2 - # First paper should have higher relevance score - paper_scores = list(scores.values()) - assert paper_scores[0] > paper_scores[1] - assert all(0.0 <= score <= 1.0 for score in paper_scores) - - -# Integration test -class TestBasicIntegration: - """Test basic integration between components.""" - - def test_components_work_together(self): - """Test that components can work together without errors.""" - from src.utils.response_models import ResponseHandlerConfig - - # Create configuration - config = ResponseHandlerConfig() - - # Initialize components - retry_manager = RetryManager(config.retry_config) - quality_monitor = QualityMonitor(config.quality_config) - content_validator = ContentQualityValidator(config.content_quality_config) - - # Test basic operations - context = RequestContext( - user_query="test query", - model_name="test_model", - attempt_count=0 - ) - - # Test quality monitoring - quality_monitor.record_success(context) - report = quality_monitor.get_quality_report() - assert report.success_rate == 1.0 - - # Test content validation - papers = [MockPaperAnalysis(title="Test Paper", key_points=(), why_relevant=())] - mock_report = MockResearchReport(query="test", papers=papers) - content_result = content_validator.validate_content_quality(mock_report, "test") - assert isinstance(content_result.has_issues, bool) - - # Test retry logic - assert retry_manager.should_retry("syntax", 0) == True - enhanced_prompt = retry_manager.enhance_prompt("test prompt", "syntax") - assert "test prompt" in enhanced_prompt - - -class TestModelAdaptation: - """Test model adaptation functionality.""" - - def test_model_adaptation_engine_initialization(self): - """Test model adaptation engine initializes correctly.""" - from src.utils.model_adaptation import ModelAdaptationEngine, ModelType - - engine = ModelAdaptationEngine() - - assert engine is not None - assert len(engine.model_patterns) > 0 - assert ModelType.OPENAI_GPT in engine.model_patterns - assert ModelType.ANTHROPIC_CLAUDE in engine.model_patterns - - def test_model_type_detection(self): - """Test detection of model types.""" - from src.utils.model_adaptation import ModelAdaptationEngine, ModelType - - engine = ModelAdaptationEngine() - - # Test OpenAI detection - context_gpt = RequestContext( - user_query="test", - model_name="gpt-4-turbo", - attempt_count=0 - ) - assert engine.detect_model_type(context_gpt) == ModelType.OPENAI_GPT - - # Test Claude detection - context_claude = RequestContext( - user_query="test", - model_name="claude-3-opus", - attempt_count=0 - ) - assert engine.detect_model_type(context_claude) == ModelType.ANTHROPIC_CLAUDE - - # Test generic fallback - context_unknown = RequestContext( - user_query="test", - model_name="unknown-model-xyz", - attempt_count=0 - ) - assert engine.detect_model_type(context_unknown) == ModelType.GENERIC - - def test_response_preprocessing(self): - """Test response preprocessing for different models.""" - from src.utils.model_adaptation import ModelAdaptationEngine - - engine = ModelAdaptationEngine() - - # Test OpenAI response with markdown code blocks - gpt_response = '''```json -{ - "query": "test", - "papers": [] -} -```''' - - context_gpt = RequestContext( - user_query="test", - model_name="gpt-4", - attempt_count=0 - ) - - result = engine.preprocess_response(gpt_response, context_gpt) - - assert result.success == True - assert "```" not in result.adapted_response - assert "{" in result.adapted_response - - def test_prompt_enhancement_for_model(self): - """Test prompt enhancement for specific models.""" - from src.utils.model_adaptation import ModelAdaptationEngine - - engine = ModelAdaptationEngine() - - original_prompt = "Analyze these papers" - - context_gpt = RequestContext( - user_query="test", - model_name="gpt-4", - attempt_count=0 - ) - - enhanced = engine.enhance_prompt_for_model(original_prompt, context_gpt) - - assert enhanced != original_prompt - assert "JSON" in enhanced or "json" in enhanced - assert original_prompt in enhanced - - def test_schema_preferences_by_model(self): - """Test schema preferences for different models.""" - from src.utils.model_adaptation import ModelAdaptationEngine, ModelType - - engine = ModelAdaptationEngine() - - context_gpt = RequestContext( - user_query="test", - model_name="gpt-4", - attempt_count=0 - ) - - prefs = engine.get_model_schema_preferences(context_gpt) - - assert isinstance(prefs, dict) - assert len(prefs) > 0 - # GPT should prefer strict types - assert prefs.get("strict_types") == True - - def test_adaptation_statistics(self): - """Test adaptation statistics collection.""" - from src.utils.model_adaptation import ModelAdaptationEngine - - engine = ModelAdaptationEngine() - - # Detect a few models to populate cache - for model_name in ["gpt-4", "claude-3", "gemini-pro"]: - context = RequestContext( - user_query="test", - model_name=model_name, - attempt_count=0 - ) - engine.detect_model_type(context) - - stats = engine.get_adaptation_statistics() - - assert "total_models_seen" in stats - assert "model_type_distribution" in stats - assert "supported_model_types" in stats - assert stats["total_models_seen"] >= 3 - - -class TestFallbackProcessing: - """Test fallback processing functionality.""" - - def test_fallback_processor_initialization(self): - """Test fallback processor initializes correctly.""" - from src.utils.fallback_processing import TextFallbackProcessor - from src.utils.response_models import RecoveryConfig - - config = RecoveryConfig() - processor = TextFallbackProcessor(config) - - assert processor.config is not None - assert processor.logger is not None - - def test_paper_extraction_from_text(self): - """Test extraction of paper mentions from unstructured text.""" - from src.utils.fallback_processing import TextFallbackProcessor - - processor = TextFallbackProcessor() - - # Sample unstructured response with paper mentions - response = ''' - The research shows several important findings. The paper "Deep Learning for Natural Language Processing" - by Smith et al. demonstrates significant improvements in accuracy. Another study titled - "Machine Learning Applications in Healthcare" published in 2023 shows promising results. - The authors found that neural networks can achieve 95% accuracy on classification tasks. - ''' - - result = processor.process_unstructured_response(response, "machine learning") - - assert result.success == True - # The processor may choose different extraction methods based on confidence - # Check that some useful content was extracted - assert result.extracted_content is not None - assert len(result.extracted_content) > 0 - - # Test the specific paper extraction method directly - paper_result = processor._extract_paper_mentions(response, "machine learning") - assert "papers" in paper_result.extracted_content - papers = paper_result.extracted_content["papers"] - assert len(papers) > 0 - - # Check that at least one paper was extracted - paper_titles = [p["title"] for p in papers] - assert any("Deep Learning" in title for title in paper_titles) - - def test_key_findings_extraction(self): - """Test extraction of key findings from text.""" - from src.utils.fallback_processing import TextFallbackProcessor - - processor = TextFallbackProcessor() - - response = ''' - The study found that machine learning models perform better with larger datasets. - Research shows that deep learning demonstrates superior performance on image recognition tasks. - The analysis indicates that preprocessing improves model accuracy by 15%. - ''' - - result = processor._extract_key_findings(response, "machine learning performance") - - assert result.success == True - assert "key_findings" in result.extracted_content - findings = result.extracted_content["key_findings"] - assert len(findings) > 0 - assert any("found that" in finding for finding in findings) - - def test_research_themes_extraction(self): - """Test extraction of research themes.""" - from src.utils.fallback_processing import TextFallbackProcessor - - processor = TextFallbackProcessor() - - response = ''' - This paper focuses on machine learning algorithms for natural language processing. - The deep learning model uses neural networks to improve classification accuracy. - Data mining techniques are applied to extract patterns from big data. - The experiment evaluates performance on multiple datasets. - ''' - - result = processor._extract_research_themes(response, "AI research") - - assert result.success == True - assert "research_themes" in result.extracted_content - themes = result.extracted_content["research_themes"] - assert len(themes) > 0 - assert any("machine learning" in theme.lower() for theme in themes) - - def test_structured_summary_creation(self): - """Test creation of structured summaries.""" - from src.utils.fallback_processing import TextFallbackProcessor - - processor = TextFallbackProcessor() - - response = ''' - Introduction: This study examines machine learning applications. - - The main findings show that neural networks achieve high accuracy. - Deep learning models outperform traditional algorithms. - - In conclusion, machine learning shows great promise for future applications. - ''' - - result = processor._create_structured_summary(response, "machine learning study") - - assert result.success == True - assert "structured_summary" in result.extracted_content - summary = result.extracted_content["structured_summary"] - assert "query_context" in summary - assert summary["query_context"] == "machine learning study" - - def test_fallback_report_creation(self): - """Test creation of research report from fallback results.""" - from src.utils.fallback_processing import create_fallback_research_report, FallbackResult, FallbackMethod - - # Create mock fallback result - fallback_result = FallbackResult( - success=True, - method=FallbackMethod.PATTERN_MATCHING, - extracted_content={ - "papers": [ - { - "title": "Test Paper", - "authors": "Test Author", - "key_points": ["Key finding 1", "Key finding 2"], - "why_relevant": "Relevant to query" - } - ] - }, - confidence_score=0.6, - warnings=["Low confidence extraction"], - raw_sections=["test section"] - ) - - report = create_fallback_research_report(fallback_result, "test query") - - assert report["query"] == "test query" - assert len(report["papers"]) == 1 - assert report["papers"][0]["title"] == "Test Paper" - assert report["metadata"]["processing_method"] == "fallback" - assert report["metadata"]["confidence"] == 0.6 \ No newline at end of file diff --git a/tests/test_structured_outputs.py b/tests/test_structured_outputs.py index 1b046f5..da68edd 100644 --- a/tests/test_structured_outputs.py +++ b/tests/test_structured_outputs.py @@ -40,10 +40,6 @@ def _ranked(title: str, abstract: str = "Findings here.") -> RankedPaper: ) -def _structured_llm() -> LLMConfig: - return LLMConfig(structured_outputs=True) - - class TestRunStructured: async def test_returns_schema_valid_output_with_test_model(self) -> None: result = await run_structured( @@ -98,7 +94,7 @@ async def fake_structured(role, prompt, output_type, config=None, **kwargs): extractions = await extract_papers( ranked, "query", - llm_config=_structured_llm(), + llm_config=LLMConfig(), synthesis_config=AppSettings( synthesis={"llm_enabled": True, "max_llm_papers": 1} ).synthesis, @@ -121,7 +117,7 @@ async def failing_structured(*args, **kwargs): extractions = await extract_papers( ranked, "query", - llm_config=_structured_llm(), + llm_config=LLMConfig(), concurrency=1, synthesis_config=AppSettings( synthesis={ @@ -153,7 +149,7 @@ async def fake_structured(role, prompt, output_type, config=None, **kwargs): "query", [PaperExtraction(paper_id="a", title="A", findings=["F"])], [], - llm_config=_structured_llm(), + llm_config=LLMConfig(), synthesis_config=AppSettings( synthesis={"llm_enabled": True} ).synthesis, @@ -172,7 +168,7 @@ async def fake_structured(role, prompt, output_type, config=None, **kwargs): result = await analyze_gaps( "query", SynthesisResult(gaps=["G"]), - llm_config=_structured_llm(), + llm_config=LLMConfig(), ) assert result is expected @@ -185,7 +181,7 @@ async def failing(*args, **kwargs): result = await analyze_gaps( "query", SynthesisResult(gaps=["Known gap"]), - llm_config=_structured_llm(), + llm_config=LLMConfig(), ) assert "Known gap" in result.gaps @@ -197,7 +193,6 @@ async def test_expansion_uses_structured_output(self, monkeypatch) -> None: from src.config.settings import get_settings get_settings.cache_clear() - monkeypatch.setenv("RA_LLM__STRUCTURED_OUTPUTS", "true") async def fake_run(role, prompt, output_type, config=None, **kwargs): assert role is AgentRole.EXPANSION diff --git a/tests/test_synthesis.py b/tests/test_synthesis.py index 4ab2ebe..c160b40 100644 --- a/tests/test_synthesis.py +++ b/tests/test_synthesis.py @@ -39,8 +39,8 @@ RetrievedPaper, SynthesisResult, ) -from src.utils.enhanced_response_handler import EnhancedResponseHandler -from src.utils.response_models import ProcessingPath, ProcessingResult + +STRUCTURED_TARGET = "src.models.structured.try_run_structured" def _paper(title: str, **kwargs: object) -> RetrievedPaper: @@ -56,40 +56,22 @@ def _ranked(title: str, **kwargs: object) -> RankedPaper: ) -def _mock_handler_success(data: object) -> EnhancedResponseHandler: - handler = MagicMock(spec=EnhancedResponseHandler) - handler.process_structured_response = AsyncMock( - return_value=ProcessingResult( - success=True, - data=data, - processing_path=ProcessingPath.DIRECT_SUCCESS, - ) - ) - return handler +def _structured_mock(*results: object) -> AsyncMock: + """AsyncMock standing in for try_run_structured. - -def _mock_handler_failure() -> EnhancedResponseHandler: - handler = MagicMock(spec=EnhancedResponseHandler) - handler.process_structured_response = AsyncMock( - return_value=ProcessingResult( - success=False, - processing_path=ProcessingPath.COMPLETE_FAILURE, - ) - ) - return handler - - -def _legacy_llm_config() -> LLMConfig: - """LLM config exercising the legacy prose-JSON handler path.""" - return LLMConfig(structured_outputs=False) + With one result it is returned for every call; with several they are + consumed in call order. ``None`` triggers the caller's heuristic fallback, + exactly like a failed structured call. + """ + if len(results) == 1: + return AsyncMock(return_value=results[0]) + return AsyncMock(side_effect=list(results)) def _llm_synthesis_config(**overrides: object): config = { "llm_enabled": True, "max_llm_papers": 10, - "extraction_max_retries": 0, - "collective_max_retries": 0, "concurrency": 2, } config.update(overrides) @@ -288,41 +270,38 @@ def test_heuristic_gap_analysis_expands_synthesis(self) -> None: class TestSynthesisWorkflow: @pytest.mark.asyncio - async def test_extract_papers_uses_handler(self) -> None: + async def test_extract_papers_uses_structured_llm(self) -> None: ranked = [_ranked("Paper One", abstract="Findings here.")] expected = PaperExtraction( paper_id="Paper One", title="Paper One", findings=["Finding"], ) - handler = _mock_handler_success(expected) - - extractions = await extract_papers( - ranked, - "test query", - llm_config=_legacy_llm_config(), - handler=handler, - concurrency=1, - synthesis_config=_llm_synthesis_config(max_llm_papers=1), - ) + structured = _structured_mock(expected) + + with patch(STRUCTURED_TARGET, structured): + extractions = await extract_papers( + ranked, + "test query", + concurrency=1, + synthesis_config=_llm_synthesis_config(max_llm_papers=1), + ) assert len(extractions) == 1 assert extractions[0].title == "Paper One" - handler.process_structured_response.assert_awaited() + structured.assert_awaited() @pytest.mark.asyncio - async def test_extract_papers_falls_back_on_handler_failure(self) -> None: + async def test_extract_papers_falls_back_on_llm_failure(self) -> None: ranked = [_ranked("Paper Two", abstract="Some results.")] - handler = _mock_handler_failure() - - extractions = await extract_papers( - ranked, - "test query", - llm_config=_legacy_llm_config(), - handler=handler, - concurrency=1, - synthesis_config=_llm_synthesis_config(max_llm_papers=1), - ) + + with patch(STRUCTURED_TARGET, _structured_mock(None)): + extractions = await extract_papers( + ranked, + "test query", + concurrency=1, + synthesis_config=_llm_synthesis_config(max_llm_papers=1), + ) assert len(extractions) == 1 assert extractions[0].title == "Paper Two" @@ -336,16 +315,14 @@ async def test_synthesize_collective_validates_schema(self) -> None: agreements=["Shared transformer usage"], gaps=["Need more benchmarks"], ) - handler = _mock_handler_success(expected) - synthesis = await synthesize_collective( - "transformers", - extractions, - [], - llm_config=_legacy_llm_config(), - handler=handler, - synthesis_config=_llm_synthesis_config(), - ) + with patch(STRUCTURED_TARGET, _structured_mock(expected)): + synthesis = await synthesize_collective( + "transformers", + extractions, + [], + synthesis_config=_llm_synthesis_config(), + ) assert synthesis.agreements == ["Shared transformer usage"] assert synthesis.gaps == ["Need more benchmarks"] @@ -361,56 +338,43 @@ async def test_run_synthesis_two_pass_workflow(self) -> None: extraction_a = PaperExtraction(paper_id="Paper A", title="Paper A", findings=["A"]) extraction_b = PaperExtraction(paper_id="Paper B", title="Paper B", findings=["B"]) synthesis = SynthesisResult(agreements=["Both use deep learning"], gaps=["Scale"]) - - handler = MagicMock(spec=EnhancedResponseHandler) - handler.process_structured_response = AsyncMock( - side_effect=[ - ProcessingResult(success=True, data=extraction_a, processing_path=ProcessingPath.DIRECT_SUCCESS), - ProcessingResult(success=True, data=extraction_b, processing_path=ProcessingPath.DIRECT_SUCCESS), - ProcessingResult(success=True, data=synthesis, processing_path=ProcessingPath.DIRECT_SUCCESS), - ] - ) - - result, extractions, analyses = await run_synthesis( - "deep learning", - ranked, - clusters, - llm_config=_legacy_llm_config(), - handler=handler, - concurrency=2, - synthesis_config=_llm_synthesis_config(max_llm_papers=2), - ) + structured = _structured_mock(extraction_a, extraction_b, synthesis) + + with patch(STRUCTURED_TARGET, structured): + result, extractions, analyses = await run_synthesis( + "deep learning", + ranked, + clusters, + concurrency=2, + synthesis_config=_llm_synthesis_config(max_llm_papers=2), + ) assert len(extractions) == 2 assert result.agreements == ["Both use deep learning"] assert len(analyses) == 2 - assert handler.process_structured_response.await_count == 3 + assert structured.await_count == 3 class TestGapAnalysis: @pytest.mark.asyncio - async def test_analyze_gaps_uses_handler(self) -> None: + async def test_analyze_gaps_uses_structured_llm(self) -> None: synthesis = SynthesisResult(gaps=["Missing ablations"]) expected = GapAnalysisResult( gaps=["Missing ablations"], opportunities=["Run systematic ablations"], ) - handler = _mock_handler_success(expected) - result = await analyze_gaps( - "query", synthesis, llm_config=_legacy_llm_config(), handler=handler - ) + with patch(STRUCTURED_TARGET, _structured_mock(expected)): + result = await analyze_gaps("query", synthesis, llm_config=LLMConfig()) assert result.opportunities == ["Run systematic ablations"] @pytest.mark.asyncio async def test_analyze_gaps_heuristic_fallback(self) -> None: synthesis = SynthesisResult(gaps=["Understudied domain"]) - handler = _mock_handler_failure() - result = await analyze_gaps( - "query", synthesis, llm_config=_legacy_llm_config(), handler=handler - ) + with patch(STRUCTURED_TARGET, _structured_mock(None)): + result = await analyze_gaps("query", synthesis, llm_config=LLMConfig()) assert "Understudied domain" in result.gaps assert result.opportunities @@ -427,16 +391,9 @@ async def test_synthesis_stage_records_artifacts(self) -> None: extraction = PaperExtraction(paper_id="Stage Paper", title="Stage Paper", findings=["F"]) synthesis = SynthesisResult(agreements=["Agreement"], gaps=["Gap"]) - handler = MagicMock(spec=EnhancedResponseHandler) - handler.process_structured_response = AsyncMock( - side_effect=[ - ProcessingResult(success=True, data=extraction, processing_path=ProcessingPath.DIRECT_SUCCESS), - ProcessingResult(success=True, data=synthesis, processing_path=ProcessingPath.DIRECT_SUCCESS), - ] - ) - - stage = SynthesisStage(handler=handler) - result = await stage.run(ctx, clusters) + stage = SynthesisStage() + with patch(STRUCTURED_TARGET, _structured_mock(extraction, synthesis)): + result = await stage.run(ctx, clusters) assert isinstance(result.output, SynthesisResult) assert ctx.get_artifact("paper_extractions") is not None @@ -448,10 +405,7 @@ async def test_gap_analysis_stage(self) -> None: synthesis = SynthesisResult(gaps=["Gap one"]) ctx = PipelineContext.create( "gap stage", - AppSettings( - synthesis={"llm_enabled": True}, - llm={"structured_outputs": False}, - ), + AppSettings(synthesis={"llm_enabled": True}), ) ctx.set_artifact("paper_clusters", [PaperCluster(theme="T", paper_ids=["p1"])]) @@ -459,10 +413,10 @@ async def test_gap_analysis_stage(self) -> None: gaps=["Gap one"], opportunities=["Opportunity one"], ) - handler = _mock_handler_success(gap_result) - stage = GapAnalysisStage(handler=handler) + stage = GapAnalysisStage() - result = await stage.run(ctx, synthesis) + with patch(STRUCTURED_TARGET, _structured_mock(gap_result)): + result = await stage.run(ctx, synthesis) assert isinstance(result.output, GapAnalysisResult) assert ctx.get_artifact("gap_analysis") == gap_result @@ -486,20 +440,20 @@ async def run(self, ctx: PipelineContext, data: object) -> object: synthesis = SynthesisResult(agreements=["A"], gaps=["G"]) gap_result = GapAnalysisResult(gaps=["G"], opportunities=["O"]) - handler = MagicMock(spec=EnhancedResponseHandler) - handler.process_structured_response = AsyncMock( - side_effect=[ - ProcessingResult(success=True, data=extraction, processing_path=ProcessingPath.DIRECT_SUCCESS), - ProcessingResult(success=True, data=synthesis, processing_path=ProcessingPath.DIRECT_SUCCESS), - ProcessingResult(success=True, data=gap_result, processing_path=ProcessingPath.DIRECT_SUCCESS), - ] - ) + by_type = { + PaperExtraction: extraction, + SynthesisResult: synthesis, + GapAnalysisResult: gap_result, + } + + async def fake_structured(role, prompt, output_type, llm_config=None, **kwargs): + return by_type[output_type] pipeline = ResearchPipeline( [ ClusteringStub(), - SynthesisStage(handler=handler), - GapAnalysisStage(handler=handler), + SynthesisStage(), + GapAnalysisStage(), ], AppSettings( pipeline={ @@ -512,7 +466,7 @@ async def run(self, ctx: PipelineContext, data: object) -> object: ), ) - with patch("src.analysis.synthesis.create_llm_agent", return_value=MagicMock()): + with patch(STRUCTURED_TARGET, AsyncMock(side_effect=fake_structured)): result = await pipeline.execute("pipeline query") assert "synthesis" in result.stage_results @@ -529,8 +483,9 @@ async def test_gap_analysis_accepts_cluster_list_after_synthesis_timeout(self) - ctx.set_artifact("ranked_papers", ranked) ctx.set_artifact("paper_clusters", clusters) - stage = GapAnalysisStage(handler=_mock_handler_failure()) - result = await stage.run(ctx, clusters) + stage = GapAnalysisStage() + with patch(STRUCTURED_TARGET, _structured_mock(None)): + result = await stage.run(ctx, clusters) assert isinstance(result.output, GapAnalysisResult) assert result.output.gaps @@ -562,7 +517,7 @@ def test_recover_stage_output_after_synthesis_timeout(self) -> None: @pytest.mark.asyncio async def test_extract_papers_limits_llm_calls(self) -> None: ranked = [_ranked(f"Paper {index}", abstract=f"Abstract {index}") for index in range(12)] - handler = _mock_handler_success( + structured = _structured_mock( PaperExtraction( paper_id="x", title="Paper", @@ -578,26 +533,24 @@ async def test_extract_papers_limits_llm_calls(self) -> None: synthesis={ "llm_enabled": True, "max_llm_papers": 3, - "extraction_max_retries": 0, "concurrency": 2, }, ) - extractions = await extract_papers( - ranked, - "query", - llm_config=_legacy_llm_config(), - handler=handler, - synthesis_config=settings.synthesis, - ) + with patch(STRUCTURED_TARGET, structured): + extractions = await extract_papers( + ranked, + "query", + synthesis_config=settings.synthesis, + ) assert len(extractions) == 12 - assert handler.process_structured_response.await_count == 3 + assert structured.await_count == 3 @pytest.mark.asyncio async def test_extract_papers_skips_llm_when_disabled(self) -> None: ranked = [_ranked(f"Paper {index}") for index in range(5)] - handler = _mock_handler_success( + structured = _structured_mock( PaperExtraction( paper_id="x", title="Paper", @@ -609,12 +562,12 @@ async def test_extract_papers_skips_llm_when_disabled(self) -> None: ) ) - extractions = await extract_papers( - ranked, - "query", - handler=handler, - synthesis_config=AppSettings(synthesis={"llm_enabled": False}).synthesis, - ) + with patch(STRUCTURED_TARGET, structured): + extractions = await extract_papers( + ranked, + "query", + synthesis_config=AppSettings(synthesis={"llm_enabled": False}).synthesis, + ) assert len(extractions) == 5 - assert handler.process_structured_response.await_count == 0 + assert structured.await_count == 0