From bd6cfce2865d041b80d699d4bb27af685c13cb00 Mon Sep 17 00:00:00 2001 From: Peekaboo <318562198+jaunatisq@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:58:48 +0800 Subject: [PATCH 1/3] docs: design opt-in Jev memory reranking --- .../2026-09-20-jev-memory-reranker-design.md | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md diff --git a/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md b/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md new file mode 100644 index 00000000..9db4823d --- /dev/null +++ b/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md @@ -0,0 +1,302 @@ +# Jev-Reranked Memory Retrieval + +**Status:** Proposed for implementation review +**Date:** 2026-09-20 +**Target:** memU `main`, as an opt-in integration with no default behavior change + +## Summary + +Add a Jev-powered second-stage relevance gate around memU's existing +`AgenticMemoryBackend` protocol. The existing backend continues to perform +embedding retrieval. The integration sends the returned segment and resource +candidates to TypeSafe Jev in one System One request, receives an independent +Noul probability for every candidate, filters and reorders the candidates, and +rolls the surviving segment scores back up to files. + +The integration is additive. `MemoryService`, its three public operations, +storage repositories, database schemas, cloud transport, and default host CLI +behavior remain unchanged. Users enable Jev explicitly and install an optional +dependency. With the feature disabled, no Jev package is imported and no +memory content is sent to TypeSafe. + +## Problem + +memU currently retrieves memory with one query embedding and vector similarity. +That path is fast and portable, but semantic similarity alone cannot reliably +separate useful evidence from topical or lexical overlap. A query about a +current deployment procedure can therefore retrieve an old discussion that +mentions the same tools without containing an actionable answer. + +Running a generative model over every retrieval would add seconds of latency, +unstructured output, and a new failure surface. Jev instead evaluates typed +questions about structured state and returns probabilities. TypeSafe reports a +70–500 ms service range for Jev-shaped calls, making a single batched relevance +evaluation suitable for memU's interactive retrieve path. This range is a +vendor report, not a latency guarantee; the implementation will measure and +publish observed warm p50 and p95 latency. + +## Goals + +- Make Jev materially responsible for the final memories returned to a caller. +- Use TypeSafe's real System One API through its official asynchronous Python + SDK, not a prompt-compatible chat model or a local heuristic. +- Make at most one Jev inference request per retrieval. +- Work with any object satisfying `AgenticMemoryBackend`, including local + `MemoryService` and `CloudMemoryClient`. +- Preserve current memU behavior and dependencies unless the integration is + explicitly enabled. +- Preserve user scope filtering by delegating the first-stage retrieval to the + configured backend without bypassing its `where` argument. +- Provide enough metadata to prove whether Jev was applied, fell back, or was + skipped because there were no candidates. +- Include deterministic tests, a credential-gated live smoke test, and a + repeatable latency benchmark. + +## Non-Goals + +- Replacing embeddings or vector indexes with Jev. +- Scanning an entire memory corpus through Jev. +- Changing the storage schema, repository protocols, or backend parity rules. +- Adding Jev calls to `MemoryService`; the core service remains embedding-only. +- Generating summaries, answers, or new memory content. +- Guaranteeing sub-100 ms or sub-second end-to-end latency across networks and + embedding providers. +- Enabling Jev silently for existing installations. + +## Considered Approaches + +### 1. Protocol-level reranking wrapper — selected + +Wrap any `AgenticMemoryBackend`. Delegate writes and listings unchanged. On +retrieval, call the wrapped backend, evaluate its bounded candidates with Jev, +and return the filtered and reordered result. + +This keeps the integration outside the composition root, works for local and +cloud backends, and needs no schema or repository changes. It also makes the +Jev dependency optional and testable behind a small evaluator protocol. + +### 2. Query routing only + +Use Jev to decide which tracks or layers to search, then use the existing vector +ranking unchanged. This sends less memory content to TypeSafe, but Jev would not +judge the actual candidate memories and would provide little protection against +false-positive similarity hits. + +### 3. Jev-only corpus scan + +Send the whole memory corpus to Jev and use its answers as retrieval. This makes +Jev central, but latency, request size, privacy exposure, and cost grow with the +corpus. It also discards memU's existing indexed candidate generation. This +approach is rejected. + +## Architecture + +### Integration package + +Create `memu.integrations.jev` containing these independent units: + +- `JevRerankConfig`: validated model, threshold, timeout, candidate bound, and + error-policy settings. +- `JevCandidate`: the minimal typed state sent for one segment or resource. +- `JevEvaluation`: model name, per-candidate probabilities, token usage, and + measured provider latency. +- `JevEvaluator` protocol: one asynchronous `evaluate(query, candidates)` call. +- `TypeSafeJevEvaluator`: the production evaluator using + `typesafe_sdk.AsyncTypeSafeClient` and Noul questions. +- `JevRerankedMemoryBackend`: an `AgenticMemoryBackend` wrapper that delegates + listing and commit calls and applies Jev only to retrieval. + +The production SDK import is lazy. Importing or running ordinary memU without +the `jev` extra must not require `typesafe-sdk`. + +### Configuration and opt-in + +Add an optional dependency: + +```toml +[project.optional-dependencies] +jev = ["typesafe-sdk>=0.7.0,<1"] +``` + +The shared backend builder will wrap the selected local or cloud backend only +when `MEMU_RETRIEVAL_RERANKER=jev`. Configuration resolves through memU's +existing process-environment then `~/.memu/config.env` lookup: + +| Setting | Default | Meaning | +| --- | --- | --- | +| `MEMU_RETRIEVAL_RERANKER` | unset | `jev` enables the wrapper | +| `TYPESAFE_API_KEY` | required when enabled | TypeSafe credential | +| `MEMU_JEV_MODEL` | `jev-latest` | TypeSafe model name or alias | +| `MEMU_JEV_MIN_RELEVANCE` | `0.5` | Minimum Noul probability kept | +| `MEMU_JEV_MAX_CANDIDATES` | `32` | Hard bound per Jev request | +| `MEMU_JEV_TIMEOUT_SECONDS` | `1.5` | Per-request SDK timeout | +| `MEMU_JEV_ON_ERROR` | `fallback` | `fallback` or `raise` | + +The API key is passed explicitly from memU's config resolver to the SDK so host +adapters and scheduled tasks can read it from the same stable config file as +other settings. It is never logged or included in result metadata. + +Developers can also construct `JevRerankedMemoryBackend` directly around a +custom backend and supply a custom evaluator or explicit configuration. + +## Retrieval Data Flow + +1. The wrapper calls `backend.progressive_retrieve(query, where=where)` exactly + once. Scope validation and candidate generation remain the wrapped backend's + responsibility. +2. It creates candidates from returned segments and resources: + - segment state contains its `text`; + - resource state contains its `caption`, falling back to its URL only when a + caption is absent. +3. It merges both layers by descending original vector score, with layer and + original position as stable tie-breakers, then truncates the combined list to + `max_candidates`. Existing memU defaults produce ten candidates, below the + proposed bound. Candidates beyond the bound are omitted from the Jev-shaped + result rather than being returned without evaluation; metadata reports the + number omitted. +4. If there are no candidates, the wrapper performs no Jev call and returns the + original three layers plus metadata explaining the skip. +5. The evaluator sends one System One request. The shared state contains the + query and the keyed candidate list. Each candidate receives one Noul + question with explicit criteria: + - true: the candidate contains concrete information directly useful for + answering the query; + - false: the candidate is irrelevant, only topically similar, or lacks + information useful for the answer. +6. Jev returns one independent probability per candidate. The wrapper keeps + candidates at or above `min_relevance`, orders them by Jev probability with + the original vector score as a deterministic tie-breaker, and adds a + `jev_score` field without overwriting the vector `score`. +7. Files are retained only when a surviving segment points to them. Their + `jev_score` is the maximum score of their surviving segments, and they are + ordered by that value. +8. The result keeps the existing `segments`, `files`, and `resources` keys and + adds a `jev` metadata object containing applied/fallback status, resolved + model, latency, token counts, candidate count, retained count, threshold, + and truncation count. + +When evaluation succeeds, this makes Jev authoritative over final inclusion and +ordering while preserving the original similarity score for inspection and +debugging. The explicit fallback policy is the only path that may return +unevaluated vector results while Jev is enabled. + +## Error Handling + +Configuration errors are always explicit: enabling Jev without the optional +package, an API key, or valid numeric settings raises before retrieval. + +Runtime provider errors follow `MEMU_JEV_ON_ERROR`: + +- `fallback` (default): return the untouched vector result plus `jev` metadata + with `applied=false`, `fallback=true`, and the exception class. Do not include + provider error messages because they may contain request details. +- `raise`: propagate the SDK exception so latency-sensitive or compliance- + sensitive callers can refuse an unjudged result. + +Malformed or incomplete Jev responses are runtime provider errors; they must +not be interpreted as zero relevance. Cancellation is never swallowed. + +The SDK client is asynchronous and connection-reusing. The wrapper exposes +`aclose` and asynchronous context-manager methods for long-lived library use; +short-lived host CLI processes may rely on their normal process teardown. + +## Privacy and Security + +Enabling the feature sends the retrieval query and bounded candidate text to +TypeSafe. Documentation must state this immediately beside the opt-in steps. +The wrapper sends neither embeddings nor unrelated corpus rows. User scope data +is not added to the Jev state, although candidate content may itself contain +personal information. + +The API key is read from configuration and passed directly to the official SDK. +Tests and logs must never print it. SDK request-body debug logging is not +enabled by the integration. + +## Compatibility + +- Default installs do not gain a new required dependency. +- Default backend construction returns the same concrete local or cloud object + as before. +- `MemoryService` retains its existing implementation and three-entry-point + public surface. +- No database migration or backend repository change is required. +- The opt-in wrapper satisfies `AgenticMemoryBackend` structurally. +- Existing result consumers that read the three documented layers continue to + work; opt-in callers may additionally consume `jev` and `jev_score`. + +## Testing + +### Deterministic tests + +- The wrapper delegates `list_all_recall_files` and `commit_results` exactly. +- Retrieval makes one evaluator call for segments and resources together. +- Returned state and Noul criteria identify the correct candidate. +- Thresholding, ordering, vector-score tie-breaking, and file roll-up are + correct. +- Candidate bounding and truncation metadata are correct. +- Empty candidates do not call Jev. +- `fallback` preserves the vector result; `raise` propagates. +- `asyncio.CancelledError` propagates under both policies. +- Enabling without the optional dependency or API key fails clearly. +- The environment builder leaves default local/cloud objects untouched and + wraps either backend only when explicitly configured. + +Tests use an injected evaluator rather than mocking TypeSafe SDK internals. +Separate contract tests mock the SDK transport to pin the real System One +request and response shapes. + +### Live smoke test + +A test marked `integration` runs only when `TYPESAFE_API_KEY` is present. It +uses the official SDK against TypeSafe, supplies one relevant and one irrelevant +candidate, asserts probabilities are in `[0, 1]`, asserts the resolved model is +returned, and proves the call is Jev rather than a local fake. CI without the +credential skips it with an explicit reason. + +### Benchmark + +Add a credential-gated script that runs a deterministic in-memory memU backend +plus the real `TypeSafeJevEvaluator`. It performs warm-up calls, then reports: + +- base retrieval latency; +- Jev provider latency; +- total retrieval latency; +- warm p50 and p95; +- candidate count and input-token usage. + +The PR will report measured numbers from the available execution environment +and label them with region, date, model, and sample size. It will not convert a +small local run into a universal latency guarantee. + +## Documentation + +- Add an accepted ADR recording why Jev lives in an opt-in protocol wrapper + instead of `MemoryService`. +- Add a README section with installation, configuration, privacy disclosure, + result shape, fallback behavior, and a runnable example. +- Document how to run deterministic tests, the live smoke test, and the + benchmark without placing a key on the command line. + +## Delivery and PR Shape + +The implementation will remain on `codex/jev-memory-reranker`, based directly +on upstream `main`. The design document is committed separately before code. +The implementation commit(s) will contain only the integration, configuration, +tests, benchmark, ADR, and user-facing documentation. No existing migration, +storage layout, or unrelated refactor is in scope. + +## Acceptance Criteria + +- Existing test suite and quality checks pass without the Jev extra enabled. +- The Jev-specific deterministic and transport-contract tests pass. +- A live credentialed run makes a real `jev-latest` call and returns calibrated + candidate probabilities. +- One retrieval produces no more than one Jev inference request. +- With the feature disabled, existing retrieval output and behavior are + unchanged. +- With the feature enabled, final segment/resource inclusion and ordering are + determined by Jev probabilities, and file ordering follows surviving segment + probabilities. +- Local and cloud backends can both be wrapped without changing their classes. +- No API key or candidate content appears in logs, exceptions created by memU, + or committed fixtures. From 9b49c067e352a6ad5237980014783645a709aeee Mon Sep 17 00:00:00 2001 From: Peekaboo <318562198+jaunatisq@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:38:02 +0800 Subject: [PATCH 2/3] feat: add opt-in Jev memory reranking --- README.md | 76 +++ .../0019-optional-jev-retrieval-reranker.md | 72 +++ docs/adr/README.md | 1 + .../2026-09-20-jev-memory-reranker-design.md | 8 +- pyproject.toml | 4 +- scripts/benchmark_jev_retrieval.py | 172 ++++++ src/memu/env.py | 61 +- src/memu/integrations/__init__.py | 1 + src/memu/integrations/jev.py | 420 +++++++++++++ tests/test_jev.py | 574 ++++++++++++++++++ tests/test_jev_sdk.py | 106 ++++ uv.lock | 88 ++- 12 files changed, 1568 insertions(+), 15 deletions(-) create mode 100644 docs/adr/0019-optional-jev-retrieval-reranker.md create mode 100644 scripts/benchmark_jev_retrieval.py create mode 100644 src/memu/integrations/__init__.py create mode 100644 src/memu/integrations/jev.py create mode 100644 tests/test_jev.py create mode 100644 tests/test_jev_sdk.py diff --git a/README.md b/README.md index 31ec150e..c999079f 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,82 @@ For Local / self-hosted installations, every CLI flag has a matching variable: Run ` doctor` to display the resolved mode and verify the same retrieval path the host uses. +### Optional Jev relevance reranking + +memU can use [TypeSafe Jev](https://typesafe.ai/) as an opt-in second-stage +relevance gate after the normal vector search. Jev evaluates all returned +segment and resource candidates in one System One request, removes candidates +below the configured probability threshold, and reranks the rest. The original +vector `score` remains in each result and the Jev probability is added as +`jev_score`. + +> [!IMPORTANT] +> Enabling this integration sends the retrieval query and the bounded candidate +> text to TypeSafe. It does not send embeddings or unrelated memory rows. Jev is +> disabled by default, and a default installation never imports its SDK or sends +> memory content to TypeSafe. + +Install the optional dependency: + +```bash +pip install "memu-cli[jev]" +``` + +Then add these values to `~/.memu/config.env` (keep the file private, for +example with mode `600` on POSIX systems): + +```dotenv +MEMU_RETRIEVAL_RERANKER=jev +TYPESAFE_API_KEY=your-typesafe-key +``` + +Do not put the API key in a command-line argument. Optional tuning settings: + +| Setting | Default | Meaning | +|---|---|---| +| `MEMU_JEV_MODEL` | `jev-latest` | TypeSafe model name or alias | +| `MEMU_JEV_MIN_RELEVANCE` | `0.5` | Minimum Noul probability retained | +| `MEMU_JEV_MAX_CANDIDATES` | `32` | Maximum candidates sent in one request | +| `MEMU_JEV_TIMEOUT_SECONDS` | `1.5` | Per-request timeout | +| `MEMU_JEV_ON_ERROR` | `fallback` | Return vector results with fallback metadata, or `raise` | + +The normal retrieve commands now use Jev automatically. Successful responses +retain the existing `segments`, `files`, and `resources` layers and add: + +```json +{ + "segments": [{"text": "...", "score": 0.81, "jev_score": 0.96}], + "files": [{"name": "deploy", "score": 0.81, "jev_score": 0.96}], + "resources": [], + "jev": { + "applied": true, + "fallback": false, + "model": "jev-1.13.0", + "latency_ms": 112, + "candidate_count": 6, + "retained_count": 2 + } +} +``` + +`fallback` is explicit: if TypeSafe is temporarily unavailable, the three +layers are returned unchanged and `jev.fallback` is `true`. Set +`MEMU_JEV_ON_ERROR=raise` when callers must refuse unevaluated results. + +To validate the SDK contract without contacting TypeSafe, run: + +```bash +uv run --extra jev python -m pytest tests/test_jev.py tests/test_jev_sdk.py -m "not integration" +``` + +With `TYPESAFE_API_KEY` exported or stored in `~/.memu/config.env`, the live +smoke test and latency benchmark are: + +```bash +MEMU_RUN_LIVE_JEV=1 uv run --extra jev python -m pytest tests/test_jev_sdk.py -m integration +uv run --extra jev python scripts/benchmark_jev_retrieval.py +``` + ### Storage backends | Provider | DSN | Vector search | Use for | diff --git a/docs/adr/0019-optional-jev-retrieval-reranker.md b/docs/adr/0019-optional-jev-retrieval-reranker.md new file mode 100644 index 00000000..9e34419a --- /dev/null +++ b/docs/adr/0019-optional-jev-retrieval-reranker.md @@ -0,0 +1,72 @@ +# ADR 0019: Put Jev in an Optional Retrieval Wrapper, Not `MemoryService` + +- Status: Accepted +- Date: 2026-09-21 +- Builds on: ADR 0002, ADR 0005, ADR 0012 + +## Context + +memU's interactive read path embeds a query, retrieves bounded segment and +resource candidates by vector similarity, and rolls segment hits up to files. +This is fast and portable, but vector similarity can rank topical overlap above +memory that directly answers the query. + +TypeSafe Jev is a System One decision model. It accepts structured state and +typed questions and returns probabilities rather than generated text. A bounded +set of Noul relevance questions can therefore distinguish useful memory from +false-positive similarity hits in one request. + +Putting that request inside `MemoryService` would violate two existing +boundaries. The service is intentionally embedding-only, and its three methods +work identically across pluggable storage backends. Jev also sends candidate +content to an external processor and adds an optional dependency, so it cannot +become a silent default. + +## Decision + +Implement Jev as `JevRerankedMemoryBackend`, a structural +`AgenticMemoryBackend` wrapper under `memu.integrations.jev`. + +The wrapper delegates `list_all_recall_files` and `commit_results` unchanged. +For `progressive_retrieve`, it: + +1. calls the configured local or cloud backend once; +2. merges the returned segment and resource candidates by vector score; +3. sends a bounded candidate set in one real Jev System One request, with one + independent Noul relevance question per candidate; +4. filters and orders segments/resources by Jev probability; and +5. rolls surviving segment probabilities up to files. + +The wrapper preserves the original vector `score`, adds `jev_score`, and emits a +small `jev` metadata object showing whether evaluation was applied or fell back. +It never changes storage, embeddings, or user-scope filtering. + +The integration disables SDK retries so the one-call bound also means at most +one outbound System One HTTP attempt. Availability remains governed by the +wrapper's explicit fallback or raise policy instead of hidden retry latency. + +The official asynchronous TypeSafe SDK is an optional `jev` package extra. The +shared backend builder installs the wrapper only when +`MEMU_RETRIEVAL_RERANKER=jev`. The default path does not import the SDK, require +a TypeSafe credential, change the backend object, or send memory content to +TypeSafe. + +Runtime provider failures either return the untouched vector result with +explicit fallback metadata or raise, selected by configuration. Cancellation +always propagates. Configuration errors are never converted into fallback. + +## Consequences + +- `MemoryService` remains embedding-only and keeps its exact public surface. +- Local and cloud retrieval gain the same opt-in Jev behavior without storage + backend changes or migrations. +- Jev is materially responsible for final inclusion and ordering when the + evaluation succeeds, while the vector stage remains the scalable candidate + generator. +- One retrieval makes no more than one Jev inference request. +- Enabling the integration sends the query and bounded candidate text to + TypeSafe; documentation must disclose this beside the opt-in instructions. +- End-to-end latency gains one network decision call. Benchmarks report observed + warm p50/p95 rather than treating TypeSafe's published range as a guarantee. +- The default fallback favors availability but is visibly distinguishable from + a Jev-ranked response; strict callers can select the raise policy. diff --git a/docs/adr/README.md b/docs/adr/README.md index ea967d68..1df2b607 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,3 +18,4 @@ - [0016: Client Event Reporting — One Envelope, a Spool by Default, Bounded Payloads](0016-client-event-reporting.md) - [0017: `config.env` Is Written by a Command — `init` for the Entry, `config` for the Detail](0017-config-env-as-a-command.md) - [0018: Mine Claude Cowork Through the Claude Code Bridge](0018-cowork-through-claude-code-bridge.md) +- [0019: Put Jev in an Optional Retrieval Wrapper, Not `MemoryService`](0019-optional-jev-retrieval-reranker.md) diff --git a/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md b/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md index 9db4823d..bd3dd04d 100644 --- a/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md +++ b/docs/superpowers/specs/2026-09-20-jev-memory-reranker-design.md @@ -1,7 +1,9 @@ # Jev-Reranked Memory Retrieval -**Status:** Proposed for implementation review -**Date:** 2026-09-20 +**Status:** Approved + +**Date:** 2026-09-20 + **Target:** memU `main`, as an opt-in integration with no default behavior change ## Summary @@ -115,7 +117,7 @@ Add an optional dependency: ```toml [project.optional-dependencies] -jev = ["typesafe-sdk>=0.7.0,<1"] +jev = ["typesafe-sdk>=0.7.0,<0.8"] ``` The shared backend builder will wrap the selected local or cloud backend only diff --git a/pyproject.toml b/pyproject.toml index fbaacfee..d17ced27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ test = [ ] [project.optional-dependencies] +jev = ["typesafe-sdk>=0.7.0,<0.8"] postgres = ["pgvector>=0.3.4", "sqlalchemy[postgresql-psycopgbinary]>=2.0.36"] [project.scripts] @@ -98,7 +99,7 @@ warn_unused_ignores = false disable_error_code = ["attr-defined", "call-arg"] [[tool.mypy.overrides]] -module = ["pgvector.*"] +module = ["pgvector.*", "typesafe_sdk.*"] ignore_missing_imports = true [tool.deptry.per_rule_ignores] @@ -170,3 +171,4 @@ testpaths = ["tests"] log_cli = true log_cli_level = "INFO" asyncio_mode = "auto" +markers = ["integration: live external-service tests requiring explicit credentials"] diff --git a/scripts/benchmark_jev_retrieval.py b/scripts/benchmark_jev_retrieval.py new file mode 100644 index 00000000..b58ecaec --- /dev/null +++ b/scripts/benchmark_jev_retrieval.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Measure warm memU + real Jev retrieval latency. + +The first-stage embeddings are deterministic and local so the reported Jev +latency isolates the real TypeSafe request rather than another provider. Set +``TYPESAFE_API_KEY`` in the environment or ``~/.memu/config.env``; never pass a +credential on the command line. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import statistics +import time +from datetime import UTC, datetime +from typing import Any + +from memu import env as menv +from memu.app import MemoryService +from memu.integrations.jev import JevRerankConfig, JevRerankedMemoryBackend, TypeSafeJevEvaluator + + +class DeterministicEmbeddingClient: + """Small local embedding fixture; Jev remains the only network model call.""" + + embed_model = "benchmark-keywords-v1" + + async def embed(self, inputs: list[str]) -> tuple[list[list[float]], None]: + return [self._vector(text) for text in inputs], None + + @staticmethod + def _vector(text: str) -> list[float]: + lowered = text.lower() + return [ + float(any(word in lowered for word in ("deploy", "release", "health", "staging"))), + float(any(word in lowered for word in ("lunch", "menu", "food"))), + float(any(word in lowered for word in ("coffee", "preference", "profile"))), + 0.1, + ] + + +def _percentile(samples: list[float], percentile: float) -> float: + ordered = sorted(samples) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +async def _seed_service() -> MemoryService: + service = MemoryService(database_config={"metadata_store": {"provider": "inmemory"}}) + embedding = DeterministicEmbeddingClient() + service._embedding_pool._cache["default"] = embedding + service._embedding_pool._cache["embedding"] = embedding + await service.commit_results( + recall_files=[ + { + "name": "deployment", + "track": "skill", + "description": "production deployment procedure", + "content": "Run make deploy.\nVerify the health endpoint.\nPromote the staging release.", + }, + { + "name": "lunch", + "track": "memory", + "description": "office lunch information", + "content": "The lunch menu changes every Friday.", + }, + { + "name": "profile", + "track": "memory", + "description": "user preferences", + "content": "The user prefers dark-roast coffee.", + }, + ], + resource=[ + {"path": "/workspace/DEPLOY.md", "description": "release and health-check runbook"}, + {"path": "/workspace/MENU.md", "description": "weekly office lunch menu"}, + ], + ) + return service + + +async def benchmark(args: argparse.Namespace) -> dict[str, Any]: + api_key = menv.env("TYPESAFE_API_KEY") + if not api_key: + message = "TYPESAFE_API_KEY is required in the environment or ~/.memu/config.env" + raise SystemExit(message) + + base = await _seed_service() + config = JevRerankConfig( + model=args.model, + min_relevance=args.min_relevance, + timeout_seconds=args.timeout, + ) + evaluator = TypeSafeJevEvaluator(api_key=api_key, config=config) + reranked = JevRerankedMemoryBackend(base, config=config, evaluator=evaluator) + query = "How do I deploy the service and verify it is healthy?" + + try: + for _ in range(args.warmup): + await reranked.progressive_retrieve(query) + + base_ms: list[float] = [] + total_ms: list[float] = [] + jev_ms: list[float] = [] + metadata: dict[str, Any] = {} + for _ in range(args.runs): + started = time.perf_counter() + await base.progressive_retrieve(query) + base_ms.append((time.perf_counter() - started) * 1000) + + started = time.perf_counter() + result = await reranked.progressive_retrieve(query) + total_ms.append((time.perf_counter() - started) * 1000) + metadata = result["jev"] + if not metadata.get("applied"): + message = f"Jev was not applied: {metadata.get('error_type', metadata.get('reason'))}" + raise RuntimeError(message) + jev_ms.append(float(metadata["latency_ms"])) + finally: + await reranked.aclose() + + return { + "measured_at": datetime.now(UTC).isoformat(), + "region": os.environ.get("MEMU_BENCH_REGION", "unspecified"), + "model": metadata.get("model", args.model), + "runs": args.runs, + "warmup_runs": args.warmup, + "candidate_count": metadata.get("candidate_count"), + "input_tokens_last_run": metadata.get("input_tokens"), + "base_retrieval_ms": { + "median": round(statistics.median(base_ms), 2), + "p95": round(_percentile(base_ms, 0.95), 2), + }, + "jev_provider_ms": { + "median": round(statistics.median(jev_ms), 2), + "p95": round(_percentile(jev_ms, 0.95), 2), + }, + "total_retrieval_ms": { + "median": round(statistics.median(total_ms), 2), + "p95": round(_percentile(total_ms, 0.95), 2), + }, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runs", type=int, default=10) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--model", default=menv.env("MEMU_JEV_MODEL", "jev-latest")) + parser.add_argument("--min-relevance", type=float, default=0.5) + parser.add_argument("--timeout", type=float, default=10.0) + return parser + + +def main() -> None: + args = _parser().parse_args() + if args.runs < 1 or args.warmup < 0: + message = "--runs must be positive and --warmup must not be negative" + raise SystemExit(message) + print(json.dumps(asyncio.run(benchmark(args)), indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/src/memu/env.py b/src/memu/env.py index 0567033b..145160f7 100644 --- a/src/memu/env.py +++ b/src/memu/env.py @@ -217,6 +217,46 @@ def cloud_api_key() -> str: return value +def retrieval_reranker() -> str | None: + """Resolve the optional post-retrieval integration.""" + value = (env("MEMU_RETRIEVAL_RERANKER") or "").strip().lower() + if not value: + return None + if value != "jev": + raise ConfigError("MEMU_RETRIEVAL_RERANKER", "must be 'jev' when set") + return value + + +def _with_retrieval_reranker(backend: AgenticMemoryBackend) -> AgenticMemoryBackend: + """Wrap ``backend`` only when the explicitly configured integration needs it.""" + if retrieval_reranker() is None: + return backend + + api_key = env("TYPESAFE_API_KEY") + if not api_key: + raise ConfigError( + "TYPESAFE_API_KEY", + f"is required when MEMU_RETRIEVAL_RERANKER=jev. Add it to {CONFIG_ENV} (or export it)", + ) + + from pydantic import ValidationError + + from memu.integrations.jev import JevRerankConfig, JevRerankedMemoryBackend + + raw_config = { + "model": env("MEMU_JEV_MODEL", "jev-latest"), + "min_relevance": env("MEMU_JEV_MIN_RELEVANCE", "0.5"), + "max_candidates": env("MEMU_JEV_MAX_CANDIDATES", "32"), + "timeout_seconds": env("MEMU_JEV_TIMEOUT_SECONDS", "1.5"), + "on_error": env("MEMU_JEV_ON_ERROR", "fallback"), + } + try: + config = JevRerankConfig.model_validate(raw_config) + except ValidationError as exc: + raise ConfigError("MEMU_JEV_*", "invalid Jev reranker configuration") from exc + return JevRerankedMemoryBackend(backend, config=config, api_key=api_key) + + def build_agentic_memory_backend_from_env( *, local_database: str | None = None, @@ -231,19 +271,20 @@ def build_agentic_memory_backend_from_env( if memory_mode() == "cloud": from memu.cloud import CloudMemoryClient - return CloudMemoryClient( + backend: AgenticMemoryBackend = CloudMemoryClient( base_url=cloud_base_url(), api_key=cloud_api_key(), ) - - from memu.app import MemoryService - - resolved_database = database_config(local_database if local_database is not None else require("MEMU_DB")) - resolved_embedding = local_embedding_profile if local_embedding_profile is not None else embedding_profile() - return MemoryService( - embedding_profiles={"default": resolved_embedding}, - database_config=resolved_database, - ) + else: + from memu.app import MemoryService + + resolved_database = database_config(local_database if local_database is not None else require("MEMU_DB")) + resolved_embedding = local_embedding_profile if local_embedding_profile is not None else embedding_profile() + backend = MemoryService( + embedding_profiles={"default": resolved_embedding}, + database_config=resolved_database, + ) + return _with_retrieval_reranker(backend) def build_service_from_env() -> Any: diff --git a/src/memu/integrations/__init__.py b/src/memu/integrations/__init__.py new file mode 100644 index 00000000..8d687058 --- /dev/null +++ b/src/memu/integrations/__init__.py @@ -0,0 +1 @@ +"""Optional integrations layered around memU's core agentic backend.""" diff --git a/src/memu/integrations/jev.py b/src/memu/integrations/jev.py new file mode 100644 index 00000000..b1d955f0 --- /dev/null +++ b/src/memu/integrations/jev.py @@ -0,0 +1,420 @@ +"""Opt-in TypeSafe Jev relevance filtering for memU retrieval results. + +The core :class:`memu.app.MemoryService` remains embedding-only. This module +wraps its three-operation backend protocol and applies one System One request +after ordinary vector retrieval. Importing the module does not require the +optional TypeSafe SDK; only constructing the production evaluator does. +""" + +from __future__ import annotations + +import inspect +import math +import time +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, Protocol, Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from memu.agentic_backend import AgenticMemoryBackend + +if TYPE_CHECKING: + from typesafe_sdk import JSONContent, NoulModel + +CandidateKind = Literal["segment", "resource"] +ErrorPolicy = Literal["fallback", "raise"] + + +class JevIntegrationError(RuntimeError): + """Base error for Jev integration configuration and response failures.""" + + +class JevDependencyError(JevIntegrationError): + """The optional official TypeSafe SDK is not installed.""" + + +class JevConfigurationError(JevIntegrationError): + """Jev was enabled without a usable credential or configuration.""" + + +class JevResponseError(JevIntegrationError): + """Jev returned probabilities that do not cover the requested candidates.""" + + +class JevRerankConfig(BaseModel): + """Validated settings for one bounded Jev relevance request.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + model: str = Field(default="jev-latest", min_length=1) + min_relevance: float = Field(default=0.5, ge=0.0, le=1.0) + max_candidates: int = Field(default=32, ge=1, le=255) + timeout_seconds: float = Field(default=1.5, gt=0.0, le=120.0) + on_error: ErrorPolicy = "fallback" + + @field_validator("model") + @classmethod + def _model_must_not_be_blank(cls, value: str) -> str: + model = value.strip() + if not model: + message = "model must not be blank" + raise ValueError(message) + return model + + +@dataclass(frozen=True, slots=True) +class JevCandidate: + """The minimal candidate state sent to Jev.""" + + key: str + kind: CandidateKind + text: str + original_score: float + position: int + + +@dataclass(frozen=True, slots=True) +class JevEvaluation: + """Normalized output from one Jev System One request.""" + + model: str + scores: dict[str, float] + latency_ms: int + input_tokens: int | None + output_tokens: int | None + + +class JevEvaluator(Protocol): + """Small injectable seam around Jev inference.""" + + async def evaluate(self, query: str, candidates: list[JevCandidate]) -> JevEvaluation: ... + + +class TypeSafeJevEvaluator: + """Real Jev evaluator backed by TypeSafe's official asynchronous SDK.""" + + def __init__( + self, + *, + api_key: str, + config: JevRerankConfig | Mapping[str, Any] | None = None, + client: Any | None = None, + ) -> None: + self.config = _validate_config(config) + if not api_key or not api_key.strip(): + message = "TYPESAFE_API_KEY must not be empty" + raise JevConfigurationError(message) + if client is None: + try: + from typesafe_sdk import AsyncTypeSafeClient, RetryPolicy + except ImportError as exc: # pragma: no cover - exact import failure is environment-specific + message = 'Install the Jev integration with: pip install "memu-cli[jev]"' + raise JevDependencyError(message) from exc + client = AsyncTypeSafeClient( + api_key=api_key, + model=self.config.model, + retry=RetryPolicy(max_retries=0), + timeout=self.config.timeout_seconds, + ) + self._client = client + self._closed = False + + async def evaluate(self, query: str, candidates: list[JevCandidate]) -> JevEvaluation: + if not candidates: + message = "Jev evaluation requires at least one candidate" + raise ValueError(message) + + state: JSONContent = { + "query": query, + "candidates": { + candidate.key: {"kind": candidate.kind, "content": candidate.text} for candidate in candidates + }, + } + questions: dict[str, NoulModel] = { + candidate.key: { + "type": "noul", + "instructions": ( + f"Does candidate {candidate.key} contain concrete information directly useful " + "for answering the query in the state?" + ), + "criteria": { + "true": "The candidate directly helps answer the query with concrete information.", + "false": ( + "The candidate is irrelevant, merely topically similar, or lacks information " + "useful for answering the query." + ), + }, + } + for candidate in candidates + } + + started = time.perf_counter() + response = await self._client.system_one( + state=state, + questions=questions, + model=self.config.model, + timeout=self.config.timeout_seconds, + ) + latency_ms = round((time.perf_counter() - started) * 1000) + scores = {key: float(answer.noul) for key, answer in response.nouls.items()} + _validate_scores(candidates, scores) + usage = response.usage + return JevEvaluation( + model=str(response.model), + scores=scores, + latency_ms=latency_ms, + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + ) + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + close = getattr(self._client, "aclose", None) + if close is not None: + result = close() + if inspect.isawaitable(result): + await result + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: Any, + ) -> None: + await self.aclose() + + +class JevRerankedMemoryBackend: + """Apply a single Jev relevance pass around any agentic memory backend.""" + + def __init__( + self, + backend: AgenticMemoryBackend, + *, + config: JevRerankConfig | Mapping[str, Any] | None = None, + evaluator: JevEvaluator | None = None, + api_key: str | None = None, + ) -> None: + self.backend = backend + self.config = _validate_config(config) + self._evaluator = evaluator or TypeSafeJevEvaluator(api_key=api_key or "", config=self.config) + self._closed = False + + async def list_all_recall_files( + self, + where: dict[str, Any] | None = None, + *, + cursor: str | None = None, + limit: int = 100, + ) -> dict[str, Any]: + return await self.backend.list_all_recall_files(where, cursor=cursor, limit=limit) + + async def commit_results( + self, + *, + recall_files: list[dict[str, Any]] | None = None, + resource: list[dict[str, Any]] | None = None, + user: dict[str, Any] | None = None, + ) -> dict[str, Any]: + return await self.backend.commit_results(recall_files=recall_files, resource=resource, user=user) + + async def progressive_retrieve( + self, + query: str, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + vector_result = await self.backend.progressive_retrieve(query, where=where) + all_candidates = _collect_candidates(vector_result) + candidates = all_candidates[: self.config.max_candidates] + truncated_count = len(all_candidates) - len(candidates) + if not candidates: + return { + **vector_result, + "jev": { + "applied": False, + "fallback": False, + "provider": "typesafe", + "model": self.config.model, + "reason": "no_candidates", + "candidate_count": 0, + "retained_count": 0, + "min_relevance": self.config.min_relevance, + "truncated_count": 0, + }, + } + + started = time.perf_counter() + try: + evaluation = await self._evaluator.evaluate(query, candidates) + _validate_scores(candidates, evaluation.scores) + except Exception as exc: + if self.config.on_error == "raise": + raise + return { + **vector_result, + "jev": { + "applied": False, + "fallback": True, + "provider": "typesafe", + "model": self.config.model, + "latency_ms": round((time.perf_counter() - started) * 1000), + "error_type": type(exc).__name__, + "candidate_count": len(candidates), + "retained_count": len(all_candidates), + "min_relevance": self.config.min_relevance, + "truncated_count": truncated_count, + }, + } + + retained = [ + candidate for candidate in candidates if evaluation.scores[candidate.key] >= self.config.min_relevance + ] + segments = _materialize_layer("segment", retained, evaluation.scores, vector_result.get("segments", [])) + resources = _materialize_layer("resource", retained, evaluation.scores, vector_result.get("resources", [])) + files = _roll_up_files(vector_result.get("files", []), segments) + return { + **vector_result, + "segments": segments, + "files": files, + "resources": resources, + "jev": { + "applied": True, + "fallback": False, + "provider": "typesafe", + "model": evaluation.model, + "latency_ms": evaluation.latency_ms, + "input_tokens": evaluation.input_tokens, + "output_tokens": evaluation.output_tokens, + "candidate_count": len(candidates), + "retained_count": len(retained), + "min_relevance": self.config.min_relevance, + "truncated_count": truncated_count, + }, + } + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + close = getattr(self._evaluator, "aclose", None) + if close is not None: + result = close() + if inspect.isawaitable(result): + await result + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: Any, + ) -> None: + await self.aclose() + + +def _validate_config(config: JevRerankConfig | Mapping[str, Any] | None) -> JevRerankConfig: + if isinstance(config, JevRerankConfig): + return config + return JevRerankConfig.model_validate(config or {}) + + +def _collect_candidates(result: Mapping[str, Any]) -> list[JevCandidate]: + candidates: list[JevCandidate] = [] + for position, item in enumerate(result.get("segments", [])): + candidates.append( + JevCandidate( + key=f"segment_{position}", + kind="segment", + text=str(item.get("text") or ""), + original_score=float(item.get("score", 0.0)), + position=position, + ) + ) + for position, item in enumerate(result.get("resources", [])): + candidates.append( + JevCandidate( + key=f"resource_{position}", + kind="resource", + text=str(item.get("caption") or item.get("url") or ""), + original_score=float(item.get("score", 0.0)), + position=position, + ) + ) + kind_order = {"segment": 0, "resource": 1} + return sorted( + candidates, + key=lambda candidate: (-candidate.original_score, kind_order[candidate.kind], candidate.position), + ) + + +def _validate_scores(candidates: list[JevCandidate], scores: Mapping[str, float]) -> None: + expected = {candidate.key for candidate in candidates} + actual = set(scores) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + message = f"Jev answer keys did not match candidates (missing={missing}, unexpected={unexpected})" + raise JevResponseError(message) + if any(not math.isfinite(score) or not 0.0 <= score <= 1.0 for score in scores.values()): + message = "Jev relevance probabilities must be finite values from 0 to 1" + raise JevResponseError(message) + + +def _materialize_layer( + kind: CandidateKind, + retained: list[JevCandidate], + scores: Mapping[str, float], + items: list[dict[str, Any]], +) -> list[dict[str, Any]]: + ranked: list[tuple[JevCandidate, dict[str, Any]]] = [] + for candidate in retained: + if candidate.kind != kind: + continue + item = dict(items[candidate.position]) + item["jev_score"] = scores[candidate.key] + ranked.append((candidate, item)) + ranked.sort(key=lambda pair: (-scores[pair[0].key], -pair[0].original_score, pair[0].position)) + return [item for _, item in ranked] + + +def _roll_up_files(files: list[dict[str, Any]], segments: list[dict[str, Any]]) -> list[dict[str, Any]]: + file_scores: dict[Any, float] = {} + for segment in segments: + file_id = segment.get("recall_file_id") + if file_id is None: + continue + score = float(segment["jev_score"]) + file_scores[file_id] = max(score, file_scores.get(file_id, -math.inf)) + + ranked: list[dict[str, Any]] = [] + for item in files: + file_id = item.get("id") + if file_id not in file_scores: + continue + materialized = dict(item) + materialized["jev_score"] = file_scores[file_id] + ranked.append(materialized) + ranked.sort(key=lambda item: (-float(item["jev_score"]), -float(item.get("score", 0.0)))) + return ranked + + +__all__ = [ + "JevCandidate", + "JevConfigurationError", + "JevDependencyError", + "JevEvaluation", + "JevEvaluator", + "JevIntegrationError", + "JevRerankConfig", + "JevRerankedMemoryBackend", + "JevResponseError", + "TypeSafeJevEvaluator", +] diff --git a/tests/test_jev.py b/tests/test_jev.py new file mode 100644 index 00000000..f5397fc6 --- /dev/null +++ b/tests/test_jev.py @@ -0,0 +1,574 @@ +from __future__ import annotations + +import asyncio +import builtins +import sys +from collections.abc import Iterator +from copy import deepcopy +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest + +from memu import env as menv +from memu.app import MemoryService +from memu.cloud import CloudMemoryClient +from memu.integrations.jev import ( + JevCandidate, + JevDependencyError, + JevEvaluation, + JevRerankConfig, + JevRerankedMemoryBackend, + TypeSafeJevEvaluator, +) + + +@pytest.fixture(autouse=True) +def _clear_memu_env_cache() -> Iterator[None]: + """Keep config-file values from leaking beyond the test that loaded them.""" + menv.reload() + yield + menv.reload() + + +def _retrieval_result() -> dict[str, Any]: + return { + "segments": [ + {"id": "s1", "recall_file_id": "f1", "text": "topical only", "score": 0.90}, + {"id": "s2", "recall_file_id": "f2", "text": "direct answer", "score": 0.80}, + {"id": "s3", "recall_file_id": "f2", "text": "supporting detail", "score": 0.70}, + ], + "files": [ + {"id": "f1", "name": "noise", "score": 0.90}, + {"id": "f2", "name": "answer", "score": 0.80}, + ], + "resources": [ + {"id": "r1", "caption": "useful reference", "url": "/useful", "score": 0.85}, + {"id": "r2", "caption": "irrelevant reference", "url": "/noise", "score": 0.60}, + ], + } + + +class StubBackend: + def __init__(self, result: dict[str, Any] | None = None) -> None: + self.result = result if result is not None else _retrieval_result() + self.retrieve_calls: list[tuple[str, dict[str, Any] | None]] = [] + self.list_calls: list[tuple[dict[str, Any] | None, str | None, int]] = [] + self.commit_calls: list[dict[str, Any]] = [] + + async def list_all_recall_files( + self, + where: dict[str, Any] | None = None, + *, + cursor: str | None = None, + limit: int = 100, + ) -> dict[str, Any]: + self.list_calls.append((where, cursor, limit)) + return {"recall_files": [{"name": "one"}], "next_cursor": None} + + async def progressive_retrieve( + self, + query: str, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + self.retrieve_calls.append((query, where)) + return deepcopy(self.result) + + async def commit_results( + self, + *, + recall_files: list[dict[str, Any]] | None = None, + resource: list[dict[str, Any]] | None = None, + user: dict[str, Any] | None = None, + ) -> dict[str, Any]: + call = {"recall_files": recall_files, "resource": resource, "user": user} + self.commit_calls.append(call) + return call + + +class StubEvaluator: + def __init__( + self, + scores: dict[str, float] | None = None, + *, + error: BaseException | None = None, + ) -> None: + self.scores = scores or {} + self.error = error + self.calls: list[tuple[str, list[JevCandidate]]] = [] + self.closed = False + + async def evaluate(self, query: str, candidates: list[JevCandidate]) -> JevEvaluation: + self.calls.append((query, list(candidates))) + if self.error is not None: + raise self.error + return JevEvaluation( + model="jev-1.13.0", + scores=dict(self.scores), + latency_ms=87, + input_tokens=123, + output_tokens=5, + ) + + async def aclose(self) -> None: + self.closed = True + + +async def test_wrapper_delegates_non_retrieval_operations() -> None: + backend = StubBackend() + evaluator = StubEvaluator() + service = JevRerankedMemoryBackend(backend, evaluator=evaluator) + + listed = await service.list_all_recall_files({"user_id": "u1"}, cursor="next", limit=7) + committed = await service.commit_results( + recall_files=[{"name": "profile"}], + resource=[{"path": "/notes"}], + user={"user_id": "u1"}, + ) + + assert listed == {"recall_files": [{"name": "one"}], "next_cursor": None} + assert backend.list_calls == [({"user_id": "u1"}, "next", 7)] + assert committed == { + "recall_files": [{"name": "profile"}], + "resource": [{"path": "/notes"}], + "user": {"user_id": "u1"}, + } + assert evaluator.calls == [] + + +async def test_one_jev_call_filters_and_reranks_every_layer() -> None: + backend = StubBackend() + evaluator = StubEvaluator({ + "segment_0": 0.20, + "segment_1": 0.95, + "segment_2": 0.60, + "resource_0": 0.70, + "resource_1": 0.40, + }) + service = JevRerankedMemoryBackend( + backend, + config=JevRerankConfig(min_relevance=0.5), + evaluator=evaluator, + ) + + result = await service.progressive_retrieve("how do I deploy?", where={"user_id": "u1"}) + + assert backend.retrieve_calls == [("how do I deploy?", {"user_id": "u1"})] + assert len(evaluator.calls) == 1 + query, candidates = evaluator.calls[0] + assert query == "how do I deploy?" + # Candidates from both layers share one request and are merged by their + # original vector score instead of starving the resource layer. + assert [candidate.key for candidate in candidates] == [ + "segment_0", + "resource_0", + "segment_1", + "segment_2", + "resource_1", + ] + assert [(candidate.kind, candidate.text) for candidate in candidates] == [ + ("segment", "topical only"), + ("resource", "useful reference"), + ("segment", "direct answer"), + ("segment", "supporting detail"), + ("resource", "irrelevant reference"), + ] + + assert [(item["id"], item["jev_score"]) for item in result["segments"]] == [ + ("s2", 0.95), + ("s3", 0.60), + ] + assert [(item["id"], item["jev_score"]) for item in result["files"]] == [("f2", 0.95)] + assert [(item["id"], item["jev_score"]) for item in result["resources"]] == [("r1", 0.70)] + # Jev is additive: the original vector score remains inspectable. + assert result["segments"][0]["score"] == 0.80 + assert result["jev"] == { + "applied": True, + "fallback": False, + "provider": "typesafe", + "model": "jev-1.13.0", + "latency_ms": 87, + "input_tokens": 123, + "output_tokens": 5, + "candidate_count": 5, + "retained_count": 3, + "min_relevance": 0.5, + "truncated_count": 0, + } + + +async def test_equal_jev_scores_use_vector_score_as_tiebreaker() -> None: + backend = StubBackend({ + "segments": [ + {"id": "low", "recall_file_id": "f1", "text": "low", "score": 0.4}, + {"id": "high", "recall_file_id": "f2", "text": "high", "score": 0.9}, + ], + "files": [{"id": "f1"}, {"id": "f2"}], + "resources": [], + }) + evaluator = StubEvaluator({"segment_0": 0.8, "segment_1": 0.8}) + service = JevRerankedMemoryBackend(backend, evaluator=evaluator) + + result = await service.progressive_retrieve("query") + + assert [item["id"] for item in result["segments"]] == ["high", "low"] + + +async def test_candidate_bound_keeps_best_cross_layer_scores_and_reports_truncation() -> None: + evaluator = StubEvaluator({"segment_0": 0.8, "resource_0": 0.7}) + service = JevRerankedMemoryBackend( + StubBackend(), + config=JevRerankConfig(max_candidates=2), + evaluator=evaluator, + ) + + result = await service.progressive_retrieve("query") + + assert [candidate.key for candidate in evaluator.calls[0][1]] == ["segment_0", "resource_0"] + assert [item["id"] for item in result["segments"]] == ["s1"] + assert [item["id"] for item in result["resources"]] == ["r1"] + assert result["jev"]["candidate_count"] == 2 + assert result["jev"]["truncated_count"] == 3 + + +async def test_empty_result_skips_jev() -> None: + evaluator = StubEvaluator() + service = JevRerankedMemoryBackend( + StubBackend({"segments": [], "files": [], "resources": []}), + evaluator=evaluator, + ) + + result = await service.progressive_retrieve("query") + + assert evaluator.calls == [] + assert result == { + "segments": [], + "files": [], + "resources": [], + "jev": { + "applied": False, + "fallback": False, + "provider": "typesafe", + "model": "jev-latest", + "reason": "no_candidates", + "candidate_count": 0, + "retained_count": 0, + "min_relevance": 0.5, + "truncated_count": 0, + }, + } + + +async def test_provider_error_falls_back_without_mutating_vector_layers() -> None: + original = _retrieval_result() + evaluator = StubEvaluator(error=RuntimeError("request body must not leak")) + service = JevRerankedMemoryBackend(StubBackend(original), evaluator=evaluator) + + result = await service.progressive_retrieve("query") + + assert {key: result[key] for key in ("segments", "files", "resources")} == original + assert result["jev"]["applied"] is False + assert result["jev"]["fallback"] is True + assert result["jev"]["error_type"] == "RuntimeError" + assert result["jev"]["retained_count"] == 5 + assert "request body" not in str(result["jev"]) + + +async def test_fallback_retains_candidates_beyond_the_jev_request_bound() -> None: + service = JevRerankedMemoryBackend( + StubBackend(), + config=JevRerankConfig(max_candidates=2), + evaluator=StubEvaluator(error=RuntimeError("down")), + ) + + result = await service.progressive_retrieve("query") + + assert len(result["segments"]) + len(result["resources"]) == 5 + assert result["jev"]["candidate_count"] == 2 + assert result["jev"]["retained_count"] == 5 + assert result["jev"]["truncated_count"] == 3 + + +async def test_provider_error_can_be_strict() -> None: + service = JevRerankedMemoryBackend( + StubBackend(), + config=JevRerankConfig(on_error="raise"), + evaluator=StubEvaluator(error=RuntimeError("down")), + ) + + with pytest.raises(RuntimeError, match="down"): + await service.progressive_retrieve("query") + + +async def test_cancellation_is_never_converted_to_fallback() -> None: + service = JevRerankedMemoryBackend( + StubBackend(), + evaluator=StubEvaluator(error=asyncio.CancelledError()), + ) + + with pytest.raises(asyncio.CancelledError): + await service.progressive_retrieve("query") + + +async def test_incomplete_evaluation_is_a_provider_error() -> None: + service = JevRerankedMemoryBackend( + StubBackend(), + evaluator=StubEvaluator({"segment_0": 0.9}), + ) + + result = await service.progressive_retrieve("query") + + assert result["jev"]["fallback"] is True + assert result["jev"]["error_type"] == "JevResponseError" + + +async def test_wrapper_closes_evaluator() -> None: + evaluator = StubEvaluator() + service = JevRerankedMemoryBackend(StubBackend(), evaluator=evaluator) + + async with service: + pass + + assert evaluator.closed is True + + +async def test_real_memory_service_scopes_candidates_before_jev() -> None: + class EmbeddingClient: + embed_model = "scope-test" + + async def embed(self, inputs: list[str]) -> tuple[list[list[float]], None]: + return [[1.0, 0.0] for _ in inputs], None + + service = MemoryService(database_config={"metadata_store": {"provider": "inmemory"}}) + embedding = EmbeddingClient() + service._embedding_pool._cache["default"] = embedding + service._embedding_pool._cache["embedding"] = embedding + await service.commit_results( + recall_files=[{"name": "private", "track": "memory", "description": "u1", "content": "u1-only memory"}], + resource=[{"path": "/u1", "description": "u1-only resource"}], + user={"user_id": "u1"}, + ) + await service.commit_results( + recall_files=[{"name": "private", "track": "memory", "description": "u2", "content": "u2-secret memory"}], + resource=[{"path": "/u2", "description": "u2-secret resource"}], + user={"user_id": "u2"}, + ) + evaluator = StubEvaluator({"segment_0": 0.9, "resource_0": 0.8}) + reranked = JevRerankedMemoryBackend(service, evaluator=evaluator) + + result = await reranked.progressive_retrieve("private memory", where={"user_id": "u1"}) + + sent_text = [candidate.text for candidate in evaluator.calls[0][1]] + assert sent_text == ["u1-only memory", "u1-only resource"] + assert all("u2-secret" not in text for text in sent_text) + assert [segment["text"] for segment in result["segments"]] == ["u1-only memory"] + assert [resource["caption"] for resource in result["resources"]] == ["u1-only resource"] + + +async def test_typesafe_evaluator_builds_one_batched_noul_request() -> None: + class Client: + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + self.closed = False + + async def system_one(self, **kwargs: Any) -> Any: + self.calls.append(kwargs) + return SimpleNamespace( + model="jev-1.13.0", + nouls={ + "segment_0": SimpleNamespace(noul=0.91), + "resource_0": SimpleNamespace(noul=0.12), + }, + usage=SimpleNamespace(input_tokens=42, output_tokens=2), + ) + + async def aclose(self) -> None: + self.closed = True + + client = Client() + evaluator = TypeSafeJevEvaluator( + api_key="test-key", + config=JevRerankConfig(model="jev-latest", timeout_seconds=2), + client=client, + ) + + result = await evaluator.evaluate( + "deploy?", + [ + JevCandidate("segment_0", "segment", "run deploy", 0.9, 0), + JevCandidate("resource_0", "resource", "lunch menu", 0.8, 0), + ], + ) + await evaluator.aclose() + + assert len(client.calls) == 1 + request = client.calls[0] + assert request["model"] == "jev-latest" + assert request["timeout"] == 2 + assert request["state"]["query"] == "deploy?" + assert set(request["questions"]) == {"segment_0", "resource_0"} + assert all(question["type"] == "noul" for question in request["questions"].values()) + assert result.scores == {"segment_0": 0.91, "resource_0": 0.12} + assert result.input_tokens == 42 + assert client.closed is True + + +def test_typesafe_evaluator_disables_sdk_retries(monkeypatch: pytest.MonkeyPatch) -> None: + created: dict[str, Any] = {} + + class RetryPolicy: + def __init__(self, *, max_retries: int) -> None: + self.max_retries = max_retries + + class Client: + def __init__(self, **kwargs: Any) -> None: + created.update(kwargs) + + sdk = ModuleType("typesafe_sdk") + sdk.AsyncTypeSafeClient = Client # type: ignore[attr-defined] + sdk.RetryPolicy = RetryPolicy # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "typesafe_sdk", sdk) + + TypeSafeJevEvaluator(api_key="test-key") + + assert created["api_key"] == "test-key" + assert created["model"] == "jev-latest" + assert created["timeout"] == 1.5 + assert created["retry"].max_retries == 0 + + +def test_typesafe_evaluator_explains_missing_optional_dependency(monkeypatch: pytest.MonkeyPatch) -> None: + real_import = builtins.__import__ + + def import_without_sdk(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "typesafe_sdk": + raise ImportError(name) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_sdk) + + with pytest.raises(JevDependencyError, match=r"memu-cli\[jev\]"): + TypeSafeJevEvaluator(api_key="test-key") + + +def test_config_validates_bounds() -> None: + with pytest.raises(ValueError): + JevRerankConfig(min_relevance=1.01) + with pytest.raises(ValueError): + JevRerankConfig(max_candidates=0) + with pytest.raises(ValueError): + JevRerankConfig(timeout_seconds=0) + + +def test_backend_builder_is_unchanged_when_reranker_is_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_DB", ":memory:") + monkeypatch.delenv("MEMU_RETRIEVAL_RERANKER", raising=False) + menv.reload() + + backend = menv.build_agentic_memory_backend_from_env() + + assert type(backend) is MemoryService + + +def test_backend_builder_wraps_local_backend_when_jev_is_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_DB", ":memory:") + monkeypatch.setenv("MEMU_RETRIEVAL_RERANKER", "jev") + monkeypatch.setenv("TYPESAFE_API_KEY", "test-key") + menv.reload() + evaluator = StubEvaluator() + monkeypatch.setattr("memu.integrations.jev.TypeSafeJevEvaluator", lambda **_: evaluator) + + backend = menv.build_agentic_memory_backend_from_env() + + assert isinstance(backend, JevRerankedMemoryBackend) + assert type(backend.backend) is MemoryService + + +def test_backend_builder_wraps_cloud_backend_when_jev_is_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_MEMORY_MODE", "cloud") + monkeypatch.setenv("MEMU_CLOUD_API_KEY", "memu-key") + monkeypatch.setenv("MEMU_RETRIEVAL_RERANKER", "jev") + monkeypatch.setenv("TYPESAFE_API_KEY", "typesafe-key") + menv.reload() + evaluator = StubEvaluator() + monkeypatch.setattr("memu.integrations.jev.TypeSafeJevEvaluator", lambda **_: evaluator) + + backend = menv.build_agentic_memory_backend_from_env() + + assert isinstance(backend, JevRerankedMemoryBackend) + assert isinstance(backend.backend, CloudMemoryClient) + + +def test_backend_builder_rejects_unknown_reranker(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_DB", ":memory:") + monkeypatch.setenv("MEMU_RETRIEVAL_RERANKER", "mystery") + menv.reload() + + with pytest.raises(menv.ConfigError, match="MEMU_RETRIEVAL_RERANKER"): + menv.build_agentic_memory_backend_from_env() + + +def test_backend_builder_requires_typesafe_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_DB", ":memory:") + monkeypatch.setenv("MEMU_RETRIEVAL_RERANKER", "jev") + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + menv.reload() + + with pytest.raises(menv.ConfigError, match="TYPESAFE_API_KEY"): + menv.build_agentic_memory_backend_from_env() + + +def test_backend_builder_reads_typesafe_key_and_tuning_from_config_file( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + config_file = tmp_path / "config.env" + config_file.write_text( + "\n".join([ + "MEMU_DB=:memory:", + "MEMU_RETRIEVAL_RERANKER=jev", + "TYPESAFE_API_KEY=file-key", + "MEMU_JEV_MIN_RELEVANCE=0.7", + "MEMU_JEV_MAX_CANDIDATES=9", + "MEMU_JEV_ON_ERROR=raise", + ]), + encoding="utf-8", + ) + monkeypatch.setenv("MEMU_CONFIG_ENV", str(config_file)) + for key in ( + "MEMU_DB", + "MEMU_RETRIEVAL_RERANKER", + "TYPESAFE_API_KEY", + "MEMU_JEV_MIN_RELEVANCE", + "MEMU_JEV_MAX_CANDIDATES", + "MEMU_JEV_ON_ERROR", + ): + monkeypatch.delenv(key, raising=False) + menv.reload() + captured: dict[str, Any] = {} + + def evaluator_factory(**kwargs: Any) -> StubEvaluator: + captured.update(kwargs) + return StubEvaluator() + + monkeypatch.setattr("memu.integrations.jev.TypeSafeJevEvaluator", evaluator_factory) + + backend = menv.build_agentic_memory_backend_from_env() + + assert isinstance(backend, JevRerankedMemoryBackend) + assert captured["api_key"] == "file-key" + assert backend.config.min_relevance == 0.7 + assert backend.config.max_candidates == 9 + assert backend.config.on_error == "raise" + + +def test_backend_builder_rejects_invalid_jev_settings(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMU_DB", ":memory:") + monkeypatch.setenv("MEMU_RETRIEVAL_RERANKER", "jev") + monkeypatch.setenv("TYPESAFE_API_KEY", "test-key") + monkeypatch.setenv("MEMU_JEV_MIN_RELEVANCE", "2") + menv.reload() + + with pytest.raises(menv.ConfigError, match=r"MEMU_JEV_\*"): + menv.build_agentic_memory_backend_from_env() diff --git a/tests/test_jev_sdk.py b/tests/test_jev_sdk.py new file mode 100644 index 00000000..c7ef366a --- /dev/null +++ b/tests/test_jev_sdk.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +import os +from typing import Any + +import pytest + +from memu import env as menv + +typesafe_sdk = pytest.importorskip("typesafe_sdk", reason="install the 'jev' extra to test the official SDK contract") +httpx2 = pytest.importorskip("httpx2") + +LIVE_API_KEY = menv.env("TYPESAFE_API_KEY") +RUN_LIVE = os.environ.get("MEMU_RUN_LIVE_JEV") == "1" + +from memu.integrations.jev import ( # noqa: E402 + JevCandidate, + JevRerankConfig, + TypeSafeJevEvaluator, +) + + +async def test_official_sdk_sends_one_system_one_request() -> None: + requests: list[Any] = [] + + async def handler(request: Any) -> Any: + requests.append(request) + return httpx2.Response( + 200, + json={ + "model": "jev-1.13.0", + "answers": { + "segment_0": {"type": "noul", "noul": 0.94}, + "resource_0": {"type": "noul", "noul": 0.08}, + }, + "usage": {"input_tokens": 77, "output_tokens": 2}, + }, + ) + + client = typesafe_sdk.AsyncTypeSafeClient( + api_key="test-key", + transport=httpx2.MockTransport(handler), + ) + evaluator = TypeSafeJevEvaluator( + api_key="test-key", + config=JevRerankConfig(model="jev-latest"), + client=client, + ) + + result = await evaluator.evaluate( + "How do I deploy?", + [ + JevCandidate("segment_0", "segment", "Run make deploy", 0.8, 0), + JevCandidate("resource_0", "resource", "Lunch menu", 0.7, 0), + ], + ) + await evaluator.aclose() + + assert len(requests) == 1 + assert requests[0].url.path == "/v1/systemone" + payload = json.loads(requests[0].content) + assert payload["model"] == "jev-latest" + assert payload["state"] == { + "query": "How do I deploy?", + "candidates": { + "segment_0": {"kind": "segment", "content": "Run make deploy"}, + "resource_0": {"kind": "resource", "content": "Lunch menu"}, + }, + } + assert set(payload["questions"]) == {"segment_0", "resource_0"} + for key, question in payload["questions"].items(): + assert question["type"] == "noul" + assert key in question["instructions"] + assert set(question["criteria"]) == {"true", "false"} + + assert result.model == "jev-1.13.0" + assert result.scores == {"segment_0": 0.94, "resource_0": 0.08} + assert result.input_tokens == 77 + assert result.output_tokens == 2 + assert result.latency_ms >= 0 + + +@pytest.mark.integration +@pytest.mark.skipif(not (RUN_LIVE and LIVE_API_KEY), reason="set MEMU_RUN_LIVE_JEV=1 and TYPESAFE_API_KEY for live Jev") +async def test_live_jev_returns_probabilities_for_memory_candidates() -> None: + assert LIVE_API_KEY is not None + evaluator = TypeSafeJevEvaluator( + api_key=LIVE_API_KEY, + config=JevRerankConfig(model=menv.env("MEMU_JEV_MODEL", "jev-latest") or "jev-latest", timeout_seconds=10), + ) + try: + result = await evaluator.evaluate( + "How do I deploy the service?", + [ + JevCandidate("segment_0", "segment", "Run make deploy and verify the health endpoint.", 0.8, 0), + JevCandidate("resource_0", "resource", "The office lunch menu changes every Friday.", 0.7, 0), + ], + ) + finally: + await evaluator.aclose() + + assert result.model.startswith("jev") + assert set(result.scores) == {"segment_0", "resource_0"} + assert all(0 <= score <= 1 for score in result.scores.values()) + assert result.input_tokens is None or result.input_tokens > 0 diff --git a/uv.lock b/uv.lock index 017c8f2e..6d3ed46b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 3 requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' or sys_platform != 'emscripten'", +] [[package]] name = "alembic" @@ -228,6 +232,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/cb/48e964c452ca2b92175a9b2dca037a553036cb053ba69e284650ce755f13/greenlet-3.3.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e29f3018580e8412d6aaf5641bb7745d38c85228dacf51a73bd4e26ddf2a6a8e", size = 274908, upload-time = "2025-12-04T14:23:26.435Z" }, { url = "https://files.pythonhosted.org/packages/28/da/38d7bff4d0277b594ec557f479d65272a893f1f2a716cad91efeb8680953/greenlet-3.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a687205fb22794e838f947e2194c0566d3812966b41c78709554aa883183fb62", size = 577113, upload-time = "2025-12-04T14:50:05.493Z" }, { url = "https://files.pythonhosted.org/packages/3c/f2/89c5eb0faddc3ff014f1c04467d67dee0d1d334ab81fadbf3744847f8a8a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4243050a88ba61842186cb9e63c7dfa677ec146160b0efd73b855a3d9c7fcf32", size = 590338, upload-time = "2025-12-04T14:57:41.136Z" }, + { url = "https://files.pythonhosted.org/packages/80/d7/db0a5085035d05134f8c089643da2b44cc9b80647c39e93129c5ef170d8f/greenlet-3.3.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:670d0f94cd302d81796e37299bcd04b95d62403883b24225c6b5271466612f45", size = 601098, upload-time = "2025-12-04T15:07:11.898Z" }, { url = "https://files.pythonhosted.org/packages/dc/a6/e959a127b630a58e23529972dbc868c107f9d583b5a9f878fb858c46bc1a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb3a8ec3db4a3b0eb8a3c25436c2d49e3505821802074969db017b87bc6a948", size = 590206, upload-time = "2025-12-04T14:26:01.254Z" }, { url = "https://files.pythonhosted.org/packages/48/60/29035719feb91798693023608447283b266b12efc576ed013dd9442364bb/greenlet-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de5a0b09eab81fc6a382791b995b1ccf2b172a9fec934747a7a23d2ff291794", size = 1550668, upload-time = "2025-12-04T15:04:22.439Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/783a23754b691bfa86bd72c3033aa107490deac9b2ef190837b860996c9f/greenlet-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4449a736606bd30f27f8e1ff4678ee193bc47f6ca810d705981cfffd6ce0d8c5", size = 1615483, upload-time = "2025-12-04T14:27:28.083Z" }, @@ -235,6 +240,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -242,6 +248,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140, upload-time = "2025-12-04T14:23:01.282Z" }, { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219, upload-time = "2025-12-04T14:50:08.309Z" }, { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211, upload-time = "2025-12-04T14:57:43.968Z" }, + { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311, upload-time = "2025-12-04T15:07:14.697Z" }, { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833, upload-time = "2025-12-04T14:26:03.669Z" }, { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256, upload-time = "2025-12-04T15:04:25.276Z" }, { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483, upload-time = "2025-12-04T14:27:30.804Z" }, @@ -249,6 +256,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/7c/f0a6d0ede2c7bf092d00bc83ad5bafb7e6ec9b4aab2fbdfa6f134dc73327/greenlet-3.3.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:60c2ef0f578afb3c8d92ea07ad327f9a062547137afe91f38408f08aacab667f", size = 275671, upload-time = "2025-12-04T14:23:05.267Z" }, { url = "https://files.pythonhosted.org/packages/44/06/dac639ae1a50f5969d82d2e3dd9767d30d6dbdbab0e1a54010c8fe90263c/greenlet-3.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5d554d0712ba1de0a6c94c640f7aeba3f85b3a6e1f2899c11c2c0428da9365", size = 646360, upload-time = "2025-12-04T14:50:10.026Z" }, { url = "https://files.pythonhosted.org/packages/e0/94/0fb76fe6c5369fba9bf98529ada6f4c3a1adf19e406a47332245ef0eb357/greenlet-3.3.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3a898b1e9c5f7307ebbde4102908e6cbfcb9ea16284a3abe15cab996bee8b9b3", size = 658160, upload-time = "2025-12-04T14:57:45.41Z" }, + { url = "https://files.pythonhosted.org/packages/93/79/d2c70cae6e823fac36c3bbc9077962105052b7ef81db2f01ec3b9bf17e2b/greenlet-3.3.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dcd2bdbd444ff340e8d6bdf54d2f206ccddbb3ccfdcd3c25bf4afaa7b8f0cf45", size = 671388, upload-time = "2025-12-04T15:07:15.789Z" }, { url = "https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955", size = 660166, upload-time = "2025-12-04T14:26:05.099Z" }, { url = "https://files.pythonhosted.org/packages/4b/d2/91465d39164eaa0085177f61983d80ffe746c5a1860f009811d498e7259c/greenlet-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac0549373982b36d5fd5d30beb8a7a33ee541ff98d2b502714a09f1169f31b55", size = 1615193, upload-time = "2025-12-04T15:04:27.041Z" }, { url = "https://files.pythonhosted.org/packages/42/1b/83d110a37044b92423084d52d5d5a3b3a73cafb51b547e6d7366ff62eff1/greenlet-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d198d2d977460358c3b3a4dc844f875d1adb33817f0613f663a656f463764ccc", size = 1683653, upload-time = "2025-12-04T14:27:32.366Z" }, @@ -256,6 +264,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/66/bd6317bc5932accf351fc19f177ffba53712a202f9df10587da8df257c7e/greenlet-3.3.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d6ed6f85fae6cdfdb9ce04c9bf7a08d666cfcfb914e7d006f44f840b46741931", size = 282638, upload-time = "2025-12-04T14:25:20.941Z" }, { url = "https://files.pythonhosted.org/packages/30/cf/cc81cb030b40e738d6e69502ccbd0dd1bced0588e958f9e757945de24404/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9125050fcf24554e69c4cacb086b87b3b55dc395a8b3ebe6487b045b2614388", size = 651145, upload-time = "2025-12-04T14:50:11.039Z" }, { url = "https://files.pythonhosted.org/packages/9c/ea/1020037b5ecfe95ca7df8d8549959baceb8186031da83d5ecceff8b08cd2/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87e63ccfa13c0a0f6234ed0add552af24cc67dd886731f2261e46e241608bee3", size = 654236, upload-time = "2025-12-04T14:57:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/69/cc/1e4bae2e45ca2fa55299f4e85854606a78ecc37fead20d69322f96000504/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2662433acbca297c9153a4023fe2161c8dcfdcc91f10433171cf7e7d94ba2221", size = 662506, upload-time = "2025-12-04T15:07:16.906Z" }, { url = "https://files.pythonhosted.org/packages/57/b9/f8025d71a6085c441a7eaff0fd928bbb275a6633773667023d19179fe815/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6e9b9c1527a78520357de498b0e709fb9e2f49c3a513afd5a249007261911b", size = 653783, upload-time = "2025-12-04T14:26:06.225Z" }, { url = "https://files.pythonhosted.org/packages/f6/c7/876a8c7a7485d5d6b5c6821201d542ef28be645aa024cfe1145b35c120c1/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:286d093f95ec98fdd92fcb955003b8a3d054b4e2cab3e2707a5039e7b50520fd", size = 1614857, upload-time = "2025-12-04T15:04:28.484Z" }, { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, @@ -283,6 +292,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11", marker = "python_full_version < '3.12' or sys_platform != 'emscripten'" }, + { name = "truststore", marker = "python_full_version < '3.12' or sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/8c/e925b1c92018abb3a1863ce1549d76d2381e334d21d65d4ac8f65dabd78a/httpcore2-2.13.0.tar.gz", hash = "sha256:2adc8be4fb285fbcd6d894298db3b52c177e74b6674eda3a76bd36be3292a3db", size = 67740, upload-time = "2026-09-14T14:18:04.717Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/0d/117a771a2bb91df334b66bf4da14cd02f21aefbcfe53180f336ce55e8f90/httpcore2-2.13.0-py3-none-any.whl", hash = "sha256:35ae5be347aa40467b4a5dc032ac67ebb6d27189fc97e8cebcf99616f6a1bb9e", size = 83162, upload-time = "2026-09-14T14:18:02.529Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -298,6 +320,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/a0/e9deef4654132857b5a5dbe4eddd0ac59c2814500e11f2f5044cd81103ee/httpx2-2.13.0.tar.gz", hash = "sha256:81bd07dc67a3701729ef1f777a3c00c915d4539604fdb5afd327f8682f6b7b44", size = 100290, upload-time = "2026-09-14T14:18:05.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/d1/a0c72b0e006df654709fbc366cc5bcb53e5aee13e1e3395152c6dd293376/httpx2-2.13.0-py3-none-any.whl", hash = "sha256:fc12720cedf72faa26cca6b4ca394e05c894e7d7933fc45cafe767960804e49a", size = 95565, upload-time = "2026-09-14T14:18:03.553Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "identify" version = "2.6.15" @@ -511,6 +559,9 @@ dependencies = [ ] [package.optional-dependencies] +jev = [ + { name = "typesafe-sdk" }, +] postgres = [ { name = "pgvector" }, { name = "sqlalchemy", extra = ["postgresql-psycopgbinary"] }, @@ -551,8 +602,9 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.4" }, { name = "sqlalchemy", extras = ["postgresql-psycopgbinary"], marker = "extra == 'postgres'", specifier = ">=2.0.36" }, { name = "sqlmodel", specifier = ">=0.0.27" }, + { name = "typesafe-sdk", marker = "extra == 'jev'", specifier = ">=0.7.0,<0.8" }, ] -provides-extras = ["postgres"] +provides-extras = ["jev", "postgres"] [package.metadata.requires-dev] dev = [ @@ -1269,6 +1321,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/92/c35e036151fe53822893979f8a13e6f235ae8191f4164a79ae60a95d66aa/sqlmodel-0.0.27-py3-none-any.whl", hash = "sha256:667fe10aa8ff5438134668228dc7d7a08306f4c5c4c7e6ad3ad68defa0e7aa49", size = 29131, upload-time = "2025-10-08T16:39:10.917Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1335,6 +1396,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "types-defusedxml" version = "0.7.0.20250822" @@ -1344,6 +1414,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/73/8a36998cee9d7c9702ed64a31f0866c7f192ecffc22771d44dbcc7878f18/types_defusedxml-0.7.0.20250822-py3-none-any.whl", hash = "sha256:5ee219f8a9a79c184773599ad216123aedc62a969533ec36737ec98601f20dcf", size = 13430, upload-time = "2025-08-22T03:02:58.466Z" }, ] +[[package]] +name = "typesafe-sdk" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "tenacity" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e2/ac317772d4d5cfabf3cecdeacd83cc838523e240bb4e1cf256837e6005bb/typesafe_sdk-0.7.0.tar.gz", hash = "sha256:930d42fd73cfed6f25bccae488ce0f30f6043eaedd8682ef59734e79ab94a876", size = 22235, upload-time = "2026-09-18T09:12:30.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/2a/16f4163e6d8827ef80b2ed7a400c80688e1ebf571c10f143f792bdafc654/typesafe_sdk-0.7.0-py3-none-any.whl", hash = "sha256:c6d2257c4b04b8d4d81cff2b55489b4188795fd44fb40b9cef26ff243b2d1080", size = 35352, upload-time = "2026-09-18T09:12:29.625Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 569bd15d1ed1bb9af4b50b5f4fcdb6616b51c562 Mon Sep 17 00:00:00 2001 From: Peekaboo <318562198+jaunatisq@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:56:49 +0800 Subject: [PATCH 3/3] fix: keep Jev typing optional --- src/memu/integrations/jev.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/memu/integrations/jev.py b/src/memu/integrations/jev.py index b1d955f0..ab1256ab 100644 --- a/src/memu/integrations/jev.py +++ b/src/memu/integrations/jev.py @@ -13,15 +13,12 @@ import time from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, Protocol, Self +from typing import Any, Literal, Protocol, Self from pydantic import BaseModel, ConfigDict, Field, field_validator from memu.agentic_backend import AgenticMemoryBackend -if TYPE_CHECKING: - from typesafe_sdk import JSONContent, NoulModel - CandidateKind = Literal["segment", "resource"] ErrorPolicy = Literal["fallback", "raise"] @@ -125,13 +122,13 @@ async def evaluate(self, query: str, candidates: list[JevCandidate]) -> JevEvalu message = "Jev evaluation requires at least one candidate" raise ValueError(message) - state: JSONContent = { + state: dict[str, object] = { "query": query, "candidates": { candidate.key: {"kind": candidate.kind, "content": candidate.text} for candidate in candidates }, } - questions: dict[str, NoulModel] = { + questions: dict[str, dict[str, object]] = { candidate.key: { "type": "noul", "instructions": (