Try it (no keys) ·
Architecture ·
From 0bfd90b53199d789845862fdaef54ff83bd0ef44 Mon Sep 17 00:00:00 2001
From: Rachit
Date: Mon, 2 Nov 2026 11:44:59 +0100
Subject: [PATCH 03/30] feat(api): where am I? match one photo against the room
scans (ORB + PnP, no model)
The pipeline now writes a place index per scan (ORB features of every
keyframe lifted to 3D with the phone's LiDAR depth and ARKit pose), and
POST /homes/{id}/locate matches one camera photo against it: the room it
was taken in, the camera's pose in the room map's world, and what to point
at. It runs beside ARKit's relocalisation, never instead of it.
On the real living-room scans: held-out keyframes 1.2 cm median error;
photos from other sessions fix 59% of the time and 89% of fixes agree
(the phone applies a pose only when two fixes agree); 22-35 ms per match.
Co-Authored-By: Claude Opus 5.5
---
Makefile | 6 +-
services/api/pyproject.toml | 3 +-
services/api/resources/openapi.yaml | 2 +
services/api/resources/paths/homes.yaml | 35 ++++
.../api/resources/schemas/LocateRequest.yaml | 12 ++
services/api/resources/schemas/PlaceFix.yaml | 27 +++
.../recall_api/adapters/inbound/http/homes.py | 40 +++-
.../adapters/inbound/http/models.py | 45 +++++
.../outbound/vision/orb_place_matcher.py | 126 +++++++++++++
.../recall_api/application/locate_by_photo.py | 128 +++++++++++++
.../api/src/recall_api/application/ports.py | 31 ++++
.../api/src/recall_api/infra/container.py | 7 +
services/api/src/recall_api/infra/guard.py | 5 +-
.../api/tests/unit/test_locate_by_photo.py | 173 ++++++++++++++++++
services/api/uv.lock | 21 +++
.../recall_pipeline/adapters/place_index.py | 102 +++++++++++
.../src/recall_pipeline/adapters/run.py | 7 +
.../src/recall_pipeline/domain/place_index.py | 52 ++++++
.../pipeline/tests/unit/test_place_index.py | 24 +++
19 files changed, 841 insertions(+), 5 deletions(-)
create mode 100644 services/api/resources/schemas/LocateRequest.yaml
create mode 100644 services/api/resources/schemas/PlaceFix.yaml
create mode 100644 services/api/src/recall_api/adapters/outbound/vision/orb_place_matcher.py
create mode 100644 services/api/src/recall_api/application/locate_by_photo.py
create mode 100644 services/api/tests/unit/test_locate_by_photo.py
create mode 100644 services/pipeline/src/recall_pipeline/adapters/place_index.py
create mode 100644 services/pipeline/src/recall_pipeline/domain/place_index.py
create mode 100644 services/pipeline/tests/unit/test_place_index.py
diff --git a/Makefile b/Makefile
index 8d09d48..0d77d43 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ BIND ?= 0.0.0.0
WEB := apps/web
REDOCLY := npx -y @redocly/cli@2.72.1
-.PHONY: help setup dev worker robot-export regression viewer public-scan live demo record probes diagrams pair rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval
+.PHONY: help setup dev worker robot-export place-index regression viewer public-scan live demo record probes diagrams pair rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval
help:
@grep -E '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | awk -F':.*?## ' '{printf " %-18s %s\n", $$1, $$2}'
@@ -30,6 +30,10 @@ ROBOT_HOME = $(or $(HOME_ID),$(if $(filter command line,$(origin HOME)),$(HOME),
robot-export: ## One room's memory for a robot (Nav2 map, things, furniture): make robot-export [HOME=flat] [ROOM=...] -> exports/robot//
cd $(API) && uv run python -m recall_api.infra.robot_export --home "$(ROBOT_HOME)" $(if $(ROOM),--room "$(ROOM)",) --out "$(CURDIR)/exports/robot/$(ROBOT_HOME)"
+place-index: ## Build the photo matcher's index (ORB + LiDAR depth, no model) for every processed scan on this Mac
+ cd $(PIPE) && for e in $(CURDIR)/data/blobs/scans/*/episode; do \
+ [ -f "$$e/result.json" ] && uv run python -m recall_pipeline.adapters.place_index "$$e"; done; true
+
regression: ## Re-run every processed scan now (no posting; cached SAM answers) and compare with the last run
cd $(API) && uv run python -c "from recall_api.infra import jobs, regression; print('\n'.join(regression.run(jobs.settings, jobs.run_pipeline)))"
diff --git a/services/api/pyproject.toml b/services/api/pyproject.toml
index c4ef66c..c421a65 100644
--- a/services/api/pyproject.toml
+++ b/services/api/pyproject.toml
@@ -23,6 +23,7 @@ dependencies = [
"huey>=2.5",
"a2a-sdk[http-server]>=1.3.0",
"pillow>=13.0.0",
+ "opencv-python-headless>=4.10,<5",
]
[project.scripts]
@@ -81,4 +82,4 @@ layers = [
name = "Domain and application stay free of frameworks and I/O"
type = "forbidden"
source_modules = ["recall_api.domain", "recall_api.application", "recall_api.util"]
-forbidden_modules = ["fastapi", "sqlalchemy", "httpx", "openai", "torch", "boto3", "starlette", "uvicorn"]
+forbidden_modules = ["fastapi", "sqlalchemy", "httpx", "openai", "torch", "boto3", "starlette", "uvicorn", "cv2"]
diff --git a/services/api/resources/openapi.yaml b/services/api/resources/openapi.yaml
index cc1544c..fe1f864 100644
--- a/services/api/resources/openapi.yaml
+++ b/services/api/resources/openapi.yaml
@@ -76,6 +76,8 @@ paths:
$ref: paths/homes.yaml#/room_map
/homes/{home_id}/ar-labels:
$ref: paths/homes.yaml#/ar_labels
+ /homes/{home_id}/locate:
+ $ref: paths/homes.yaml#/locate
/homes/{home_id}/robot-package:
$ref: paths/homes.yaml#/robot_package
/homes/{home_id}/robot-export:
diff --git a/services/api/resources/paths/homes.yaml b/services/api/resources/paths/homes.yaml
index 0df4a15..2730a9b 100644
--- a/services/api/resources/paths/homes.yaml
+++ b/services/api/resources/paths/homes.yaml
@@ -143,3 +143,38 @@ ar_labels:
schema: {$ref: ../schemas/ArLabels.yaml}
"404":
$ref: ../schemas/Responses.yaml#/NotFound
+locate:
+ post:
+ tags: [homes]
+ operationId: locateByPhoto
+ summary: Where am I? One camera photo matched against the room scans (classic features + PnP, no model)
+ description: >
+ Asked by the phone while ARKit hasn't yet recognised the saved room map, alongside it (never instead of it).
+ Answers the room the photo was taken in when one room clearly wins, the camera's pose in the requested map's
+ world when that room matched well enough, and what to point the phone at. The photo is matched in memory and not
+ stored.
+ parameters:
+ - $ref: ../schemas/Params.yaml#/home_id
+ requestBody:
+ required: true
+ content:
+ multipart/form-data:
+ schema:
+ $ref: ../schemas/LocateRequest.yaml
+ responses:
+ "413":
+ $ref: ../schemas/Responses.yaml#/TooLarge
+ "429":
+ $ref: ../schemas/Responses.yaml#/RateLimited
+ "401":
+ $ref: ../schemas/Responses.yaml#/Unauthorized
+ "422":
+ $ref: ../schemas/Responses.yaml#/Invalid
+ "200":
+ description: What the photo matched
+ content:
+ application/json:
+ schema:
+ $ref: ../schemas/PlaceFix.yaml
+ "404":
+ $ref: ../schemas/Responses.yaml#/NotFound
diff --git a/services/api/resources/schemas/LocateRequest.yaml b/services/api/resources/schemas/LocateRequest.yaml
new file mode 100644
index 0000000..2df5ff0
--- /dev/null
+++ b/services/api/resources/schemas/LocateRequest.yaml
@@ -0,0 +1,12 @@
+type: object
+required: [image, pose, scan_id]
+properties:
+ image: {type: string, format: binary, description: "JPEG of the camera now (any size; intrinsics are for this image)"}
+ pose:
+ type: string
+ description: >
+ JSON: {"pose": [16 numbers, ARKit camera-to-world, column-major, in the phone's session (not yet the map's)],
+ "intrinsics": {"fx", "fy", "cx", "cy"} for the JPEG, "image_size": [w, h]}. Only the intrinsics are used on
+ the Mac; the phone keeps the session pose to turn the answer into its own correction.
+ scan_id: {type: string, maxLength: 64, description: "The scan whose room map the phone runs: the pose is answered in its world"}
+ room: {type: string, maxLength: 60, description: "The room the phone is trying to recognise (for the hint)"}
diff --git a/services/api/resources/schemas/PlaceFix.yaml b/services/api/resources/schemas/PlaceFix.yaml
new file mode 100644
index 0000000..176e71f
--- /dev/null
+++ b/services/api/resources/schemas/PlaceFix.yaml
@@ -0,0 +1,27 @@
+type: object
+required: [room, scan_id, score, runner_up_room, runner_up_score, pose, inliers, reproj_px, hint, ms]
+properties:
+ room:
+ type: [string, "null"]
+ description: The room the photo was taken in, when one room clearly matched (it may not be the requested map's room)
+ scan_id: {type: [string, "null"], description: "The scan whose world `pose` is in"}
+ score: {type: integer, minimum: 0, description: "RANSAC inliers of the best room"}
+ runner_up_room: {type: [string, "null"]}
+ runner_up_score: {type: integer, minimum: 0}
+ pose:
+ type: [array, "null"]
+ items: {type: number}
+ minItems: 16
+ maxItems: 16
+ description: >
+ The camera's pose (ARKit camera-to-world, column-major) in the requested map's world; null unless that room
+ matched with at least 30 inliers and at most 4 px mean reprojection error. The phone applies it only when two
+ fixes in a row agree.
+ inliers: {type: integer, minimum: 0, description: "RANSAC inliers in the requested map's world"}
+ reproj_px: {type: [number, "null"], description: "Mean reprojection error of those inliers, pixels"}
+ hint:
+ type: array
+ items: {type: string}
+ maxItems: 3
+ description: What to point the phone at in that room, one-of-a-kind furniture first
+ ms: {type: integer, minimum: 0, description: "Time the Mac took"}
diff --git a/services/api/src/recall_api/adapters/inbound/http/homes.py b/services/api/src/recall_api/adapters/inbound/http/homes.py
index b5342a2..45c0b57 100644
--- a/services/api/src/recall_api/adapters/inbound/http/homes.py
+++ b/services/api/src/recall_api/adapters/inbound/http/homes.py
@@ -2,17 +2,32 @@
from __future__ import annotations
-from fastapi import APIRouter, Depends, HTTPException, Response
+import json
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form, HTTPException, Response, UploadFile
+from fastapi.concurrency import run_in_threadpool
from recall_api.application.ar_labels import NoRoomScan
from recall_api.application.correct_object import CorrectionError, CorrectionInput
from recall_api.application.export_for_robot import RobotExportError
from recall_api.application.list_things import NoSuchHome, ThingView
+from recall_api.application.locate_by_photo import LocateError
from recall_api.application.plan_compass_route import CompassRouteError
+from recall_api.application.ports import PlaceQuery
from recall_api.domain.memory import Home, Trust
from .deps import container, require_device_key
-from .models import ArLabels, CompassRoute, ObjectCorrection, ObjectLocation, RobotExport, RoomModel, TrustUpdate
+from .models import (
+ ArLabels,
+ CompassRoute,
+ ObjectCorrection,
+ ObjectLocation,
+ PlaceFix,
+ RobotExport,
+ RoomModel,
+ TrustUpdate,
+)
from .models import Trust as TrustLevel
router = APIRouter(tags=["homes"], dependencies=[Depends(require_device_key)])
@@ -137,3 +152,24 @@ def get_ar_labels(home_id: str, room: str, c=Depends(container)) -> ArLabels:
raise HTTPException(404, {"error": "not_found", "detail": f"No processed scan of {room} yet."}) from e
return ArLabels(scan_id=r.scan_id, room=r.room, floor_y_arkit=r.floor_y_arkit,
labels=[vars(x) | {"position_arkit": list(x.position_arkit)} for x in r.labels]) # fmt: skip
+
+
+@router.post("/homes/{home_id}/locate", response_model=PlaceFix)
+async def locate(
+ home_id: str,
+ image: UploadFile,
+ pose: Annotated[str, Form()],
+ scan_id: Annotated[str, Form(max_length=64)],
+ room: Annotated[str | None, Form(max_length=60)] = None,
+ c=Depends(container),
+) -> PlaceFix:
+ try:
+ k = json.loads(pose)["intrinsics"]
+ query = PlaceQuery(await image.read(), float(k["fx"]), float(k["fy"]), float(k["cx"]), float(k["cy"]))
+ except (KeyError, ValueError, TypeError) as e:
+ raise HTTPException(422, {"error": "invalid", "detail": "Bad photo: pose must carry the intrinsics."}) from e
+ try:
+ f = await run_in_threadpool(c.locate_by_photo, home_id, scan_id, query, room) # CPU work off the event loop
+ except LocateError as e:
+ raise HTTPException(e.status, {"error": e.code, "detail": str(e)}) from e
+ return PlaceFix.model_validate(vars(f) | {"pose": list(f.pose) if f.pose else None})
diff --git a/services/api/src/recall_api/adapters/inbound/http/models.py b/services/api/src/recall_api/adapters/inbound/http/models.py
index 9ccd763..b9c2016 100644
--- a/services/api/src/recall_api/adapters/inbound/http/models.py
+++ b/services/api/src/recall_api/adapters/inbound/http/models.py
@@ -636,6 +636,51 @@ class ArLabels(BaseModel):
labels: list[Label]
+class LocateRequest(BaseModel):
+ image: Annotated[bytes, Field(description="JPEG of the camera now (any size; intrinsics are for this image)")]
+ pose: Annotated[
+ str,
+ Field(
+ description='JSON: {"pose": [16 numbers, ARKit camera-to-world, column-major, in the phone\'s session (not yet the map\'s)], "intrinsics": {"fx", "fy", "cx", "cy"} for the JPEG, "image_size": [w, h]}. Only the intrinsics are used on the Mac; the phone keeps the session pose to turn the answer into its own correction.\n'
+ ),
+ ]
+ scan_id: Annotated[
+ str,
+ Field(description="The scan whose room map the phone runs: the pose is answered in its world", max_length=64),
+ ]
+ room: Annotated[
+ str | None, Field(description="The room the phone is trying to recognise (for the hint)", max_length=60)
+ ] = None
+
+
+class PlaceFix(BaseModel):
+ room: Annotated[
+ str | None,
+ Field(
+ description="The room the photo was taken in, when one room clearly matched (it may not be the requested map's room)"
+ ),
+ ]
+ scan_id: Annotated[str | None, Field(description="The scan whose world `pose` is in")]
+ score: Annotated[int, Field(description="RANSAC inliers of the best room", ge=0)]
+ runner_up_room: str | None
+ runner_up_score: Annotated[int, Field(ge=0)]
+ pose: Annotated[
+ list[float] | None,
+ Field(
+ description="The camera's pose (ARKit camera-to-world, column-major) in the requested map's world; null unless that room matched with at least 30 inliers and at most 4 px mean reprojection error. The phone applies it only when two fixes in a row agree.\n",
+ max_length=16,
+ min_length=16,
+ ),
+ ]
+ inliers: Annotated[int, Field(description="RANSAC inliers in the requested map's world", ge=0)]
+ reproj_px: Annotated[float | None, Field(description="Mean reprojection error of those inliers, pixels")]
+ hint: Annotated[
+ list[str],
+ Field(description="What to point the phone at in that room, one-of-a-kind furniture first", max_length=3),
+ ]
+ ms: Annotated[int, Field(description="Time the Mac took", ge=0)]
+
+
class Leg(BaseModel):
bearing_deg: Annotated[float, Field(description="Magnetic bearing, clockwise from north", ge=0.0, le=360.0)]
metres: Annotated[float, Field(ge=0.0)]
diff --git a/services/api/src/recall_api/adapters/outbound/vision/orb_place_matcher.py b/services/api/src/recall_api/adapters/outbound/vision/orb_place_matcher.py
new file mode 100644
index 0000000..ad56b3c
--- /dev/null
+++ b/services/api/src/recall_api/adapters/outbound/vision/orb_place_matcher.py
@@ -0,0 +1,126 @@
+"""PlaceMatcher on classic OpenCV: one photo from the phone against a scan's place index (place_index.npz, written by
+the pipeline: ORB features of every keyframe lifted to 3D with LiDAR depth and the ARKit pose).
+
+1. ORB features of the photo (1000, on a 640-pixel copy: the same settings the index was built with).
+2. Retrieval: every descriptor's nearest neighbour in the whole index votes for its keyframe; the 3 keyframes with the
+ most close matches are kept.
+3. Matching against those keyframes with Lowe's ratio test gives 2D–3D pairs; `solvePnPRansac` + LM refinement gives
+ the camera's pose in the scan's ARKit world, and the RANSAC inliers are the score.
+No model anywhere; the photo is decoded in memory and never stored.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from functools import lru_cache
+
+import cv2
+import numpy as np
+
+from recall_api.application.ports import BlobStore, PlaceMatch, PlaceQuery
+
+FEATURES = 1000
+LONG_EDGE = 640
+CLOSE = 48 # Hamming distance that counts as a vote in retrieval (of 256 bits)
+RATIO = 0.8
+TOP_FRAMES = 3
+REPROJ_PX = 4.0
+_FLIP = np.diag([1.0, -1.0, -1.0, 1.0]) # OpenCV camera (+y down, +z forward) <-> ARKit camera (+y up, −z forward)
+
+
+@dataclass(frozen=True)
+class _Index:
+ desc: np.ndarray # (N, 32) uint8
+ xyz: np.ndarray # (N, 3) float32, the scan's ARKit world
+ frame: np.ndarray # (N,) keyframe row of each point
+ starts: np.ndarray # first point of each keyframe (points are stored keyframe by keyframe)
+ frames: int
+
+
+@lru_cache(maxsize=16)
+def _load(path: str, mtime: float) -> _Index: # mtime in the key: a rebuilt index is read again
+ with np.load(path) as z:
+ frame = z["frame"].astype(np.int64)
+ n = int(len(z["frame_ids"]))
+ return _Index(z["desc"], z["xyz"].astype(np.float64), frame, np.searchsorted(frame, np.arange(n + 1)), n)
+
+
+class OrbPlaceMatcher:
+ def __init__(self, blobs: BlobStore) -> None:
+ self.blobs = blobs
+ self.orb = cv2.ORB_create(nfeatures=FEATURES, scaleFactor=1.2, nlevels=8, fastThreshold=12)
+ self.bf = cv2.BFMatcher(cv2.NORM_HAMMING)
+
+ def _index(self, key: str) -> _Index | None:
+ path = self.blobs.local_path(key)
+ if not path or not os.path.exists(path):
+ return None
+ return _load(path, os.path.getmtime(path))
+
+ def _features(self, jpeg: bytes) -> tuple[np.ndarray, np.ndarray]:
+ gray = cv2.imdecode(np.frombuffer(jpeg, np.uint8), cv2.IMREAD_GRAYSCALE)
+ if gray is None:
+ return np.zeros((0, 2)), np.zeros((0, 32), np.uint8)
+ h, w = gray.shape
+ s = min(LONG_EDGE / max(h, w), 1.0)
+ small = cv2.resize(gray, (round(w * s), round(h * s)), interpolation=cv2.INTER_AREA) if s < 1 else gray
+ kps, desc = self.orb.detectAndCompute(small, None)
+ if desc is None or not kps:
+ return np.zeros((0, 2)), np.zeros((0, 32), np.uint8)
+ return np.array([k.pt for k in kps], dtype=np.float64) / s, desc
+
+ def match(self, query: PlaceQuery, candidates: dict[str, str], budget_s: float = 0.4) -> list[PlaceMatch]:
+ """One result per candidate scan (scan id -> index key) that has an index, best first; scans not reached
+ within the time budget are left out."""
+ t0 = time.perf_counter()
+ uv, desc = self._features(query.jpeg)
+ if len(desc) < 30:
+ return []
+ K = np.array([[query.fx, 0, query.cx], [0, query.fy, query.cy], [0, 0, 1.0]])
+ out = []
+ for scan_id, key in candidates.items():
+ if time.perf_counter() - t0 > budget_s:
+ break
+ idx = self._index(key)
+ if idx is None:
+ continue
+ out.append(self._one(scan_id, idx, uv, desc, K))
+ return sorted(out, key=lambda m: -m.inliers)
+
+ def _one(self, scan_id: str, idx: _Index, uv: np.ndarray, desc: np.ndarray, K: np.ndarray) -> PlaceMatch:
+ nearest = self.bf.match(desc, idx.desc)
+ votes = np.bincount([idx.frame[m.trainIdx] for m in nearest if m.distance <= CLOSE], minlength=idx.frames)
+ top = [int(f) for f in np.argsort(-votes)[:TOP_FRAMES] if votes[f] > 0]
+ pairs: dict[int, tuple[float, int]] = {} # query keypoint -> (distance, index point)
+ for f in top:
+ a, b = int(idx.starts[f]), int(idx.starts[f + 1])
+ if b - a < 2:
+ continue
+ for m in self.bf.knnMatch(desc, idx.desc[a:b], k=2):
+ if len(m) == 2 and m[0].distance < RATIO * m[1].distance:
+ q, d = m[0].queryIdx, m[0].distance
+ if q not in pairs or d < pairs[q][0]:
+ pairs[q] = (d, a + m[0].trainIdx)
+ keyframe = top[0] if top else None
+ if len(pairs) < 12:
+ return PlaceMatch(scan_id, len(pairs) // 4, None, None, keyframe)
+ q_idx = np.array(list(pairs))
+ obj = idx.xyz[[pairs[q][1] for q in q_idx]]
+ img = uv[q_idx]
+ ok, rvec, tvec, inl = cv2.solvePnPRansac(
+ obj, img, K, None, iterationsCount=300, reprojectionError=REPROJ_PX, confidence=0.995,
+ flags=cv2.SOLVEPNP_EPNP,
+ ) # fmt: skip
+ if not ok or inl is None or len(inl) < 6:
+ return PlaceMatch(scan_id, 0, None, None, keyframe)
+ inl = inl[:, 0]
+ rvec, tvec = cv2.solvePnPRefineLM(obj[inl], img[inl], K, None, rvec, tvec)
+ proj, _ = cv2.projectPoints(obj[inl], rvec, tvec, K, None)
+ err = float(np.mean(np.linalg.norm(proj[:, 0] - img[inl], axis=1)))
+ R, _ = cv2.Rodrigues(rvec)
+ world_to_cam = np.eye(4)
+ world_to_cam[:3, :3], world_to_cam[:3, 3] = R, tvec[:, 0]
+ cam2world = np.linalg.inv(world_to_cam) @ _FLIP # OpenCV camera -> ARKit camera, same world
+ return PlaceMatch(scan_id, int(len(inl)), tuple(float(v) for v in cam2world.T.reshape(-1)), err, keyframe)
diff --git a/services/api/src/recall_api/application/locate_by_photo.py b/services/api/src/recall_api/application/locate_by_photo.py
new file mode 100644
index 0000000..3d768ae
--- /dev/null
+++ b/services/api/src/recall_api/application/locate_by_photo.py
@@ -0,0 +1,128 @@
+"""Where am I? One photo from the phone, matched against the scans' place indexes (classic features + PnP, no model).
+
+It runs alongside ARKit's own relocalisation, never instead of it: the phone asks while ARKit hasn't recognised the
+room yet and uses whichever answers first. It answers:
+- the room the photo was taken in, when one room clearly wins (it may not be the room the thing is in);
+- the camera's pose in the world of the room map the phone runs, when that room matched well enough;
+- what to point at: the room's biggest one-of-a-kind furniture, for the spoken coaching.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import Counter
+from dataclasses import dataclass
+
+from recall_api.application.ports import PlaceMatch, PlaceMatcher, PlaceQuery
+from recall_api.domain.memory import Scan, ScanStatus
+from recall_api.domain.ports import HomeRepository, ScanRepository
+
+ROOM_MIN_INLIERS = 30 # a room is named from this many RANSAC inliers…
+ROOM_MARGIN = 1.5 # …and this many times the next room's
+POSE_MIN_INLIERS = 30 # on real scans: 59% of photos from another session fix, 89% of those right (evals/results.md)
+POSE_MAX_REPROJ_PX = 4.0
+HINTS = 3
+
+
+class LocateError(Exception):
+ def __init__(self, message: str, code: str = "not_found", status: int = 404) -> None:
+ super().__init__(message)
+ self.code, self.status = code, status
+
+
+@dataclass(frozen=True)
+class PlaceFix:
+ room: str | None # None: no room clearly matched
+ scan_id: str | None # the scan whose world `pose` is in (the requested map's)
+ score: int
+ runner_up_room: str | None
+ runner_up_score: int
+ pose: tuple[float, ...] | None # ARKit camera-to-world, column-major, in the requested map's world
+ inliers: int
+ reproj_px: float | None
+ hint: list[str]
+ ms: int
+
+
+def index_key(scan: Scan) -> str:
+ return f"{scan.folder}/episode/place_index.npz"
+
+
+class LocateByPhoto:
+ def __init__(self, homes: HomeRepository, scans: ScanRepository, matcher: PlaceMatcher) -> None:
+ self.homes, self.scans, self.matcher = homes, scans, matcher
+
+ def __call__(self, home_id: str, scan_id: str, query: PlaceQuery, room: str | None = None) -> PlaceFix:
+ t0 = time.perf_counter()
+ home = self.homes.get(home_id)
+ current = self.scans.get(scan_id)
+ if home is None or current is None or current.home_id != home_id:
+ raise LocateError("No such home or scan.")
+ processed = [s for s in self.scans.list_for_home(home_id) if s.status == ScanStatus.PROCESSED]
+ # the map's own world: its scan, the scan it was recognised in, and every rescan recognised in either
+ root = current.aligned_to or current.id
+ family = {s.id: s for s in processed if root in (s.id, s.aligned_to)} | {current.id: current}
+ others: dict[str, Scan] = {}
+ for s in sorted(processed, key=lambda s: s.created_at, reverse=True): # each other room's newest scan
+ if s.room != current.room and s.room not in {o.room for o in others.values()}:
+ others[s.id] = s
+ candidates = {sid: index_key(s) for sid, s in (family | others).items()}
+ matches = self.matcher.match(query, candidates)
+ rooms = {**{sid: s.room for sid, s in family.items()}, **{sid: s.room for sid, s in others.items()}}
+
+ best_by_room: dict[str, PlaceMatch] = {}
+ for m in matches:
+ r = rooms[m.scan_id]
+ if r not in best_by_room or m.inliers > best_by_room[r].inliers:
+ best_by_room[r] = m
+ ranked = sorted(best_by_room.items(), key=lambda kv: -kv[1].inliers)
+ top_room, top = ranked[0] if ranked else (None, None)
+ second_room, second = ranked[1] if len(ranked) > 1 else (None, None)
+ score = top.inliers if top else 0
+ clear = score >= ROOM_MIN_INLIERS and score >= ROOM_MARGIN * (second.inliers if second else 0)
+ named = top_room if clear else None
+
+ # a pose only in the map's own world, and only when its room clearly won (two rooms alike: no pose)
+ mine = max((m for m in matches if m.scan_id in family), key=lambda m: m.inliers, default=None)
+ pose = None
+ if (
+ mine is not None
+ and mine.pose is not None
+ and mine.inliers >= POSE_MIN_INLIERS
+ and (mine.reproj_px or 99) <= POSE_MAX_REPROJ_PX
+ and named == current.room
+ ):
+ pose = mine.pose
+ hint_room = named or room or current.room
+ return PlaceFix(
+ room=named,
+ scan_id=current.id,
+ score=score,
+ runner_up_room=second_room,
+ runner_up_score=second.inliers if second else 0,
+ pose=pose,
+ inliers=mine.inliers if mine else 0,
+ reproj_px=mine.reproj_px if mine and pose else None,
+ hint=hints([lm for lm in home.landmarks if lm.room in (hint_room, None)]),
+ ms=round((time.perf_counter() - t0) * 1000),
+ )
+
+
+def hints(landmarks: list) -> list[str]:
+ """What to point the phone at: one-of-a-kind furniture first (a TV, not one of six chairs), biggest first."""
+ counts = Counter(lm.label for lm in landmarks)
+ seen: set[str] = set()
+ out = []
+
+ def size(lm) -> float:
+ lo, hi = lm.bbox.lo, lm.bbox.hi
+ return (hi.x - lo.x) * (hi.y - lo.y) * max(hi.z - lo.z, 0.05)
+
+ for lm in sorted(landmarks, key=lambda lm: (counts[lm.label] > 1, -size(lm))):
+ if lm.label in seen or lm.label == "door":
+ continue
+ seen.add(lm.label)
+ out.append(lm.label)
+ if len(out) == HINTS:
+ break
+ return out
diff --git a/services/api/src/recall_api/application/ports.py b/services/api/src/recall_api/application/ports.py
index 7c9dfc0..8da04d3 100644
--- a/services/api/src/recall_api/application/ports.py
+++ b/services/api/src/recall_api/application/ports.py
@@ -191,3 +191,34 @@ class PromptScreener(Protocol):
"""A Meta model (Muse Spark Open or Glimmer) checks an object name from outside before any tool runs."""
def screen(self, text: str) -> Screening: ...
+
+
+@dataclass(frozen=True)
+class PlaceQuery:
+ """One photo from the phone, for the photo matcher: the JPEG and its camera intrinsics (pixels of that JPEG)."""
+
+ jpeg: bytes
+ fx: float
+ fy: float
+ cx: float
+ cy: float
+
+
+@dataclass(frozen=True)
+class PlaceMatch:
+ """How one scan matched the photo: RANSAC inliers (the score) and, when the pose solved, the camera's pose in
+ that scan's ARKit world (camera-to-world, column-major) with its mean reprojection error in pixels."""
+
+ scan_id: str
+ inliers: int
+ pose: tuple[float, ...] | None
+ reproj_px: float | None
+ keyframe: int | None
+
+
+class PlaceMatcher(Protocol):
+ """Classic photo matching (features + PnP, no model) against the scans' place indexes."""
+
+ def match(self, query: PlaceQuery, candidates: dict[str, str], budget_s: float = 0.4) -> list[PlaceMatch]:
+ """`candidates`: scan id -> blob key of its place index. Best first; scans without an index are left out."""
+ ...
diff --git a/services/api/src/recall_api/infra/container.py b/services/api/src/recall_api/infra/container.py
index b8319c7..6a1993f 100644
--- a/services/api/src/recall_api/infra/container.py
+++ b/services/api/src/recall_api/infra/container.py
@@ -40,6 +40,7 @@
from recall_api.application.get_scan import GetScan
from recall_api.application.ingest_scan import IngestScan
from recall_api.application.list_things import ListThings
+from recall_api.application.locate_by_photo import LocateByPhoto
from recall_api.application.mark_scan_failed import MarkScanFailed
from recall_api.application.plan_compass_route import PlanCompassRoute
from recall_api.application.plan_guidance import PlanGuidance
@@ -150,6 +151,12 @@ def correct_object(self) -> CorrectObject:
def guidance(self) -> PlanGuidance:
return PlanGuidance(self.homes, self.scans, self.objects, self.blobs, new_id, trust=self.trust)
+ @cached_property
+ def locate_by_photo(self) -> LocateByPhoto:
+ from recall_api.adapters.outbound.vision.orb_place_matcher import OrbPlaceMatcher
+
+ return LocateByPhoto(self.homes, self.scans, OrbPlaceMatcher(self.blobs))
+
@cached_property
def ar_labels(self) -> ArLabelsInRoom:
return ArLabelsInRoom(self.list_things, self.scans)
diff --git a/services/api/src/recall_api/infra/guard.py b/services/api/src/recall_api/infra/guard.py
index 6411e69..9505ff4 100644
--- a/services/api/src/recall_api/infra/guard.py
+++ b/services/api/src/recall_api/infra/guard.py
@@ -18,15 +18,18 @@
("POST", re.compile(r"^/scans/[^/]+/results$"), 50 * MB), # the pipeline's result
("POST", re.compile(r"^/guidance/sessions/[^/]+/verify$"), 25 * MB), # one photo and its depth
("POST", re.compile(r"^/transcribe$"), 10 * MB), # a spoken question
+ ("POST", re.compile(r"^/homes/[^/]+/locate$"), 4 * MB), # one photo (and its depth) to match
("*", re.compile(r".*"), 1 * MB),
]
-# (method, path pattern, calls allowed per minute per key): the routes that reach a model or start the pipeline
+# (method, path pattern, calls allowed per minute per key): the routes that reach a model, start the pipeline or
+# match photos
RATE_LIMITS: list[tuple[str, re.Pattern[str], int]] = [
("POST", re.compile(r"^/ask$"), 30),
("POST", re.compile(r"^/transcribe$"), 30),
("POST", re.compile(r"^/guidance/sessions/[^/]+/verify$"), 20),
("POST", re.compile(r"^/scans/[^/]+/complete$"), 10),
+ ("POST", re.compile(r"^/homes/[^/]+/locate$"), 90), # no model, but CPU: the phone asks about once a second
]
diff --git a/services/api/tests/unit/test_locate_by_photo.py b/services/api/tests/unit/test_locate_by_photo.py
new file mode 100644
index 0000000..d3c2ab8
--- /dev/null
+++ b/services/api/tests/unit/test_locate_by_photo.py
@@ -0,0 +1,173 @@
+"""Where am I? A photo matched against the scans' place indexes: the room, the pose in the map's world, the hint."""
+
+import json
+
+import cv2
+import numpy as np
+import pytest
+
+from recall_api.adapters.outbound.vision.orb_place_matcher import OrbPlaceMatcher
+from recall_api.application.locate_by_photo import LocateByPhoto, LocateError, hints, index_key
+from recall_api.application.ports import PlaceMatch, PlaceQuery
+from recall_api.domain.memory import BBox, Landmark, Position, ScanStatus
+from tests.unit.test_matching_and_ingest import Repos
+
+POSE = tuple(float(v) for v in np.eye(4).reshape(-1))
+Q = PlaceQuery(b"jpeg", 700.0, 700.0, 512.0, 384.0)
+
+
+class FakeMatcher:
+ def __init__(self, inliers: dict[str, int]):
+ self.inliers, self.asked = inliers, {}
+
+ def match(self, query, candidates, budget_s=0.4):
+ self.asked = candidates
+ out = [PlaceMatch(s, n, POSE if n >= 6 else None, 1.5, 0) for s, n in self.inliers.items() if s in candidates]
+ return sorted(out, key=lambda m: -m.inliers)
+
+
+def _home(inliers):
+ r = Repos()
+ for sid, hour, room, aligned in [
+ ("o1", 9, "Office", None),
+ ("o2", 11, "Office", "o1"),
+ ("b1", 10, "Bedroom", None),
+ ]:
+ s = r.scan(sid, hour, room=room, aligned_to=aligned)
+ s.status = ScanStatus.PROCESSED
+ r.scans.save(s)
+ home = r.homes.get("flat")
+ box = lambda lo, hi: BBox(Position(*lo), Position(*hi)) # noqa: E731
+ home.landmarks = [
+ Landmark("l1", "chair", box((0, 0, 0), (0.5, 0.5, 0.9)), Position(0.2, 0.2, 0.4), room="Office"),
+ Landmark("l2", "chair", box((1, 0, 0), (1.5, 0.5, 0.9)), Position(1.2, 0.2, 0.4), room="Office"),
+ Landmark("l3", "tv", box((2, 0, 1), (3, 0.1, 1.6)), Position(2.5, 0.05, 1.3), room="Office"),
+ Landmark("l4", "bed", box((0, 0, 0), (2, 1.6, 0.5)), Position(1, 0.8, 0.25), room="Bedroom"),
+ ]
+ r.homes.save(home)
+ m = FakeMatcher(inliers)
+ return LocateByPhoto(r.homes, r.scans, m), m, r
+
+
+def test_in_the_maps_room_the_pose_comes_back_and_the_whole_family_is_tried():
+ use, m, r = _home({"o1": 20, "o2": 64, "b1": 9})
+ fix = use("flat", "o2", Q, room="Office")
+ assert (fix.room, fix.pose, fix.inliers, fix.scan_id) == ("Office", POSE, 64, "o2")
+ assert set(m.asked) == {"o1", "o2", "b1"} # the map's family (same world) and the other room's newest scan
+ assert m.asked["b1"] == index_key(r.scans.get("b1")) and m.asked["b1"].endswith("/episode/place_index.npz")
+ assert fix.hint[0] == "tv" and "chair" in fix.hint # one of a kind first
+
+
+def test_in_another_room_it_says_so_and_gives_no_pose():
+ fix, *_ = _home({"o1": 12, "o2": 10, "b1": 80})
+ f = fix("flat", "o1", Q, room="Office")
+ assert (f.room, f.pose, f.runner_up_room) == ("Bedroom", None, "Office")
+ assert f.hint == ["bed"] # what to point at where the phone is
+
+
+def test_two_rooms_alike_name_neither_and_give_no_pose():
+ use, *_ = _home({"o1": 40, "b1": 35})
+ f = use("flat", "o1", Q)
+ assert f.room is None and f.pose is None
+
+
+def test_too_few_inliers_give_no_pose():
+ use, *_ = _home({"o1": 25})
+ assert use("flat", "o1", Q).pose is None
+
+
+def test_a_scan_of_another_home_is_refused():
+ use, *_ = _home({})
+ with pytest.raises(LocateError):
+ use("elsewhere", "o1", Q)
+
+
+def test_hints_skip_doors_and_repeat_kinds():
+ box = BBox(Position(0, 0, 0), Position(1, 1, 1))
+ lms = [Landmark(str(i), lab, box, Position(0, 0, 0)) for i, lab in enumerate(["door", "chair", "chair", "sofa"])]
+ assert hints(lms) == ["sofa", "chair"]
+
+
+# The real matcher on a synthetic scene: a textured wall 2 m in front of the camera.
+FX, CX, CY, W, H, Z = 700.0, 512.0, 384.0, 1024, 768, 2.0
+
+
+def _wall(seed: int) -> np.ndarray:
+ rng = np.random.default_rng(seed)
+ img = np.full((H, W), 128, np.uint8)
+ for _ in range(400):
+ x, y = int(rng.integers(0, W)), int(rng.integers(0, H))
+ if rng.random() < 0.5:
+ cv2.circle(img, (x, y), int(rng.integers(4, 30)), int(rng.integers(0, 255)), -1)
+ else:
+ cv2.rectangle(
+ img, (x, y), (x + int(rng.integers(5, 40)), y + int(rng.integers(5, 40))), int(rng.integers(0, 255)), -1
+ )
+ return img
+
+
+def _index(tmp_path, img) -> str:
+ """The pipeline's index for one keyframe: ORB (same settings) lifted onto the wall, camera at the ARKit origin."""
+ orb = cv2.ORB_create(nfeatures=1000, scaleFactor=1.2, nlevels=8, fastThreshold=12)
+ s = 640 / W
+ kps, desc = orb.detectAndCompute(cv2.resize(img, (640, 480), interpolation=cv2.INTER_AREA), None)
+ uv = np.array([k.pt for k in kps]) / s
+ cam = np.c_[(uv[:, 0] - CX) / FX * Z, (uv[:, 1] - CY) / FX * Z, np.full(len(uv), Z)]
+ world = cam * [1, -1, -1] # OpenCV camera -> ARKit world (identity ARKit pose)
+ path = tmp_path / "place_index.npz"
+ np.savez(path, format=np.array("recall-place-index/1"), desc=desc, xyz=world.astype(np.float32),
+ frame=np.zeros(len(desc), np.int32), frame_ids=np.array([0], np.int32),
+ poses=np.eye(4, dtype=np.float32).reshape(1, 16), image_size=np.array([W, H], np.int32)) # fmt: skip
+ return str(path)
+
+
+class Disk:
+ def local_path(self, key):
+ return key
+
+
+def _jpeg(img) -> bytes:
+ return cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, 90])[1].tobytes()
+
+
+def test_the_matcher_finds_how_far_the_camera_moved(tmp_path):
+ wall = _wall(1)
+ key = _index(tmp_path, wall)
+ # the camera 10 cm to its right: the wall moves 35 px to the left in the photo
+ dx = FX * 0.10 / Z
+ moved = cv2.warpAffine(wall, np.float32([[1, 0, -dx], [0, 1, 0]]), (W, H), borderValue=128)
+ m = OrbPlaceMatcher(Disk()).match(PlaceQuery(_jpeg(moved), FX, FX, CX, CY), {"s": key})[0]
+ assert m.inliers >= 30 and m.reproj_px < 2
+ T = np.array(m.pose).reshape(4, 4).T # column-major -> matrix; ARKit camera-to-world
+ assert np.allclose(T[:3, 3], [0.10, 0, 0], atol=0.02) # ARKit +x is the camera's right
+ assert np.allclose(T[:3, :3], np.eye(3), atol=0.02)
+
+
+def test_a_photo_of_somewhere_else_gives_no_pose(tmp_path):
+ key = _index(tmp_path, _wall(1))
+ m = OrbPlaceMatcher(Disk()).match(PlaceQuery(_jpeg(_wall(2)), FX, FX, CX, CY), {"s": key})
+ assert not m or m[0].pose is None or m[0].inliers < 30
+
+
+def test_the_route_maps_the_form_and_refuses_a_bad_pose(tmp_path):
+ from fastapi.testclient import TestClient
+
+ from recall_api.config.settings import Settings
+ from recall_api.infra.app import create_app
+
+ use, m, _ = _home({"o1": 50})
+ app = create_app(Settings(RECALL_DATA_DIR=tmp_path, RECALL_DEVICE_KEY="k", RECALL_TRACES="off"))
+ app.state.container.__dict__["locate_by_photo"] = use
+ client = TestClient(app)
+ pose = {"pose": list(POSE), "intrinsics": {"fx": 700, "fy": 700, "cx": 512, "cy": 384}, "image_size": [1024, 768]}
+ files = {"image": ("a.jpg", b"jpeg", "image/jpeg")}
+ auth = {"Authorization": "Bearer k"}
+ r = client.post("/homes/flat/locate", files=files, data={"pose": json.dumps(pose), "scan_id": "o1"}, headers=auth)
+ assert r.status_code == 200, r.text
+ assert r.json()["room"] == "Office" and len(r.json()["pose"]) == 16 and r.json()["hint"][0] == "tv"
+ bad = client.post("/homes/flat/locate", files=files, data={"pose": "{}", "scan_id": "o1"}, headers=auth)
+ assert bad.status_code == 422
+ gone = client.post(
+ "/homes/flat/locate", files=files, data={"pose": json.dumps(pose), "scan_id": "zz"}, headers=auth
+ )
+ assert gone.status_code == 404
diff --git a/services/api/uv.lock b/services/api/uv.lock
index 49191cb..38ff684 100644
--- a/services/api/uv.lock
+++ b/services/api/uv.lock
@@ -760,6 +760,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1a/07/161270b0c2eec56e4c905f6d6d22e1b836887b2cb189d3f5820aa588e9dd/numpy-2.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:aa1cce2ff3f8d953de38b76bf44602caeb69f101430208f64a10067f7cb4b1d3", size = 10521630, upload-time = "2026-10-10T20:03:06.767Z" },
]
+[[package]]
+name = "opencv-python-headless"
+version = "4.14.0.94"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/67/8a/51ff1c6287be0b6cd3070cf095875da4ff7d6eda38507e6ada5608786a78/opencv_python_headless-4.14.0.94.tar.gz", hash = "sha256:4afa2ea1214453648be88259f035712454faa9039b686de7753569ba8eec1577", size = 96405664, upload-time = "2026-07-28T19:23:48.758Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9a/56/12f9b05628cf4ab6aad54479b9124f7c1f2f92b6a4e47c071beb63d926a2/opencv_python_headless-4.14.0.94-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:bc7db37dc234f7bb3190a158fd9dd750357246fc7d8adb23698843ef721a993b", size = 46491231, upload-time = "2026-07-29T03:52:11.902Z" },
+ { url = "https://files.pythonhosted.org/packages/23/3b/2f9c42466f62aa27e97aaf6c9d914008895ecde28013ba4beed7f541f593/opencv_python_headless-4.14.0.94-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:1777f43c9fa064f54b916ad70d944b4fde0a644a17e49f04a966bb24a4b5f1e1", size = 33084176, upload-time = "2026-07-28T20:05:59.902Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/91/83c51e3fe000b5dfce4f3be5c5d320e07ebab5c1742abc4e45674503f90f/opencv_python_headless-4.14.0.94-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:29714d7716dbfddf9fec20ffb878765e94ccaecd7d3ebd6750877420296e2dc5", size = 35405458, upload-time = "2026-07-28T19:19:29.997Z" },
+ { url = "https://files.pythonhosted.org/packages/de/06/14c8c5d331bc72738683a9b227966aceaa4c6d08752f33d64578aebcb6f8/opencv_python_headless-4.14.0.94-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5e02669eac0ba67b2a22d7245af1e8ee1a2ef1185ee526a063d8f0555224bd52", size = 57534165, upload-time = "2026-07-28T19:19:52.742Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/0e/0b98bd582582253f61c4506f9b465cf5a8aa7fa4f7237dd99e1b5151a159/opencv_python_headless-4.14.0.94-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:97c6e818c6f71c0cfa214e12293b1d0266d679c38f4e086de08357c8ac6ece0d", size = 38104309, upload-time = "2026-07-28T19:20:08.818Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/ce/7f538891722a4f06921419d55bac6f41729258d54442861edb2de0dbdf5e/opencv_python_headless-4.14.0.94-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:211e581f5a4670acbbe08fff36a35e9946039d2eea28b80394632d036d1be527", size = 61960165, upload-time = "2026-07-28T19:20:33.074Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/91/ded03abad74d7264fa6bc7c923d7cade6f94a9023eeb372cfad54c2d0334/opencv_python_headless-4.14.0.94-cp37-abi3-win32.whl", hash = "sha256:f70296aa7ac9d7ade0d925c43fdc006c83e20a23f30e2f40f1b82c74a7a54460", size = 33030271, upload-time = "2026-07-28T18:32:39.447Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/8d/db8673846ee53cbb5de4c2b4decc11cf733e203eb7d5146297869f69bd48/opencv_python_headless-4.14.0.94-cp37-abi3-win_amd64.whl", hash = "sha256:cbed65415b8f6a9541c705afe3e64795840524d0ff3bc58f507826284a1dc64b", size = 41028030, upload-time = "2026-07-28T18:32:36.724Z" },
+]
+
[[package]]
name = "opentelemetry-api"
version = "1.45.2"
@@ -1214,6 +1233,7 @@ dependencies = [
{ name = "httpx" },
{ name = "huey" },
{ name = "numpy" },
+ { name = "opencv-python-headless" },
{ name = "opentelemetry-exporter-otlp-proto-http" },
{ name = "opentelemetry-instrumentation-fastapi" },
{ name = "opentelemetry-instrumentation-httpx" },
@@ -1245,6 +1265,7 @@ requires-dist = [
{ name = "httpx", specifier = ">=0.28" },
{ name = "huey", specifier = ">=2.5" },
{ name = "numpy", specifier = ">=2.1" },
+ { name = "opencv-python-headless", specifier = ">=4.10,<5" },
{ name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30" },
{ name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.51b0" },
{ name = "opentelemetry-instrumentation-httpx", specifier = ">=0.51b0" },
diff --git a/services/pipeline/src/recall_pipeline/adapters/place_index.py b/services/pipeline/src/recall_pipeline/adapters/place_index.py
new file mode 100644
index 0000000..3ac1de6
--- /dev/null
+++ b/services/pipeline/src/recall_pipeline/adapters/place_index.py
@@ -0,0 +1,102 @@
+"""Builds a scan's place index (place_index.npz in the episode folder): ORB features of every keyframe, lifted to 3D
+with the frame's own LiDAR depth and ARKit pose. Classic OpenCV features, no model. The API's photo matcher reads it.
+
+ python -m recall_pipeline.adapters.place_index EPISODE_DIR [EPISODE_DIR ...]
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+from recall_pipeline.domain.geometry import column_major_to_matrix
+from recall_pipeline.domain.place_index import lift_keypoints
+
+FORMAT = "recall-place-index/1"
+FEATURES = 1000
+LONG_EDGE = 640 # ORB runs on a 640-pixel frame (the API matches the phone's photo at the same size)
+
+
+def orb() -> cv2.ORB:
+ return cv2.ORB_create(nfeatures=FEATURES, scaleFactor=1.2, nlevels=8, fastThreshold=12)
+
+
+def features(gray: np.ndarray, detector: cv2.ORB | None = None) -> tuple[np.ndarray, np.ndarray]:
+ """Keypoints (N, 2) in the given image's pixels and their ORB descriptors (N, 32), on a 640-pixel copy."""
+ h, w = gray.shape
+ s = LONG_EDGE / max(h, w)
+ small = cv2.resize(gray, (round(w * s), round(h * s)), interpolation=cv2.INTER_AREA) if s < 1 else gray
+ kps, desc = (detector or orb()).detectAndCompute(small, None)
+ if desc is None or not kps:
+ return np.zeros((0, 2)), np.zeros((0, 32), np.uint8)
+ uv = np.array([k.pt for k in kps], dtype=np.float64) / min(s, 1.0)
+ return uv, desc
+
+
+def build(episode_dir: str | Path) -> Path | None:
+ """Writes place_index.npz next to the episode's frames; None when the episode has no frames or depth."""
+ ep = Path(episode_dir)
+ manifest = json.loads((ep / "manifest.json").read_text())
+ k = manifest["intrinsics"]
+ K = np.array([[k["fx"], 0, k["cx"]], [0, k["fy"], k["cy"]], [0, 0, 1.0]])
+ size = (manifest["image"]["width"], manifest["image"]["height"])
+ poses = {}
+ for line in (ep / "poses.jsonl").read_text().splitlines():
+ p = json.loads(line)
+ if p.get("tracking", "normal") == "normal": # a pose ARKit wasn't sure of would place the points wrongly
+ poses[p["index"]] = column_major_to_matrix(p["pose"])
+ detector = orb()
+ descs, points, frames, frame_ids, frame_poses = [], [], [], [], []
+ for index in sorted(poses):
+ img, dep = ep / "frames" / f"{index:06d}.jpg", ep / "depth" / f"{index:06d}.png"
+ if not img.exists() or not dep.exists():
+ continue
+ gray = cv2.imread(str(img), cv2.IMREAD_GRAYSCALE)
+ depth = cv2.imread(str(dep), cv2.IMREAD_UNCHANGED)
+ if gray is None or depth is None:
+ continue
+ uv, desc = features(gray, detector)
+ xyz, keep = lift_keypoints(uv, depth, K, size, poses[index])
+ if len(keep) < 20:
+ continue
+ descs.append(desc[keep])
+ points.append(xyz.astype(np.float32))
+ frames.append(np.full(len(keep), len(frame_ids), dtype=np.int32))
+ frame_ids.append(index)
+ frame_poses.append(poses[index].T.reshape(-1)) # column-major, as the phone writes it
+ if not frame_ids:
+ return None
+ out = ep / "place_index.npz"
+ np.savez_compressed(
+ out,
+ format=np.array(FORMAT),
+ desc=np.concatenate(descs),
+ xyz=np.concatenate(points),
+ frame=np.concatenate(frames),
+ frame_ids=np.array(frame_ids, dtype=np.int32),
+ poses=np.array(frame_poses, dtype=np.float32),
+ image_size=np.array(size, dtype=np.int32),
+ )
+ return out
+
+
+def main(argv: list[str]) -> int:
+ for d in argv:
+ t0 = time.perf_counter()
+ out = build(d)
+ if out is None:
+ print(f"{d}: no frames with depth, no index", file=sys.stderr)
+ continue
+ with np.load(out) as z:
+ n, f = len(z["desc"]), len(z["frame_ids"])
+ print(f"{out}: {f} keyframes, {n} points, {time.perf_counter() - t0:.1f} s")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/services/pipeline/src/recall_pipeline/adapters/run.py b/services/pipeline/src/recall_pipeline/adapters/run.py
index 482eb1e..62080b9 100644
--- a/services/pipeline/src/recall_pipeline/adapters/run.py
+++ b/services/pipeline/src/recall_pipeline/adapters/run.py
@@ -89,4 +89,11 @@ def process(episode_dir: str, scan_id: str, post: bool = True, out: str | None =
target.write_text(json.dumps(result, indent=1))
print(json.dumps(result["stats"]))
print(f"{len(result['landmarks'])} landmarks, {len(result['objects'])} objects -> {target}")
+ try: # the photo matcher's index: a failure here never fails the scan (ARKit's room map still works)
+ from recall_pipeline.adapters.place_index import build
+
+ index = build(episode_dir)
+ print(f"place index -> {index}" if index else "place index: no frames with depth")
+ except Exception as e: # noqa: BLE001
+ print(f"place index failed: {e}", file=sys.stderr)
return 0
diff --git a/services/pipeline/src/recall_pipeline/domain/place_index.py b/services/pipeline/src/recall_pipeline/domain/place_index.py
new file mode 100644
index 0000000..20bee7d
--- /dev/null
+++ b/services/pipeline/src/recall_pipeline/domain/place_index.py
@@ -0,0 +1,52 @@
+"""A scan's place index: the keyframes' feature points lifted to 3D with the phone's own LiDAR depth and ARKit pose.
+
+The API matches one photo from the phone against it to say which room the phone is in and where it stands there
+(classic features + PnP, no model), alongside ARKit's own relocalisation. Everything is in the scan's ARKit world
+(metres, y up), the frame the phone's saved room map uses.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from recall_pipeline.domain.geometry import arkit_to_opencv
+
+DEPTH_MIN_M, DEPTH_MAX_M = 0.2, 6.0 # LiDAR is reliable in this range
+
+
+def depth_at(depth_mm: np.ndarray, uv_depth: np.ndarray, reach: int = 2) -> np.ndarray:
+ """Depth in metres at (fractional) depth-map pixels: the nearest pixel, or else the median of the valid pixels
+ within `reach` (an edge pixel often has no LiDAR return); NaN where there is none or it is out of range."""
+ h, w = depth_mm.shape
+ out = np.full(len(uv_depth), np.nan)
+ cols = np.clip(np.floor(uv_depth[:, 0]).astype(int), 0, w - 1)
+ rows = np.clip(np.floor(uv_depth[:, 1]).astype(int), 0, h - 1)
+ for i, (r, c) in enumerate(zip(rows, cols, strict=True)):
+ v = float(depth_mm[r, c])
+ if v <= 0:
+ patch = depth_mm[max(r - reach, 0) : r + reach + 1, max(c - reach, 0) : c + reach + 1]
+ valid = patch[patch > 0]
+ v = float(np.median(valid)) if valid.size else 0.0
+ m = v / 1000.0
+ if DEPTH_MIN_M <= m <= DEPTH_MAX_M:
+ out[i] = m
+ return out
+
+
+def lift_keypoints(
+ uv: np.ndarray, depth_mm: np.ndarray, K: np.ndarray, image_size: tuple[int, int], cam2world_arkit: np.ndarray
+) -> tuple[np.ndarray, np.ndarray]:
+ """Image pixels (N, 2) of a keyframe → (world points (M, 3), indices of the kept pixels). `K` is for `image_size`
+ (width, height); the depth map may be smaller (LiDAR is 256 x 192) and is sampled at the same relative spot."""
+ if len(uv) == 0:
+ return np.zeros((0, 3)), np.zeros(0, dtype=int)
+ w_img, h_img = image_size
+ dh, dw = depth_mm.shape
+ z = depth_at(depth_mm, uv * np.array([dw / w_img, dh / h_img]))
+ keep = np.flatnonzero(np.isfinite(z))
+ z = z[keep]
+ x = (uv[keep, 0] - K[0, 2]) / K[0, 0] * z
+ y = (uv[keep, 1] - K[1, 2]) / K[1, 1] * z
+ cam = np.stack([x, y, z], axis=1)
+ T = arkit_to_opencv(cam2world_arkit)
+ return cam @ T[:3, :3].T + T[:3, 3], keep
diff --git a/services/pipeline/tests/unit/test_place_index.py b/services/pipeline/tests/unit/test_place_index.py
new file mode 100644
index 0000000..85a8f22
--- /dev/null
+++ b/services/pipeline/tests/unit/test_place_index.py
@@ -0,0 +1,24 @@
+import numpy as np
+
+from recall_pipeline.domain.place_index import depth_at, lift_keypoints
+
+
+def test_keypoints_lift_onto_a_wall_in_front_of_an_arkit_camera():
+ # ARKit camera at (1, 1.5, 0) looking down −z at a wall 2 m away; the image is 4x the depth map
+ K = np.array([[700.0, 0, 512], [0, 700.0, 384], [0, 0, 1]])
+ depth = np.full((192, 256), 2000, np.uint16)
+ pose = np.eye(4)
+ pose[:3, 3] = [1.0, 1.5, 0.0]
+ uv = np.array([[512.0, 384.0], [862.0, 384.0], [512.0, 34.0]])
+ xyz, keep = lift_keypoints(uv, depth, K, (1024, 768), pose)
+ assert list(keep) == [0, 1, 2]
+ # straight ahead; 350 px right = 1 m right; 350 px up (smaller v) = 1 m up (ARKit +y)
+ assert np.allclose(xyz, [[1.0, 1.5, -2.0], [2.0, 1.5, -2.0], [1.0, 2.5, -2.0]])
+
+
+def test_missing_depth_borrows_a_neighbour_and_out_of_range_is_dropped():
+ depth = np.zeros((10, 10), np.uint16)
+ depth[5, 6] = 1500 # one valid pixel next to the asked one
+ depth[0, 0] = 9000 # 9 m: past LiDAR's range
+ z = depth_at(depth, np.array([[5.2, 5.2], [0.1, 0.1], [9.0, 9.0]]))
+ assert z[0] == 1.5 and np.isnan(z[1]) and np.isnan(z[2])
From bd3ea514d40519193b14a14a442bbde209f08d26 Mon Sep 17 00:00:00 2001
From: Rachit
Date: Mon, 2 Nov 2026 12:16:29 +0100
Subject: [PATCH 04/30] feat(ios): recognise the room from anywhere: photo
fixes beside ARKit, a growing map, named coaching
- Guide warms up and uses the map of the thing's own room (it used the
newest scan of any room).
- While ARKit hasn't matched, PhotoAssist sends about one photo a second
to the Mac's matcher: it says when the phone is in another room, names
furniture to point at, and applies a pose only when two fixes agree
(25 cm, 10 degrees). ARKit's own match always replaces it.
- ARLocator keeps labels, arrows and the route under one map node, so a
photo correction moves them together.
- Once ARKit recognises the room, the map it grew is saved beside the
scan's own (worldmap.grown.arworldmap), so the next visit matches from
there too.
- See it in AR shows the list at 25 s but keeps looking for 2 minutes and
switches the labels on when the room is recognised.
Co-Authored-By: Claude Opus 5.5
---
.../RecallApplication/GuideToThing.swift | 15 +-
.../RecallApplication/PhotoAssist.swift | 78 +++++++++
.../Sources/RecallCapture/ARLocator.swift | 109 +++++++++++-
.../RecallDomain/Locate/PhotoFix.swift | 83 +++++++++
.../Sources/RecallDomain/Ports/Ports.swift | 13 ++
.../Sources/RecallDomain/Setup/Privacy.swift | 3 +-
.../Sources/RecallNetworking/RecallAPI.swift | 32 ++++
.../Sources/RecallNetworking/openapi.json | 164 ++++++++++++++++++
.../RecallPersistence/FileStores.swift | 13 +-
.../Sources/RecallTestSupport/Fakes.swift | 37 +++-
.../PhotoAssistTests.swift | 86 +++++++++
.../RecallDomainTests/PhotoFixTests.swift | 48 +++++
apps/ios/Recall/App/AppContainer.swift | 32 +++-
.../Recall/Features/AR/ARLabelsModel.swift | 54 +++++-
apps/ios/Recall/Features/Find/FindModel.swift | 6 +-
apps/ios/Recall/Models/ScansModel.swift | 19 +-
16 files changed, 749 insertions(+), 43 deletions(-)
create mode 100644 apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift
create mode 100644 apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift
create mode 100644 apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift
create mode 100644 apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift
index 65a3235..aa7cfe2 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift
@@ -45,15 +45,18 @@ public final class GuideToThing {
private let haptics: any HapticsOutput
private let clock: any Clock
private let compass: (any CompassSensor)?
+ private let photos: (places: any PlaceLocating, scan: String)?
private let settings: Settings
private var cancelled = false
+ /// `photos`: the Mac's photo matcher and the scan whose room map this is, to recognise the room alongside ARKit.
public init(home: String, thing: Thing, roomMap: Data?, locator: any Locator, guide: any GuideService,
speech: any SpeechOutput, haptics: any HapticsOutput, clock: any Clock,
- compass: (any CompassSensor)? = nil, settings: Settings = .init()) {
+ compass: (any CompassSensor)? = nil, photos: (places: any PlaceLocating, scan: String)? = nil,
+ settings: Settings = .init()) {
self.home = home; self.thing = thing; self.roomMap = roomMap; self.locator = locator; self.guide = guide
self.speech = speech; self.haptics = haptics; self.clock = clock; self.compass = compass
- self.settings = settings
+ self.photos = photos; self.settings = settings
}
/// The compass heading when it can be trusted (else directions leave out "north-east").
@@ -122,8 +125,14 @@ public final class GuideToThing {
// MARK: steps
- /// Waits for tracking to turn normal (ARKit recognised the room) or the timeout.
+ /// Waits for tracking to turn normal (ARKit recognised the room, or two photo fixes agreed) or the timeout.
private func locate() async -> TrackingState {
+ let assist = photos.map {
+ PhotoAssist(home: home, scan: $0.scan, room: thing.room, locator: locator, places: $0.places, speech: speech,
+ clock: clock)
+ }
+ let helping = assist.map { a in Task { await a.run() } }
+ defer { assist?.cancel(); helping?.cancel() }
let started = clock.now()
while !cancelled, clock.now().timeIntervalSince(started) < settings.locateTimeout {
if locator.reading()?.tracking == .normal { return .normal }
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift
new file mode 100644
index 0000000..2d11d25
--- /dev/null
+++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift
@@ -0,0 +1,78 @@
+import Foundation
+import RecallDomain
+
+/// Helps ARKit recognise the room, never replaces it: while ARKit hasn't matched the saved room map, about once a
+/// second one camera photo goes to the Mac's photo matcher (classic features + PnP, no model). From its answers:
+/// - the room is somewhere else → said once ("You seem to be in Bedroom, not Living room…");
+/// - what to point at → said every few seconds ("Point the phone slowly at the TV.");
+/// - two fixes in a row that agree → the locator takes the correction and tracking reads normal at once.
+/// The moment ARKit matches by itself it stops asking. No Mac, or an error: nothing is said and ARKit carries on.
+@MainActor
+public final class PhotoAssist {
+ public struct Settings: Sendable {
+ public var firstAfter: Double = 1.0 // ARKit often matches within a second: then nothing is sent
+ public var every: Double = 1.0 // the Mac answers in about 35 ms; one photo a second is plenty
+ public var maxAsks = 40
+ public var hintEvery: Double = 8 // seconds between "point at …" lines
+ public init() {}
+ }
+
+ private let home: String
+ private let scan: String
+ private let room: String?
+ private let locator: any Locator
+ private let places: any PlaceLocating
+ private let speech: any SpeechOutput
+ private let clock: any Clock
+ private let settings: Settings
+ private var cancelled = false
+
+ /// Whether a photo fix (not ARKit) located the phone.
+ public private(set) var fixed = false
+
+ public init(home: String, scan: String, room: String?, locator: any Locator, places: any PlaceLocating,
+ speech: any SpeechOutput, clock: any Clock, settings: Settings = .init()) {
+ self.home = home; self.scan = scan; self.room = room; self.locator = locator; self.places = places
+ self.speech = speech; self.clock = clock; self.settings = settings
+ }
+
+ public func cancel() { cancelled = true }
+
+ private var located: Bool { locator.reading()?.tracking == .normal }
+
+ /// Until the room is recognised (by ARKit or two agreeing fixes), `cancel()` or the ask limit.
+ public func run() async {
+ try? await clock.sleep(seconds: settings.firstAfter)
+ var agreement = FixAgreement()
+ var turn = 0
+ var lastHint: Date?
+ var toldElsewhere = false
+ for _ in 0..= settings.hintEvery }) ?? true,
+ let line = LocateCoaching.point(at: fix.hint, turn: turn) {
+ speech.say(line, interrupt: false)
+ turn += 1
+ lastHint = clock.now()
+ }
+ if let pose = fix.pose {
+ let session = Pose(columnMajor: frame.pose)
+ if let agreed = agreement.add(MapCorrection.between(map: pose, session: session), session: session) {
+ locator.correct(mapFromSession: Pose(columnMajor: MapCorrection.columnMajor(agreed)))
+ fixed = true
+ return
+ }
+ }
+ }
+ try? await clock.sleep(seconds: settings.every)
+ }
+ }
+}
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift
index 6e2a75e..d7ece00 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift
@@ -13,13 +13,27 @@ public final class ARLocator: Locator {
/// What the camera view draws over the feed: the arrow above the thing.
public let scene = SCNScene()
private var arrow: SCNNode?
+ /// Everything placed in the room map's world hangs here: the session's own world when ARKit matched the map,
+ /// or turned by a photo fix's correction until it does.
+ private let mapRoot = SCNNode()
private let track = SCNNode()
private let tags = SCNNode()
private var shown: [ArLabel] = []
+ /// T(map ← session) from two agreeing photo fixes, until ARKit recognises the room itself.
+ private var correction: simd_double4x4?
+ private var watching: Task?
+ /// ARKit itself matched the saved map in this run (not a photo fix): the map may grow from here.
+ public private(set) var recognisedByARKit = false
+ /// Handed a grown room map (ARKit's map of this run, which keeps the saved map's world) once ARKit has matched
+ /// and mapped what it sees; the caller saves it beside the scan's own so the next visit matches from here too.
+ public var onMapGrown: ((Data) -> Void)?
+ private var growing: Task?
+
public init() {
- scene.rootNode.addChildNode(track)
- scene.rootNode.addChildNode(tags)
+ scene.rootNode.addChildNode(mapRoot)
+ mapRoot.addChildNode(track)
+ mapRoot.addChildNode(tags)
}
public static var isSupported: Bool { ARWorldTrackingConfiguration.isSupported }
@@ -43,6 +57,7 @@ public final class ARLocator: Locator {
}
guard let roomMap else {
if running, !runningWithMap { return true }
+ forgetRun()
session.run(config, options: [.resetTracking, .removeExistingAnchors])
running = true; runningWithMap = false
return true
@@ -54,6 +69,7 @@ public final class ARLocator: Locator {
// TODO(verify) on the iPhone how quickly tracking returns to normal after a resume.
session.run(config, options: [])
} else {
+ forgetRun()
config.initialWorldMap = current.map
session.run(config, options: [.resetTracking, .removeExistingAnchors])
}
@@ -64,6 +80,7 @@ public final class ARLocator: Locator {
return false
}
current = (roomMap, map)
+ forgetRun()
config.initialWorldMap = map
session.run(config, options: [.resetTracking, .removeExistingAnchors])
running = true; runningWithMap = true
@@ -74,6 +91,79 @@ public final class ARLocator: Locator {
public func stop() {
session.pause()
running = false
+ growing?.cancel()
+ }
+
+ /// A fresh ARKit run: no photo correction, nothing recognised yet.
+ private func forgetRun() {
+ correct(mapFromSession: nil)
+ recognisedByARKit = false
+ growing?.cancel()
+ growing = nil
+ }
+
+ // MARK: Photo fixes and ARKit's own match
+
+ public func correct(mapFromSession: Pose?) {
+ correction = mapFromSession.map(MapCorrection.matrix)
+ let inverse = correction.map { Self.float($0.inverse) } ?? matrix_identity_float4x4
+ mapRoot.simdTransform = inverse // the scene renders in the session's world: map → session
+ watching?.cancel()
+ guard correction != nil else { return }
+ watching = Task { [weak self] in // ARKit's own match replaces the photo fix the moment it comes
+ while !Task.isCancelled {
+ try? await Task.sleep(for: .milliseconds(250))
+ guard let self, !Task.isCancelled else { return }
+ _ = self.reading()
+ if self.correction == nil { return }
+ }
+ }
+ }
+
+ /// The camera in the room map's world, and whether the room is recognised (by ARKit or a photo fix).
+ private func mapCamera() -> (transform: simd_float4x4, tracking: TrackingState)? {
+ guard let camera = session.currentFrame?.camera else { return nil }
+ let raw = CaptureSession.tracking(camera.trackingState).0
+ if raw == .normal, runningWithMap {
+ if correction != nil { correct(mapFromSession: nil) } // the session's world is now the map's
+ if !recognisedByARKit { recognisedByARKit = true; growMapSoon() }
+ }
+ guard let correction else { return (camera.transform, raw) }
+ return (Self.float(correction * Self.double(camera.transform)), .normal)
+ }
+
+ /// A few seconds after ARKit matched, once it has mapped what the camera sees, the run's map (saved map plus what
+ /// this visit added, in the same world) goes to `onMapGrown`. At most once a run; never from a photo fix alone.
+ private func growMapSoon() {
+ guard onMapGrown != nil, growing == nil else { return }
+ growing = Task { [weak self] in
+ try? await Task.sleep(for: .seconds(6))
+ guard let self, !Task.isCancelled, self.running, self.recognisedByARKit,
+ self.session.currentFrame?.worldMappingStatus == .mapped else { return }
+ guard let data = await self.currentMapData(), data.count < 50_000_000 else { return }
+ guard !Task.isCancelled else { return }
+ self.onMapGrown?(data)
+ }
+ }
+
+ static func float(_ m: simd_double4x4) -> simd_float4x4 {
+ simd_float4x4(columns: (SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2),
+ SIMD4(m.columns.3)))
+ }
+
+ static func double(_ m: simd_float4x4) -> simd_double4x4 {
+ simd_double4x4(columns: (SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2),
+ SIMD4(m.columns.3)))
+ }
+
+ private func currentMapData() async -> Data? {
+ await withCheckedContinuation { cont in
+ session.getCurrentWorldMap { map, _ in
+ cont.resume(returning: map.flatMap {
+ try? NSKeyedArchiver.archivedData(withRootObject: $0, requiringSecureCoding: true)
+ })
+ }
+ }
}
private let encoder = FrameEncoder()
@@ -84,7 +174,9 @@ public final class ARLocator: Locator {
// ARKit's intrinsics are for the full camera image; the JPEG is scaled to a 1024-pixel long edge
let s = Double(size.width) / Double(frame.camera.imageResolution.width)
let k = frame.camera.intrinsics
- let pose = frame.camera.transform.columnMajor.map(Double.init)
+ // in the room map's world once a photo fix corrected it (the Mac lifts the arrival photo in that world)
+ let camera = correction.map { Self.float($0 * Self.double(frame.camera.transform)) } ?? frame.camera.transform
+ let pose = camera.columnMajor.map(Double.init)
guard let (mm, w, h) = Self.millimetres(depth) else { return nil }
return ArrivalFrame(jpeg: jpeg, depthMM: mm, depthSize: (w, h), imageSize: (Int(size.width), Int(size.height)),
intrinsics: (Double(k.columns.0.x) * s, Double(k.columns.1.y) * s,
@@ -123,7 +215,7 @@ public final class ARLocator: Locator {
up.timingMode = .easeInEaseOut
node.runAction(.repeatForever(.sequence([up, up.reversed()])))
}
- scene.rootNode.addChildNode(node)
+ mapRoot.addChildNode(node)
arrow = node
}
@@ -186,7 +278,7 @@ public final class ARLocator: Locator {
/// The label the camera points at most directly (within 25°), for "what's this?".
public func labelInView() -> ArLabel? {
- guard let cam = session.currentFrame?.camera.transform else { return nil }
+ guard let cam = mapCamera()?.transform else { return nil }
let eye = SIMD3(Double(cam.columns.3.x), Double(cam.columns.3.y), Double(cam.columns.3.z))
let fwd = -SIMD3(Double(cam.columns.2.x), Double(cam.columns.2.y), Double(cam.columns.2.z))
let scored = shown.map { label -> (ArLabel, Double) in
@@ -216,7 +308,7 @@ public final class ARLocator: Locator {
let chevrons = ChevronLayout.place(path: path)
guard !chevrons.isEmpty else { return }
// without the floor from the Mac, assume the phone is held about 1.4 m above it
- let y = Float(floorY ?? Double((session.currentFrame?.camera.transform.columns.3.y ?? 1.4) - 1.4)) + 0.01
+ let y = Float(floorY ?? Double((mapCamera()?.transform.columns.3.y ?? 1.4) - 1.4)) + 0.01
let still = UIAccessibility.isReduceMotionEnabled
for (i, c) in chevrons.enumerated() {
let node = Self.chevronNode()
@@ -296,10 +388,9 @@ public final class ARLocator: Locator {
}
public func reading() -> (pose: Pose, tracking: TrackingState)? {
- guard let camera = session.currentFrame?.camera else { return nil }
- let m = camera.transform
+ guard let (m, tracking) = mapCamera() else { return nil }
let pose = Pose(columnMajor: [m.columns.0, m.columns.1, m.columns.2, m.columns.3]
.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] })
- return (pose, CaptureSession.tracking(camera.trackingState).0)
+ return (pose, tracking)
}
}
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift
new file mode 100644
index 0000000..521d0ec
--- /dev/null
+++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift
@@ -0,0 +1,83 @@
+import Foundation
+import simd
+
+/// What the Mac's photo matcher said about one camera photo (classic features + PnP against the scans; no model).
+public struct PlaceFix: Sendable, Equatable {
+ /// The room the photo was taken in, when one room clearly matched (it may not be the room being looked for).
+ public let room: String?
+ /// The camera's pose in the room map's world, when that room matched well enough.
+ public let pose: Pose?
+ public let inliers: Int
+ /// What to point the phone at in that room, one-of-a-kind furniture first ("tv", "cabinet").
+ public let hint: [String]
+
+ public init(room: String?, pose: Pose?, inliers: Int, hint: [String]) {
+ self.room = room; self.pose = pose; self.inliers = inliers; self.hint = hint
+ }
+}
+
+/// From a photo fix to the phone's own poses: T(map ← session) = map pose × session pose⁻¹, for the same photo.
+public enum MapCorrection {
+ public static func between(map: Pose, session: Pose) -> simd_double4x4 { matrix(map) * matrix(session).inverse }
+
+ /// A session pose in the room map's world.
+ public static func apply(_ T: simd_double4x4, to pose: Pose) -> Pose { Pose(columnMajor: columnMajor(T * matrix(pose))) }
+
+ public static func matrix(_ p: Pose) -> simd_double4x4 {
+ let c = p.columnMajor
+ return simd_double4x4(columns: (SIMD4(c[0], c[1], c[2], c[3]), SIMD4(c[4], c[5], c[6], c[7]),
+ SIMD4(c[8], c[9], c[10], c[11]), SIMD4(c[12], c[13], c[14], c[15])))
+ }
+
+ public static func columnMajor(_ m: simd_double4x4) -> [Double] {
+ [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] }
+ }
+
+ /// How far apart two transforms are: metres between their origins, degrees between their rotations.
+ public static func distance(_ a: simd_double4x4, _ b: simd_double4x4) -> (metres: Double, degrees: Double) {
+ let ta = SIMD3(a.columns.3.x, a.columns.3.y, a.columns.3.z), tb = SIMD3(b.columns.3.x, b.columns.3.y, b.columns.3.z)
+ let ra = simd_double3x3(SIMD3(a.columns.0.x, a.columns.0.y, a.columns.0.z), SIMD3(a.columns.1.x, a.columns.1.y, a.columns.1.z),
+ SIMD3(a.columns.2.x, a.columns.2.y, a.columns.2.z))
+ let rb = simd_double3x3(SIMD3(b.columns.0.x, b.columns.0.y, b.columns.0.z), SIMD3(b.columns.1.x, b.columns.1.y, b.columns.1.z),
+ SIMD3(b.columns.2.x, b.columns.2.y, b.columns.2.z))
+ let d = ra.transpose * rb
+ let cosine = max(-1, min(1, (d.columns.0.x + d.columns.1.y + d.columns.2.z - 1) / 2))
+ return (simd_distance(ta, tb), acos(cosine) * 180 / .pi)
+ }
+}
+
+/// One photo fix can be wrong (on real scans about 1 in 9 from another session), so the phone trusts photo fixes
+/// only when two in a row agree on the correction: within 25 cm and 10°. Two wrong fixes rarely agree.
+public struct FixAgreement: Sendable {
+ public var maxMetres = 0.25
+ public var maxDegrees = 10.0
+ private var last: (correction: simd_double4x4, session: Pose)?
+
+ public init() {}
+
+ /// Adds a fix's correction and the session pose its photo had; the agreed correction once two in a row agree.
+ public mutating func add(_ correction: simd_double4x4, session: Pose) -> simd_double4x4? {
+ defer { last = (correction, session) }
+ // a session pose that never changed means ARKit isn't tracking: a correction from it would hold nothing
+ guard let last, last.session != session else { return nil }
+ let d = MapCorrection.distance(last.correction, correction)
+ return d.metres <= maxMetres && d.degrees <= maxDegrees ? correction : nil
+ }
+}
+
+/// The spoken lines while the room is being recognised.
+public enum LocateCoaching {
+ /// "Point the phone slowly at the TV." then "Now at the cabinet." (`turn` counts the lines said so far).
+ public static func point(at hint: [String], turn: Int) -> String? {
+ guard !hint.isEmpty else { return nil }
+ let name = spoken(hint[turn % hint.count])
+ return turn == 0 ? "Point the phone slowly at the \(name)." : "Now at the \(name)."
+ }
+
+ public static func elsewhere(found: String, wanted: String) -> String {
+ "You seem to be in \(found), not \(wanted). Walk to \(wanted) and I'll keep looking."
+ }
+
+ /// A furniture label as said aloud: short ones are initials ("tv" → "TV").
+ public static func spoken(_ label: String) -> String { label.count <= 2 ? label.uppercased() : label }
+}
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift
index 055cf17..2ee4cf6 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift
@@ -112,7 +112,10 @@ public protocol ScanLibrary: Sendable {
public protocol EpisodeFiles: Sendable {
func directory(scan: String) -> URL
func manifest(scan: String) -> Manifest?
+ /// The scan's room map: the one ARKit grew from later visits when there is one, else the one the scan saved.
func roomMap(scan: String) -> Data?
+ /// Keeps a grown room map (the scan's own stays as uploaded).
+ func saveGrownRoomMap(scan: String, _ data: Data)
func delete(scan: String)
}
@@ -169,6 +172,16 @@ public protocol Clock: Sendable {
func arrivalFrame() -> ArrivalFrame?
/// Lays arrows on the floor along the route (ARKit x, z), at the floor's height when known; empty clears them.
func showPath(_ path: [SIMD2], floorY: Double?)
+ /// A photo fix's correction, T(map ← session) as a column-major 4 x 4: from now on `reading()` and
+ /// `arrivalFrame()` answer in the room map's world and tracking reads normal. nil drops it; so does ARKit's own
+ /// match of the room (its world then is the map's).
+ func correct(mapFromSession: Pose?)
+}
+
+/// The Mac's photo matcher: which room a camera photo shows, and the camera's pose in a room map's world.
+public protocol PlaceLocating: Sendable {
+ /// `scan`: the scan whose room map the phone runs; `room`: the room being looked for.
+ func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix
}
/// The phone's compass (magnetic north; no location is read).
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift
index 9b5dea1..afa3e09 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift
@@ -43,7 +43,8 @@ public enum PrivacyNote {
"On your iPhone: your scans until you delete them, the room map used to find your way, and the Mac's "
+ "address. The device key is in the Keychain.",
"Between them: your home network, or an encrypted tunnel you set up (Tailscale or Cloudflare Tunnel). "
- + "Every request needs the device key.",
+ + "Every request needs the device key. While Recall finds the room, about one camera photo a second goes "
+ + "to your Mac (only there), to be matched against your scans; it is not stored.",
]
public static let never = [
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift
index c0906be..eb07846 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift
@@ -70,6 +70,38 @@ public struct RecallAPI: Sendable {
}
}
+// MARK: PlaceLocating
+
+extension RecallAPI: PlaceLocating {
+ /// Where am I? One photo to the Mac's matcher (LocateRequest.yaml). A short timeout: an answer that comes late is
+ /// worth less than the next photo.
+ public func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix {
+ let boundary = "recall-\(UUID().uuidString)"
+ var body = Data()
+ func part(_ name: String, filename: String? = nil, type: String? = nil, _ data: Data) {
+ body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"\(name)\"".utf8))
+ if let filename { body.append(Data("; filename=\"\(filename)\"".utf8)) }
+ body.append(Data("\r\n".utf8))
+ if let type { body.append(Data("Content-Type: \(type)\r\n".utf8)) }
+ body.append(Data("\r\n".utf8))
+ body.append(data)
+ body.append(Data("\r\n".utf8))
+ }
+ part("image", filename: "now.jpg", type: "image/jpeg", frame.jpeg)
+ part("pose", frame.poseJSON)
+ part("scan_id", Data(scan.utf8))
+ if let room { part("room", Data(room.utf8)) }
+ body.append(Data("--\(boundary)--\r\n".utf8))
+ let path = "homes/\(home.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? home)/locate"
+ var req = request(path, method: "POST", contentType: "multipart/form-data; boundary=\(boundary)", body: body)
+ req.timeoutInterval = 4
+ let json = try await send(req)
+ let pose = (json["pose"] as? [Double]).flatMap { $0.count == 16 ? Pose(columnMajor: $0) : nil }
+ return PlaceFix(room: json["room"] as? String, pose: pose, inliers: json["inliers"] as? Int ?? 0,
+ hint: json["hint"] as? [String] ?? [])
+ }
+}
+
// MARK: HomeMemory
extension RecallAPI: HomeMemory {
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json
index 13b9233..4d7cfe7 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json
+++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json
@@ -846,6 +846,58 @@
}
}
},
+ "/homes/{home_id}/locate": {
+ "post": {
+ "tags": [
+ "homes"
+ ],
+ "operationId": "locateByPhoto",
+ "summary": "Where am I? One camera photo matched against the room scans (classic features + PnP, no model)",
+ "description": "Asked by the phone while ARKit hasn't yet recognised the saved room map, alongside it (never instead of it). Answers the room the photo was taken in when one room clearly wins, the camera's pose in the requested map's world when that room matched well enough, and what to point the phone at. The photo is matched in memory and not stored.\n",
+ "parameters": [
+ {
+ "$ref": "#/components/parameters/home_id"
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "multipart/form-data": {
+ "schema": {
+ "$ref": "#/components/schemas/LocateRequest"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "What the photo matched",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/PlaceFix"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/Unauthorized"
+ },
+ "404": {
+ "$ref": "#/components/responses/NotFound"
+ },
+ "413": {
+ "$ref": "#/components/responses/TooLarge"
+ },
+ "422": {
+ "$ref": "#/components/responses/Invalid"
+ },
+ "429": {
+ "$ref": "#/components/responses/RateLimited"
+ }
+ }
+ }
+ },
"/homes/{home_id}/robot-package": {
"get": {
"tags": [
@@ -3120,6 +3172,118 @@
}
}
},
+ "LocateRequest": {
+ "type": "object",
+ "required": [
+ "image",
+ "pose",
+ "scan_id"
+ ],
+ "properties": {
+ "image": {
+ "type": "string",
+ "format": "binary",
+ "description": "JPEG of the camera now (any size; intrinsics are for this image)"
+ },
+ "pose": {
+ "type": "string",
+ "description": "JSON: {\"pose\": [16 numbers, ARKit camera-to-world, column-major, in the phone's session (not yet the map's)], \"intrinsics\": {\"fx\", \"fy\", \"cx\", \"cy\"} for the JPEG, \"image_size\": [w, h]}. Only the intrinsics are used on the Mac; the phone keeps the session pose to turn the answer into its own correction.\n"
+ },
+ "scan_id": {
+ "type": "string",
+ "maxLength": 64,
+ "description": "The scan whose room map the phone runs: the pose is answered in its world"
+ },
+ "room": {
+ "type": "string",
+ "maxLength": 60,
+ "description": "The room the phone is trying to recognise (for the hint)"
+ }
+ }
+ },
+ "PlaceFix": {
+ "type": "object",
+ "required": [
+ "room",
+ "scan_id",
+ "score",
+ "runner_up_room",
+ "runner_up_score",
+ "pose",
+ "inliers",
+ "reproj_px",
+ "hint",
+ "ms"
+ ],
+ "properties": {
+ "room": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The room the photo was taken in, when one room clearly matched (it may not be the requested map's room)"
+ },
+ "scan_id": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The scan whose world `pose` is in"
+ },
+ "score": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "RANSAC inliers of the best room"
+ },
+ "runner_up_room": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "runner_up_score": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "pose": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "number"
+ },
+ "minItems": 16,
+ "maxItems": 16,
+ "description": "The camera's pose (ARKit camera-to-world, column-major) in the requested map's world; null unless that room matched with at least 30 inliers and at most 4 px mean reprojection error. The phone applies it only when two fixes in a row agree.\n"
+ },
+ "inliers": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "RANSAC inliers in the requested map's world"
+ },
+ "reproj_px": {
+ "type": [
+ "number",
+ "null"
+ ],
+ "description": "Mean reprojection error of those inliers, pixels"
+ },
+ "hint": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "maxItems": 3,
+ "description": "What to point the phone at in that room, one-of-a-kind furniture first"
+ },
+ "ms": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "Time the Mac took"
+ }
+ }
+ },
"CompassRoute": {
"type": "object",
"description": "A walk to a thing from where its room's scan began, as compass legs (for a watch with no camera)",
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift
index 24cb59b..0c2e2e0 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift
@@ -38,7 +38,18 @@ public struct FileEpisodeFiles: EpisodeFiles {
}
public func roomMap(scan: String) -> Data? {
- try? Data(contentsOf: directory(scan: scan).appending(path: "worldmap.arworldmap"))
+ let dir = directory(scan: scan)
+ return (try? Data(contentsOf: dir.appending(path: Self.grownMap)))
+ ?? (try? Data(contentsOf: dir.appending(path: "worldmap.arworldmap")))
+ }
+
+ /// ARKit's map grown on later visits sits beside the scan's own (which stays exactly as uploaded and checksummed).
+ static let grownMap = "worldmap.grown.arworldmap"
+
+ public func saveGrownRoomMap(scan: String, _ data: Data) {
+ let dir = directory(scan: scan)
+ guard FileManager.default.fileExists(atPath: dir.path) else { return }
+ try? data.write(to: dir.appending(path: Self.grownMap), options: [.atomic, .completeFileProtection])
}
/// The folder, its zip and the upload's part files.
diff --git a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift
index 2424d99..f3e53b9 100644
--- a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift
+++ b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift
@@ -153,10 +153,41 @@ public final class FakeClock: Clock, @unchecked Sendable {
public func showPath(_ path: [SIMD2], floorY: Double?) { paths.append(path) }
/// The frame the arrival check gets (nil: the camera has none).
public var frame: ArrivalFrame? = ArrivalFrame.sample
- public func arrivalFrame() -> ArrivalFrame? { frame }
+ /// Each frame 1 cm further along x than the last (a live, tracking session), when set.
+ public var moves = false
+ private var step = 0.0
+ public func arrivalFrame() -> ArrivalFrame? {
+ guard moves, let f = frame else { return frame }
+ step += 0.01
+ var pose = f.pose
+ pose[12] += step
+ return ArrivalFrame(jpeg: f.jpeg, depthMM: f.depthMM, depthSize: f.depthSize, imageSize: f.imageSize,
+ intrinsics: f.intrinsics, pose: pose)
+ }
+ /// The photo correction in use (nil: none), and every one applied.
+ public private(set) var correction: Pose?
+ public private(set) var corrections: [Pose?] = []
+ public func correct(mapFromSession: Pose?) { correction = mapFromSession; corrections.append(mapFromSession) }
+ /// Like ARLocator: with a correction, poses are in the map's world and tracking reads normal, until ARKit's own
+ /// match (a normal reading) drops it.
public func reading() -> (pose: Pose, tracking: TrackingState)? {
guard !readings.isEmpty else { return nil }
- return readings.count > 1 ? readings.removeFirst() : readings[0]
+ let r = readings.count > 1 ? readings.removeFirst() : readings[0]
+ guard let c = correction else { return r }
+ if r.tracking == .normal { correction = nil; return r }
+ return (MapCorrection.apply(MapCorrection.matrix(c), to: r.pose), .normal)
+ }
+}
+
+/// The Mac's photo matcher, answering from a script (the last answer repeats); records what it was asked.
+public final class FakePlaces: PlaceLocating, @unchecked Sendable {
+ public var answers: [Result]
+ public private(set) var asked: [(scan: String, room: String?)] = []
+ public init(_ answers: [Result]) { self.answers = answers }
+ public func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix {
+ asked.append((scan, room))
+ guard !answers.isEmpty else { throw RecallError.server("no answer") }
+ return try (answers.count > 1 ? answers.removeFirst() : answers[0]).get()
}
}
@@ -200,6 +231,8 @@ public final class FakeEpisodeFiles: EpisodeFiles, @unchecked Sendable {
public func directory(scan: String) -> URL { URL(fileURLWithPath: "/tmp/scan-\(scan)") }
public func manifest(scan: String) -> Manifest? { manifests[scan] }
public func roomMap(scan: String) -> Data? { maps[scan] }
+ public private(set) var grown: [String: Data] = [:]
+ public func saveGrownRoomMap(scan: String, _ data: Data) { grown[scan] = data; maps[scan] = data }
public func delete(scan: String) { deleted.append(scan) }
}
diff --git a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift
new file mode 100644
index 0000000..1121e30
--- /dev/null
+++ b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift
@@ -0,0 +1,86 @@
+import Foundation
+import RecallDomain
+import RecallTestSupport
+import Testing
+@testable import RecallApplication
+
+private let mapPose = Pose(columnMajor: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 1.4, -1, 1])
+
+/// A locator that hasn't recognised the room, whose camera moves a little between photos (a live session).
+@MainActor private func limitedLocator() -> FakeLocator {
+ let l = FakeLocator(readings: [(.identity, .limited)])
+ l.moves = true
+ return l
+}
+
+@MainActor private func assist(_ locator: FakeLocator, _ places: FakePlaces, _ speech: FakeSpeech, _ clock: FakeClock,
+ room: String? = "Living room") -> PhotoAssist {
+ var s = PhotoAssist.Settings()
+ s.maxAsks = 6
+ return PhotoAssist(home: "flat", scan: "s1", room: room, locator: locator, places: places, speech: speech, clock: clock,
+ settings: s)
+}
+
+@MainActor @Test func whenARKitMatchesFirstNoPhotoIsSent() async {
+ let locator = FakeLocator(readings: [(.identity, .normal)])
+ let places = FakePlaces([.success(PlaceFix(room: nil, pose: nil, inliers: 0, hint: []))])
+ let a = assist(locator, places, FakeSpeech(), FakeClock())
+ await a.run()
+ #expect(places.asked.isEmpty && !a.fixed)
+}
+
+@MainActor @Test func twoAgreeingFixesLocateThePhoneAtOnce() async {
+ let locator = limitedLocator(), clock = FakeClock()
+ let places = FakePlaces([.success(PlaceFix(room: "Living room", pose: mapPose, inliers: 80, hint: ["tv"]))])
+ let speech = FakeSpeech()
+ let a = assist(locator, places, speech, clock)
+ await a.run()
+ #expect(a.fixed && places.asked.count == 2 && places.asked[0].scan == "s1")
+ #expect(locator.correction != nil)
+ #expect(locator.reading()?.tracking == .normal) // the guide can plan now
+ #expect(speech.lines == ["Point the phone slowly at the TV."])
+}
+
+@MainActor @Test func inAnotherRoomItSaysSoOnceAndGivesNoFix() async {
+ let locator = limitedLocator(), clock = FakeClock()
+ let places = FakePlaces([.success(PlaceFix(room: "Bedroom", pose: nil, inliers: 0, hint: ["bed"]))])
+ let speech = FakeSpeech()
+ let a = assist(locator, places, speech, clock)
+ await a.run()
+ #expect(!a.fixed && locator.corrections.isEmpty && places.asked.count == 6)
+ #expect(speech.lines == ["You seem to be in Bedroom, not Living room. Walk to Living room and I'll keep looking."])
+}
+
+@MainActor @Test func withoutTheMacNothingChanges() async {
+ let locator = limitedLocator(), clock = FakeClock()
+ let speech = FakeSpeech()
+ let a = assist(locator, FakePlaces([.failure(RecallError.status(503))]), speech, clock)
+ await a.run()
+ #expect(!a.fixed && locator.corrections.isEmpty && speech.lines.isEmpty)
+}
+
+@MainActor @Test func fixesThatDisagreeAreNotUsed() async {
+ let locator = limitedLocator(), clock = FakeClock()
+ let elsewhere = Pose(columnMajor: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, -2, 1.4, 3, 1])
+ let fix = { (p: Pose) in Result.success(PlaceFix(room: nil, pose: p, inliers: 40, hint: [])) }
+ let places = FakePlaces([fix(mapPose), fix(elsewhere), fix(mapPose), fix(elsewhere), fix(mapPose), fix(elsewhere)])
+ let a = assist(locator, places, FakeSpeech(), clock)
+ await a.run()
+ #expect(!a.fixed && locator.corrections.isEmpty)
+}
+
+@MainActor @Test func theGuideIsLocatedByPhotosWhenARKitIsSlow() async {
+ let locator = limitedLocator(), clock = FakeClock()
+ let places = FakePlaces([.success(PlaceFix(room: "Office", pose: mapPose, inliers: 80, hint: []))])
+ let service = FakeGuide(routes: [Route(sessionId: "gs", landmarksOnly: true, reason: "no clear path",
+ steps: [.init(clock: 12, metres: 2, say: "Your keys are on the desk.")],
+ waypoints: [], arriveSay: "")])
+ var settings = GuideToThing.Settings()
+ settings.locateTimeout = 10
+ let g = GuideToThing(home: "flat", thing: Thing.sample(), roomMap: Data("map".utf8), locator: locator, guide: service,
+ speech: FakeSpeech(), haptics: FakeHaptics(), clock: clock, photos: (places: places, scan: "s1"),
+ settings: settings)
+ await g.run()
+ #expect(locator.correction != nil || !locator.corrections.isEmpty)
+ #expect(service.routeRequests.first?.tracking == .normal) // planned as located, not landmarks-only for weak tracking
+}
diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift
new file mode 100644
index 0000000..27e5e25
--- /dev/null
+++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift
@@ -0,0 +1,48 @@
+import Foundation
+import simd
+import Testing
+@testable import RecallDomain
+
+/// A pose at (x, y, z), turned `yaw` degrees about the vertical (ARKit y).
+private func pose(_ x: Double, _ y: Double, _ z: Double, yaw: Double = 0) -> Pose {
+ let a = yaw * .pi / 180
+ return Pose(columnMajor: [cos(a), 0, -sin(a), 0, 0, 1, 0, 0, sin(a), 0, cos(a), 0, x, y, z, 1])
+}
+
+@Test func aCorrectionTurnsSessionPosesIntoTheMapsWorld() {
+ let T = MapCorrection.matrix(pose(2, 0, -1, yaw: 30)) // the session's world inside the map's
+ let session = pose(0.5, 1.4, 0.2, yaw: -10)
+ let map = MapCorrection.apply(T, to: session)
+ let back = MapCorrection.between(map: map, session: session)
+ let d = MapCorrection.distance(back, T)
+ #expect(d.metres < 1e-9 && d.degrees < 1e-6)
+ // a later session pose lands where the same correction puts it
+ let later = MapCorrection.apply(back, to: pose(1.5, 1.4, 0.2, yaw: -10))
+ #expect(abs(simd_distance(SIMD2(later.columnMajor[12], later.columnMajor[14]),
+ SIMD2(map.columnMajor[12], map.columnMajor[14])) - 1) < 1e-9)
+}
+
+@Test func twoFixesMustAgreeAndComeFromALiveSession() {
+ var a = FixAgreement()
+ let T1 = MapCorrection.matrix(pose(2, 0, -1, yaw: 30)), near = MapCorrection.matrix(pose(2.1, 0, -1, yaw: 33))
+ #expect(a.add(T1, session: pose(0, 1.4, 0)) == nil) // one fix proves nothing
+ #expect(a.add(near, session: pose(0.05, 1.4, 0)) != nil) // 10 cm and 3° apart: agreed
+
+ var far = FixAgreement()
+ _ = far.add(T1, session: pose(0, 1.4, 0))
+ #expect(far.add(MapCorrection.matrix(pose(3, 0, -1, yaw: 30)), session: pose(0.05, 1.4, 0)) == nil) // 1 m apart
+ #expect(far.add(MapCorrection.matrix(pose(3, 0, -1, yaw: 31)), session: pose(0.1, 1.4, 0)) != nil) // then two agree
+
+ var frozen = FixAgreement()
+ _ = frozen.add(T1, session: pose(0, 1.4, 0))
+ #expect(frozen.add(T1, session: pose(0, 1.4, 0)) == nil) // the same session pose twice: ARKit isn't tracking
+}
+
+@Test func coachingNamesThingsToPointAt() {
+ #expect(LocateCoaching.point(at: ["tv", "cabinet"], turn: 0) == "Point the phone slowly at the TV.")
+ #expect(LocateCoaching.point(at: ["tv", "cabinet"], turn: 1) == "Now at the cabinet.")
+ #expect(LocateCoaching.point(at: ["tv", "cabinet"], turn: 2) == "Now at the TV.")
+ #expect(LocateCoaching.point(at: [], turn: 0) == nil)
+ #expect(LocateCoaching.elsewhere(found: "Bedroom", wanted: "Living room")
+ == "You seem to be in Bedroom, not Living room. Walk to Living room and I'll keep looking.")
+}
diff --git a/apps/ios/Recall/App/AppContainer.swift b/apps/ios/Recall/App/AppContainer.swift
index c26ec85..ee5e568 100644
--- a/apps/ios/Recall/App/AppContainer.swift
+++ b/apps/ios/Recall/App/AppContainer.swift
@@ -67,7 +67,8 @@ final class AppContainer {
idle?.cancel()
let scans = scans
return ARLabelsModel(room: room, home: settings.connection.homeId, source: arLabelsSource(), locator: locator,
- roomMap: { scans.roomMap(scan: $0) }, speech: speech)
+ roomMap: { scans.roomMap(scan: $0) }, saveGrownMap: { scans.saveGrownMap(scan: $0, $1) },
+ places: UITestSamples.enabled ? nil : server, speech: speech, haptics: haptics, clock: clock)
}
func arLabelsSource() -> any ArLabelsSource {
@@ -93,13 +94,14 @@ final class AppContainer {
func findModel() -> FindModel {
FindModel(ask: AskWhereIs(guide: UITestSamples.enabled ? UITestSamples.Guide() : server), transcriber: server, voice: WavRecorder(), things: things,
settings: settings, speech: speech, haptics: haptics,
- prepareGuide: { [weak self] in self?.prepareGuide() })
+ prepareGuide: { [weak self] thing in self?.prepareGuide(room: thing.room) })
}
- /// Starts recognising the saved room in the background; paused again if no guide starts within a minute.
- func prepareGuide() {
- guard ARLocator.isSupported, !guiding, let map = scans.roomMap() else { return }
- locator.prepare(roomMap: map)
+ /// Starts recognising the thing's room in the background; paused again if no guide starts within a minute.
+ func prepareGuide(room: String?) {
+ guard ARLocator.isSupported, !guiding, let map = scans.guideMap(room: room) else { return }
+ growMap(scan: map.scan)
+ locator.prepare(roomMap: map.data)
idle?.cancel()
idle = Task { [weak self] in
try? await Task.sleep(for: .seconds(60))
@@ -111,12 +113,21 @@ final class AppContainer {
func guideModel(for thing: Thing) -> GuideModel {
idle?.cancel()
guiding = true
- let use = GuideToThing(home: settings.connection.homeId, thing: thing, roomMap: scans.roomMap(), locator: locator,
- guide: server, speech: speech, haptics: haptics, clock: clock, compass: compass)
+ let map = scans.guideMap(room: thing.room)
+ if let map { growMap(scan: map.scan) }
+ let use = GuideToThing(home: settings.connection.homeId, thing: thing, roomMap: map?.data, locator: locator,
+ guide: server, speech: speech, haptics: haptics, clock: clock, compass: compass,
+ photos: map.map { (places: server, scan: $0.scan) })
return GuideModel(guide: use, session: locator.session, scene: locator.scene, supported: ARLocator.isSupported,
ended: { [weak self] in self?.guiding = false })
}
+ /// When ARKit recognises this scan's room, the map it grew is kept for next time.
+ private func growMap(scan: String) {
+ let scans = scans
+ locator.onMapGrown = { data in scans.saveGrownMap(scan: scan, data) }
+ }
+
/// `again`: a processed scan of this room to scan again in the same frame (its room map is recognised first).
func scanModel(room: String, again: Scan? = nil) -> ScanModel {
idle?.cancel()
@@ -135,7 +146,7 @@ struct SystemClock: Clock {
/// The Mac, as every Mac-facing port, reached with the current connection settings. Changing Settings swaps the
/// client underneath; a call made before setup throws the setup issue, in words.
-final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, ConnectionCheck, ScanServer, RobotSharing,
+final class LiveServer: HomeMemory, GuideService, PlaceLocating, Transcriber, ScanProcessing, ConnectionCheck, ScanServer, RobotSharing,
RoomMaps, RobotPackages, CompassRoutes, ArLabelsSource, TrustSettings, PrivacySource, EvidenceSource, @unchecked Sendable {
private let lock = NSLock()
private var api: RecallAPI?
@@ -179,6 +190,9 @@ final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, C
func arLabels(home: String, room: String) async throws -> ArLabels {
try await current().arLabels(home: home, room: room)
}
+ func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix {
+ try await current().locate(home: home, scan: scan, room: room, frame: frame)
+ }
func compassRoute(home: String, thing: String) async throws -> CompassRoute {
try await current().compassRoute(home: home, thing: thing)
}
diff --git a/apps/ios/Recall/Features/AR/ARLabelsModel.swift b/apps/ios/Recall/Features/AR/ARLabelsModel.swift
index da326b2..b243466 100644
--- a/apps/ios/Recall/Features/AR/ARLabelsModel.swift
+++ b/apps/ios/Recall/Features/AR/ARLabelsModel.swift
@@ -1,6 +1,7 @@
import ARKit
import Foundation
import Observation
+import RecallApplication
import RecallCapture
import RecallDomain
import SwiftUI
@@ -20,13 +21,26 @@ final class ARLabelsModel {
private let source: any ArLabelsSource
private let home: String
private let roomMap: (String) -> Data?
+ private let saveGrownMap: (String, Data) -> Void
+ private let places: (any PlaceLocating)?
private let speech: any SpeechOutput
+ private let haptics: (any HapticsOutput)?
+ private let clock: any Clock
private var task: Task?
+ private var assist: PhotoAssist?
+ /// Seconds before the list is shown instead, and how long the room is still looked for after that.
+ static let listAfter = 25.0
+ static let keepLooking = 120.0
+
+ /// `places`: the Mac's photo matcher, to recognise the room alongside ARKit; `saveGrownMap`: keeps the map ARKit
+ /// grew on this visit, so the next one recognises the room from here too.
init(room: String, home: String, source: any ArLabelsSource, locator: ARLocator, roomMap: @escaping (String) -> Data?,
- speech: any SpeechOutput) {
+ saveGrownMap: @escaping (String, Data) -> Void = { _, _ in }, places: (any PlaceLocating)? = nil,
+ speech: any SpeechOutput, haptics: (any HapticsOutput)? = nil, clock: any Clock = SystemClock()) {
self.room = room; self.home = home; self.source = source; self.locator = locator; self.roomMap = roomMap
- self.speech = speech
+ self.saveGrownMap = saveGrownMap; self.places = places; self.speech = speech; self.haptics = haptics
+ self.clock = clock
}
func start() {
@@ -35,6 +49,8 @@ final class ARLabelsModel {
func stop() {
task?.cancel()
+ assist?.cancel()
+ locator.onMapGrown = nil
locator.clearLabels()
locator.stop()
}
@@ -57,20 +73,40 @@ final class ARLabelsModel {
}
phase = .locating
speech.say("Point the phone around \(room) so I can recognise it.", interrupt: true)
+ let scan = found.scanId
+ locator.onMapGrown = { [saveGrownMap] data in saveGrownMap(scan, data) }
locator.start(roomMap: map)
- // ARKit says the room is recognised when tracking turns normal in the saved map
- for _ in 0..<50 {
- try await Task.sleep(for: .milliseconds(500))
- if locator.reading()?.tracking == .normal { break }
+ // the Mac's photo matcher helps alongside ARKit (named furniture to point at, a fix from a photo)
+ if let places {
+ let a = PhotoAssist(home: home, scan: scan, room: room, locator: locator, places: places, speech: speech,
+ clock: clock)
+ assist = a
+ Task { await a.run() }
}
- guard locator.reading()?.tracking == .normal else {
- return phase = .listOnly("I couldn't recognise \(room) from here, so here is the list.")
+ // recognised when tracking turns normal: ARKit matched the saved map, or two photo fixes agreed
+ let started = clock.now()
+ var late = false
+ while locator.reading()?.tracking != .normal {
+ let waited = clock.now().timeIntervalSince(started)
+ if !late, waited >= Self.listAfter {
+ late = true // the list now, but keep looking: the labels come on if the room is recognised
+ phase = .listOnly("I couldn't recognise \(room) from here yet, so here is the list. Still looking.")
+ }
+ if waited >= Self.listAfter + Self.keepLooking {
+ assist?.cancel()
+ return phase = .listOnly("I couldn't recognise \(room) from here, so here is the list.")
+ }
+ try await Task.sleep(for: .milliseconds(250))
}
+ assist?.cancel()
+ haptics?.tap()
let tags = labels.map { (label: $0, tag: Self.tagImage($0)) }
locator.showLabels(tags, floorY: found.floorY)
phase = .showing
let n = labels.count == 1 ? "1 thing" : "\(labels.count) things"
- speech.say("\(n) in \(room). Point at a label and double tap with two fingers to hear it.", interrupt: false)
+ let lead = late ? "Recognised \(room). " : ""
+ speech.say("\(lead)\(n) in \(room). Point at a label and double tap with two fingers to hear it.",
+ interrupt: late)
} catch is CancellationError {
} catch {
phase = .failed(UserMessage.explain(error))
diff --git a/apps/ios/Recall/Features/Find/FindModel.swift b/apps/ios/Recall/Features/Find/FindModel.swift
index 521a891..7d9eb82 100644
--- a/apps/ios/Recall/Features/Find/FindModel.swift
+++ b/apps/ios/Recall/Features/Find/FindModel.swift
@@ -24,10 +24,10 @@ final class FindModel {
private let speech: any SpeechOutput
private let haptics: any HapticsOutput
/// Starts recognising the room while the answer is read out, so "Take me there" begins at once.
- private let prepareGuide: @MainActor () -> Void
+ private let prepareGuide: @MainActor (Thing) -> Void
init(ask: AskWhereIs, transcriber: any Transcriber, voice: any VoiceInput, things: ThingsModel, settings: SettingsModel,
- speech: any SpeechOutput, haptics: any HapticsOutput, prepareGuide: @escaping @MainActor () -> Void = {}) {
+ speech: any SpeechOutput, haptics: any HapticsOutput, prepareGuide: @escaping @MainActor (Thing) -> Void = { _ in }) {
self.ask = ask; self.transcriber = transcriber; self.voice = voice; self.things = things
self.settings = settings; self.speech = speech; self.haptics = haptics; self.prepareGuide = prepareGuide
}
@@ -72,7 +72,7 @@ final class FindModel {
about: lastAnswer?.answer.thingId)
lastAnswer = result
state = .answered(result)
- if result.thing != nil { prepareGuide() }
+ if let thing = result.thing { prepareGuide(thing) } // its room's map starts recognising now
speech.say(result.answer.text + (result.thing == nil ? "" : " Tap Take me there for directions."), interrupt: true)
} catch {
state = .failed(UserMessage.explain(error))
diff --git a/apps/ios/Recall/Models/ScansModel.swift b/apps/ios/Recall/Models/ScansModel.swift
index 65d5954..2eeb20d 100644
--- a/apps/ios/Recall/Models/ScansModel.swift
+++ b/apps/ios/Recall/Models/ScansModel.swift
@@ -106,14 +106,21 @@ final class ScansModel {
return await Task.detached { archiver.archive(episode: dir) }.value
}
- /// The saved room map to relocalise in: the home's first processed scan made on this phone (routes are planned
- /// in that scan's world).
- /// The room map guidance starts from: the newest processed scan's (a recognised rescan's map holds the first
- /// scan's world, extended).
- func roomMap() -> Data? {
- scans.filter(\.isProcessed).sorted { $0.createdAt > $1.createdAt }.lazy.compactMap { self.files.roomMap(scan: $0.id) }.first
+ /// The room map to guide to a thing in `room`: that room's newest processed scan with a map on this phone (the
+ /// Mac plans in that scan's world: a scan that wasn't recognised started the room's memory over). Without a room,
+ /// or no map of it, the newest processed scan's.
+ func guideMap(room: String?) -> (scan: String, data: Data)? {
+ let ready = scans.filter(\.isProcessed).sorted { $0.createdAt > $1.createdAt }
+ let inRoom = room.map { r in ready.filter { $0.room == r } } ?? []
+ for s in inRoom + ready {
+ if let data = files.roomMap(scan: s.id) { return (s.id, data) }
+ }
+ return nil
}
+ /// Keeps the room map ARKit grew on a later visit (the next visit recognises the room from there too).
+ func saveGrownMap(scan: String, _ data: Data) { files.saveGrownRoomMap(scan: scan, data) }
+
/// One scan's saved room map (the AR labels are placed in it).
func roomMap(scan id: String) -> Data? { files.roomMap(scan: id) }
From 5e427c644149898b3b0a6031326b61829b2c5e18 Mon Sep 17 00:00:00 2001
From: Rachit
Date: Mon, 2 Nov 2026 12:16:30 +0100
Subject: [PATCH 05/30] docs: how the room is recognised from anywhere, with
measured results
Co-Authored-By: Claude Opus 5.5
---
CLAUDE.md | 3 +-
README.md | 14 +++-
apps/web/i18n/dictionaries/en.ts | 2 +-
docs/privacy.md | 3 +-
evals/locate_photo.py | 113 +++++++++++++++++++++++++++++++
evals/results.md | 40 +++++++++++
6 files changed, 171 insertions(+), 4 deletions(-)
create mode 100644 evals/locate_photo.py
create mode 100644 evals/results.md
diff --git a/CLAUDE.md b/CLAUDE.md
index 75f96ec..5d2b037 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -106,7 +106,8 @@ domain ← application ← adapters (inbound + outbound)
- **API `domain/ports.py`** (one repository per aggregate): `HomeRepository`, `ScanRepository`, `TrackedObjectRepository`.
- **API `application/ports.py`** (needs of use cases): `LlmProvider`, `VisionProvider` (Muse Spark 1.1, tie-break only), `Transcriber`, `BlobStore`,
`PipelineTrigger`, `Clock`, `Segmenter` (SAM 3.1 over HTTP on the Meta API, for the Verifier), `ArrivalVerifier`,
- `PromptScreener` (Muse Spark Open or Glimmer, after schema validation).
+ `PromptScreener` (Muse Spark Open or Glimmer, after schema validation), `PlaceMatcher` (classic ORB + PnP photo
+ matching against each scan's `place_index.npz`, no model; `POST /homes/{id}/locate`, ADR-039).
- **Pipeline `application/ports.py`**: `Reconstructor`, `Segmenter`, `EpisodeSource` (reads from R2 or disk), `ResultSink` (posts to the API).
**torch and local models live only in the pipeline**; the API reaches vision models over HTTP only.
diff --git a/README.md b/README.md
index 4cbf5aa..0f60997 100644
--- a/README.md
+++ b/README.md
@@ -429,6 +429,7 @@ routes match it. While `make demo` or `make live` runs, **Swagger UI** is at `ht
| `DELETE /scans/{scan_id}` | Erase a scan from your computer, with what only it found |
| `GET /homes/{home_id}/objects`, `GET /homes/{home_id}/room`, `GET /homes/{home_id}/room-map` | What Recall found, the room model, and each room's map for Find |
| `POST /guidance/sessions`, `POST /guidance/sessions/{session_id}/verify` | Directions, and the arrival check |
+| `POST /homes/{home_id}/locate` | Where am I? One camera photo matched against the room scans: the room, the pose, what to point at |
| `GET /privacy`, `GET /audit` | What is sent where right now; who did what |
| `GET /homes/{home_id}/robot-export`, `/a2a/home-memory` | For robots: the Nav2 map, and the Home Memory agent ([Agent Card](services/api/resources/agents/home-memory.card.json)) |
@@ -451,6 +452,15 @@ Every route except `/health` needs a key: the owner's device key, or a robot's o
4. *Not found is a fixed sentence,* never the model's: "I haven't seen that yet."
5. *Tested:* `tests/unit/test_grounding.py` (an invented place or number is caught) and `tests/unit/test_ask_where_is.py` (an invented place is corrected, then replaced by the tool facts; a place with no sighting behind it is never said).
- **Every answer is traceable.** Sightings are append-only. Every answer is logged with its model and tool steps and the sightings it rests on, and **Why?** on the phone (or `GET /answers/{answer_id}/evidence`, or asking the Guide "why?") leads back to the camera's view, the scan and the models that made it. An answer that names a place with no sighting behind it is never said.
+- **Recognising the room from anywhere, never on one photo's word.** ARKit's own match of the saved room map is
+ the main signal. While it hasn't matched, the phone sends about one photo a second to the Mac, which matches it
+ against every room's scan photos with classic ORB features, the scan's own LiDAR depth and PnP (OpenCV, no model;
+ `adapters/outbound/vision/orb_place_matcher.py`). The phone uses a photo pose only when **two fixes in a row agree**
+ within 25 cm and 10°, and ARKit's own match replaces it the moment it comes. The Mac also says which room the photo
+ shows ("You seem to be in Bedroom, not Living room") and what to point at ("the TV"). Each time ARKit recognises
+ the room, the map it grew is kept on the phone, so the next visit matches from there too. Measured on the real
+ living-room scans: held-out photos 1.2 cm median error; photos from other sessions fix 59% of the time, 89% of
+ them right; 22 to 35 ms per match ([evals/results.md](evals/results.md)).
- **The Verifier can only downgrade.** The vision model may turn a result into "uncertain"; only SAM plus geometry can confirm.
- **Two checks on the door.** A robot or outside app can send only one object name (letters, digits, spaces and hyphens, at most 40 characters, `services/api/resources/agents/find-object.input.schema.json`); free text is refused and never reaches a model or a tool. Then Muse Spark Open (or Glimmer) must answer `{"verdict": "safe", ...}` in strict JSON; anything else, a timeout or "injection" refuses. Measured: 21 ordinary names allowed, 18 instruction-like names refused, none wrong.
- **Privacy and security.** See [Security and privacy](#security-and-privacy): what stays on the Mac, what is sent where, and how the home's memory is protected.
@@ -553,7 +563,9 @@ erDiagram
- **Indexes:** scans, landmarks, objects and rejections by home; observations and set-asides by object; audit events
by time and by action; the answer log by home and time.
- **The job queue** is a separate SQLite file (`data/jobs.db`, Huey) that holds only queued scan jobs.
-- **Not in SQL:** the scan files (`data/blobs` on the Mac, or Cloudflare R2). On the iPhone there is no database:
+- **Not in SQL:** the scan files (`data/blobs` on the Mac, or Cloudflare R2), and each scan's `place_index.npz` beside
+ them (ORB descriptors of the keyframes and their 3D points, for the photo matcher; built by the pipeline, or
+ `make place-index`). On the iPhone there is no database:
scans are folders in Application Support, the device key is in the Keychain, and the Mac's address is in
UserDefaults.
diff --git a/apps/web/i18n/dictionaries/en.ts b/apps/web/i18n/dictionaries/en.ts
index 4348cd5..a6726ea 100644
--- a/apps/web/i18n/dictionaries/en.ts
+++ b/apps/web/i18n/dictionaries/en.ts
@@ -500,7 +500,7 @@ export const en = {
kept: [
"On your Mac: every scan you upload (photos, LiDAR depth, the phone's path through the room), the 3D map, the things and furniture found, your own names and notes for things, and the activity log.",
"On your iPhone: your scans until you delete them, the room map used to find your way, and the Mac's address. The device key is in the iPhone's Keychain.",
- "Between them: your home network, or an encrypted tunnel you set up yourself (Tailscale or Cloudflare Tunnel). Every request needs the device key.",
+ "Between them: your home network, or an encrypted tunnel you set up yourself (Tailscale or Cloudflare Tunnel). Every request needs the device key. While Recall finds the room, about one camera photo a second goes to your Mac (only there), to be matched against your scans; it is not stored.",
],
sentTitle: "What leaves your Mac, to whom, and when",
sentLabels: { to: "To", when: "When" },
diff --git a/docs/privacy.md b/docs/privacy.md
index 8960c3c..f954921 100644
--- a/docs/privacy.md
+++ b/docs/privacy.md
@@ -15,7 +15,8 @@ nobody else holds your scans or your memory. Some steps use **Meta's models onli
- **On your iPhone**: your scans until you delete them, the room map used to find your way, and the Mac's address.
The device key is in the iPhone's Keychain.
- Between them: your home network, or an encrypted tunnel you set up yourself (Tailscale or Cloudflare Tunnel). Every
- request needs the device key.
+ request needs the device key. While Recall finds the room, about one camera photo a second goes to your Mac (only
+ there), to be matched against your scans; it is not stored.
## What leaves your Mac, to whom, and when
diff --git a/evals/locate_photo.py b/evals/locate_photo.py
new file mode 100644
index 0000000..5885bb8
--- /dev/null
+++ b/evals/locate_photo.py
@@ -0,0 +1,113 @@
+"""Replays real scans through the photo matcher (evals/results.md, "Recognising the room from a photo").
+
+ cd services/api && uv run python ../../evals/locate_photo.py REFERENCE_SCAN_ID [OTHER_SCAN_ID ...]
+
+Needs the scans on this Mac (data/blobs/scans) with their place indexes (`make place-index`). Prints the held-out
+accuracy on the reference scan, then for each other scan the fix rate and the share of fixes that agree with the
+consensus scan-to-scan transform.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+import numpy as np
+
+from recall_api.adapters.outbound.vision.orb_place_matcher import OrbPlaceMatcher
+from recall_api.application.ports import PlaceQuery
+
+SCANS = Path(__file__).resolve().parents[1] / "data" / "blobs" / "scans"
+
+
+class Disk:
+ def local_path(self, key):
+ return key
+
+
+def episode(prefix: str) -> Path:
+ return next(SCANS.glob(f"*_{prefix}*/episode"))
+
+
+def poses(ep: Path) -> dict[int, np.ndarray]:
+ out = {}
+ for line in (ep / "poses.jsonl").read_text().splitlines():
+ p = json.loads(line)
+ out[p["index"]] = np.array(p["pose"], float).reshape(4, 4).T
+ return out
+
+
+def query(ep: Path, i: int) -> PlaceQuery:
+ k = json.loads((ep / "manifest.json").read_text())["intrinsics"]
+ return PlaceQuery((ep / "frames" / f"{i:06d}.jpg").read_bytes(), k["fx"], k["fy"], k["cx"], k["cy"])
+
+
+def angle(a: np.ndarray, b: np.ndarray) -> float:
+ d = a[:3, :3].T @ b[:3, :3]
+ return float(np.degrees(np.arccos(np.clip((np.trace(d) - 1) / 2, -1, 1))))
+
+
+def fixed(r) -> bool:
+ return bool(r) and r[0].pose is not None and r[0].inliers >= 30 and r[0].reproj_px <= 4
+
+
+def held_out(matcher: OrbPlaceMatcher, ep: Path, out: Path) -> None:
+ z = dict(np.load(ep / "place_index.npz"))
+ ids = z["frame_ids"]
+ rows = np.arange(len(ids))[1::2] # the index keeps every other keyframe; the rest are the photos
+ sel = np.isin(z["frame"], rows)
+ remap = {r: k for k, r in enumerate(rows)}
+ sub = {**z, "desc": z["desc"][sel], "xyz": z["xyz"][sel], "frame": np.array([remap[f] for f in z["frame"][sel]]),
+ "frame_ids": ids[rows], "poses": z["poses"][rows]} # fmt: skip
+ np.savez(out / "held.npz", **sub)
+ truth = poses(ep)
+ errs, angs, times = [], [], []
+ photos = [int(ids[r]) for r in range(0, len(ids), 2)]
+ for i in photos:
+ t = time.perf_counter()
+ r = matcher.match(query(ep, i), {"held": str(out / "held.npz")})
+ times.append(time.perf_counter() - t)
+ if fixed(r):
+ P = np.array(r[0].pose).reshape(4, 4).T
+ errs.append(np.linalg.norm(P[:3, 3] - truth[i][:3, 3]))
+ angs.append(angle(P, truth[i]))
+ print(f"held-out: {len(errs)}/{len(photos)} fixes; {np.median(errs) * 100:.1f} cm median, "
+ f"{np.percentile(errs, 90) * 100:.1f} cm p90; {np.median(angs):.2f} deg median; "
+ f"{np.median(times) * 1000:.0f} ms median") # fmt: skip
+
+
+def other_session(matcher: OrbPlaceMatcher, ref: Path, other: Path) -> None:
+ truth = poses(other)
+ photos = [i for i in sorted(truth)[::3] if (other / "frames" / f"{i:06d}.jpg").exists()]
+ Ts = []
+ for i in photos:
+ r = matcher.match(query(other, i), {"ref": str(ref / "place_index.npz")})
+ if fixed(r):
+ Ts.append(np.array(r[0].pose).reshape(4, 4).T @ np.linalg.inv(truth[i]))
+
+ def close(a, b):
+ return np.linalg.norm(a[:3, 3] - b[:3, 3]) < 0.25 and angle(a, b) < 10
+
+ if not Ts:
+ print(f"{other.parent.name[:40]}: 0/{len(photos)} fixes")
+ return
+ best = Ts[int(np.argmax([sum(close(a, b) for b in Ts) for a in Ts]))]
+ right = sum(close(best, t) for t in Ts)
+ print(f"{other.parent.name[:40]}: {len(Ts)}/{len(photos)} fixes, {right} right")
+
+
+def main(argv: list[str]) -> int:
+ matcher = OrbPlaceMatcher(Disk())
+ ref = episode(argv[0])
+ with tempfile.TemporaryDirectory() as d:
+ held_out(matcher, ref, Path(d))
+ for o in argv[1:]:
+ other_session(matcher, ref, episode(o))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/evals/results.md b/evals/results.md
new file mode 100644
index 0000000..0db1b90
--- /dev/null
+++ b/evals/results.md
@@ -0,0 +1,40 @@
+# Evaluation results
+
+Dated measurements on real scans. Each row names what was run, so it can be run again.
+
+## Recognising the room from a photo (2026-11-02)
+
+The photo matcher behind `POST /homes/{home_id}/locate`: ORB features of one camera photo matched against a scan's
+place index (every keyframe's ORB features lifted to 3D with the phone's own LiDAR depth and ARKit pose), then PnP
+with RANSAC. Classic OpenCV, no model. Data: the author's living room, six scans made Oct 30 to Nov 2 with the
+iPhone 16 Pro; the reference is the published scan (`bb449f4c`, 162 keyframes).
+
+| Test | Photos | Fix (≥ 30 inliers, ≤ 4 px) | Right | Accuracy | Time on the Mac (M5 Pro) |
+|---|---|---|---|---|---|
+| Held-out keyframes of the same scan (index from the other half) | 81 | 100% | 100% | 1.2 cm median, 4.1 cm p90; 0.55° median | 22 ms median, 25 ms p90 |
+| Photos from four other sessions (Nov 1 to Nov 2) | 242 | 62% | 90% | — | 33 to 35 ms median |
+| Photos from the oldest session (Oct 30; furniture since moved) | 19 | 26% | 60% | — | 34 ms median |
+
+"Right" for other sessions: their ARKit worlds differ from the reference, so a fix is right when the scan-to-scan
+transform it implies agrees (within 25 cm and 10°) with the one most fixes agree on.
+
+Threshold sweep over all 261 photos from other sessions (fix rate, then share right):
+
+| Inliers ≥ | 15 | 20 | 30 | 40 | 50 | 70 | 100 |
+|---|---|---|---|---|---|---|---|
+| Fix rate | 66% | 63% | 59% | 53% | 49% | 38% | 30% |
+| Right | 87% | 89% | 89% | 94% | 95% | 98% | 97% |
+
+**What the phone does with it:** a photo pose is used only when two fixes in a row agree within 25 cm and 10°
+(`FixAgreement`), and ARKit's own match replaces it as soon as it comes. With about 6 in 10 photos giving a fix and
+one photo a second, two agreeing fixes typically take 2 to 4 s.
+
+**Found on the way:** none of the five rescans of this room was recognised by ARKit at the start of the rescan (all
+are marked `fresh` in the database), which is the problem this feature addresses.
+
+To run it again: `make place-index`, then
+`cd services/api && uv run python ../../evals/locate_photo.py bb449f4c 926c7d73 56c966eb 63bdddb5 236dc98e 7a736bf2`.
+
+**Still to measure on the iPhone:** time from opening "See it in AR" to the labels, from five start spots (doorway,
+corner, sofa, facing a wall, hallway), before and after; and whether a grown room map matches from a spot that failed
+before.
From c73238d00db10ff116d61073f797ea45dd635a9f Mon Sep 17 00:00:00 2001
From: Rachit
Date: Mon, 2 Nov 2026 12:41:54 +0100
Subject: [PATCH 06/30] docs(readme): recognising the room in how it works,
algorithms and results; today's test counts
Co-Authored-By: Claude Opus 5.5
---
README.md | 24 +++++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 0f60997..16aaf11 100644
--- a/README.md
+++ b/README.md
@@ -192,7 +192,7 @@ services/
api/ FastAPI: domain/ → application/ → adapters/ + infra/ ; resources/ = the OpenAPI spec (source of truth)
pipeline/ the scan pipeline, same layers: MapAnything, SAM 3.1, planes, merging, the room model
docs/ architecture, privacy, verification, Meta Muse (docs/muse.md)
-evals/ questions, usability plan
+evals/ questions, usability plan, results and the replay scripts
```
- **Hexagonal (ports and adapters) in every service**: domain ← application ← adapters and infra.
@@ -378,14 +378,18 @@ runs its software or its models; every model Recall itself calls is a Meta model
1. The phone sends a spoken question to the Mac, and Muse Voice Transcribe turns it into text (typed questions skip this).
2. The phone says what it is looking for at once ("Looking for your keys.").
3. The Guide calls `find_object`, which returns each match with its room, landmark, relation ("on the desk"), shelf level, height band and when it was seen; one call usually answers. The answer is logged with the sightings behind it, and **Why?** reads them out.
-4. **Take me there**: `localize` from the phone's ARKit pose, `plan_route` over an occupancy grid, clock-face steps spoken as you walk.
+4. **Take me there**: the phone recognises the room (ARKit's match of the saved room map, helped by a photo match on
+ the Mac about once a second, trusted only when two fixes agree, and spoken hints such as "point the phone at the
+ TV"; landmark directions if neither matches), then `localize` from the phone's pose, `plan_route` over an
+ occupancy grid, clock-face steps spoken as you walk.
5. On arrival the Verifier checks the camera frame; if the thing is nearby, memory moves it.
**OS and browser features (not model calls)**
| Feature | Used for | Where |
|---|---|---|
-| LiDAR depth, ARKit tracking, ARWorldMap | Capture in metres; relocalisation for live guidance | iPhone |
+| LiDAR depth, ARKit tracking, ARWorldMap | Capture in metres; relocalisation for live guidance; the map grows on each visit | iPhone |
+| OpenCV ORB features + PnP (classic geometry, not a model) | Recognising the room from one camera photo, beside ARKit | Mac (`services/api/.../vision/orb_place_matcher.py`) |
| ARKit poses, compass (magnetic north, no location) | Speed and turn rate for scan coaching come from the ARKit poses; the scan learns where north lies in the room, so directions can say "north-east" | iPhone |
| Haptics | Turn and arrival cues | iPhone (Apple Watch planned) |
| VoiceOver, Dynamic Type | The app is usable without looking | iPhone |
@@ -580,6 +584,10 @@ erDiagram
- **Spatial relations:** "on" (within 5 cm of a surface and inside its footprint), left or right of a landmark's front, nearest landmark within 1.5 m, and five height bands (floor, knee, waist, chest, head).
- **What changed:** a rescan matches each kind of thing with the room's last scan by optimal assignment on 3D distance; moved more than 30 cm, new, or gone is decided by geometry, and things outside the rescan's view are left alone.
- **Navigation:** an occupancy grid at 10 cm from the point cloud, obstacles widened by a 0.3 m body radius and paths kept to the middle of free space, A* to a reachable cell near the object, simplified to turns and spoken as clock-face steps. A gap too narrow gives "no safe path" and landmark directions instead.
+- **Recognising the room:** after processing, each scan's keyframes become ORB features lifted to 3D with the phone's
+ LiDAR depth and ARKit pose (`place_index.npz`). One camera photo is matched against them (ratio test, PnP with
+ RANSAC; at least 30 inliers within 4 px), giving the room and the camera's pose in the room map's world. The phone
+ trusts two fixes in a row that agree within 25 cm and 10°, and ARKit's own match replaces them.
- **Arrival check:** SAM 3.1's box in one camera frame, lifted to 3D with the LiDAR depth; within 30 cm of the remembered place is confirmed.
- **Viewer:** voxel downsampling to about 150k points; a vertex shader that gathers every point from a scattered shell to its real position, floor first.
@@ -593,14 +601,18 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima
- **Without a model:** LiDAR depth back-projected with the ARKit poses put all 10 objects of the living room within **2 to 7 cm** of the MapAnything run, in **0.2 s**.
- **Answer time:** the Guide on Muse Spark Open answers in **2.3 s** median, **2.7 s** p90 (10 real questions, one tool call each); the phone speaks an acknowledgement at once.
- **Arrival check:** on the real keys, SAM 3.1 plus the LiDAR depth put them **3 cm** from the remembered place, in **4.9 s**.
+- **Recognising the room from a photo** ([evals/results.md](evals/results.md)): held-out photos of the living room
+ came out **1.2 cm** (median) and **0.55°** from ARKit's own poses; photos from other sessions gave a fix **59 %** of
+ the time and **89 %** of those fixes were right, so the phone waits for two that agree; **22 to 35 ms** per match on
+ the M5 Pro. On the way: none of five rescans of that room was recognised by ARKit alone, the problem this fixes.
- **Robot screening:** **21** ordinary object names allowed and **18** instruction-like names refused, none wrong.
- **Cost, measured:** a room scan is **456** SAM 3.1 calls and **48 MB** on disk; a question is **2** Muse Spark Open calls, about **2,460 + 261 tokens**.
- **Flat surfaces:** the geometry-only detector found **12 level patches** in the room; the two largest sit at the heights of the two measured desks.
- **Viewer:** about 150k points, the gather animation, photo and confidence colouring, and the floor grid, height bands and surfaces as layers, on a phone and a laptop.
-- **Tests (2 November):** API 202 (including Schemathesis against the spec), pipeline 59 (including a synthetic room run through the whole pipeline), website 190 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, keyboard-only and 200 % text usability, security headers); on the iPhone, 182 unit tests and 31 UI tests: accessibility audits and usability tasks done with VoiceOver labels at default and largest text; the Watch app builds.
+- **Tests (2 November):** API 213 (including Schemathesis against the spec), pipeline 64 (including a synthetic room run through the whole pipeline), website 194 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, keyboard-only and 200 % text usability, security headers); on the iPhone, 197 unit tests and 34 UI tests (2 skipped): accessibility audits and usability tasks done with VoiceOver labels at default and largest text; the Watch app builds.
- **Usability, honestly:** the checks above are automated. In person, the author's screen-off run (VoiceOver Screen Curtain: the screen stays black) is planned, and a session with blind and low-vision testers is planned; its plan is ready in `evals/usability/`.
-**Planned evaluation:** median position error against hand-measured positions, questions answered correctly, Verifier precision, and time to find an object by touch versus with Recall, scored without AI; the dated table will live in `evals/results.md`. The session plan for a blind or low-vision tester is in `evals/usability/`.
+**Planned evaluation:** median position error against hand-measured positions, questions answered correctly, Verifier precision, and time to find an object by touch versus with Recall, scored without AI. The dated table is [`evals/results.md`](evals/results.md); recognising the room from a photo is its first entry. The session plan for a blind or low-vision tester is in `evals/usability/`.
## Installation
@@ -726,6 +738,8 @@ Planned next, not built or not working yet (updated before submission; removed i
- A physical robot running the ROS 2 example (the example and the robot package are in the repo).
- The tape-measured 20 cm accuracy check with the phone's depth and poses.
- Quantitative evaluations and the Verifier precision figure.
+- Timing room recognition on the iPhone from five start spots, and whether a grown room map matches from a spot that
+ failed before (the photo matcher is measured on the Mac only so far).
- The usability session with blind and low-vision testers (plan ready in `evals/usability/`), and the author's screen-off run.
- Daily spend caps on model calls (the API and the website's voice route are rate-limited per key or visitor; there is no daily cap yet).
- The 3-minute demo video.
From 832de811bd2fa7ff1899536d2d1e8c3c8ed5ca64 Mon Sep 17 00:00:00 2001
From: Rachit
Date: Tue, 3 Nov 2026 02:34:45 +0100
Subject: [PATCH 07/30] docs(examples): ros2 Dockerfile names its real build
command
Co-Authored-By: Claude Opus 5.5
---
examples/ros2-robot/Dockerfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/ros2-robot/Dockerfile b/examples/ros2-robot/Dockerfile
index 5a7e5d6..26e05b3 100644
--- a/examples/ros2-robot/Dockerfile
+++ b/examples/ros2-robot/Dockerfile
@@ -1,5 +1,6 @@
# Recall's robot skill on ROS 2 Jazzy with Nav2 and its loopback simulator (a simulated robot that moves exactly as
-# commanded: classic software, no models). Built by `make robot-demo`; nothing here runs on the iPhone or the Mac's API.
+# commanded: classic software, no models). An in-progress example, not part of the submission or Recall's install
+# (Recall itself runs without Docker). Build: docker build -t recall-fetch examples/ros2-robot
FROM ros:jazzy-ros-base
RUN apt-get update && apt-get install -y --no-install-recommends \
From 78e2d20d8f3c9a076282cdb9851721410c04b30f Mon Sep 17 00:00:00 2001
From: Rachit
Date: Tue, 3 Nov 2026 03:01:43 +0100
Subject: [PATCH 08/30] fix(navigation): say "1 metre", never "1 metres" or
"100 centimetres"
Co-Authored-By: Claude Opus 5.5
---
services/api/src/recall_api/domain/navigation/route.py | 10 +++++++---
services/api/tests/unit/test_navigation.py | 8 ++++++++
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/services/api/src/recall_api/domain/navigation/route.py b/services/api/src/recall_api/domain/navigation/route.py
index d8ab6c5..d4d7e44 100644
--- a/services/api/src/recall_api/domain/navigation/route.py
+++ b/services/api/src/recall_api/domain/navigation/route.py
@@ -108,9 +108,13 @@ def to_clock_steps(waypoints: list[tuple[float, float]], heading_xy: tuple[float
def _say_m(m: float) -> str:
- if m < 1:
- return f"{int(round(m * 10)) * 10} centimetres"
- return f"{m:.1f} metres".replace(".0 metres", " metres")
+ """Spoken to the nearest 10 cm: "70 centimetres", "1 metre" (not "1 metres" or "100 centimetres"), "1.5 metres"."""
+ tenths = round(m * 10)
+ if tenths < 10:
+ return f"{tenths * 10} centimetres"
+ if tenths == 10:
+ return "1 metre"
+ return f"{tenths / 10:.1f} metres".replace(".0 metres", " metres")
def bearing(dx: float, dy: float, north_deg: float) -> float:
diff --git a/services/api/tests/unit/test_navigation.py b/services/api/tests/unit/test_navigation.py
index 6b22755..c13f1de 100644
--- a/services/api/tests/unit/test_navigation.py
+++ b/services/api/tests/unit/test_navigation.py
@@ -76,3 +76,11 @@ def test_steps_are_spoken_as_turns_and_metres():
steps = to_clock_steps([(0, 0), (2, 0), (2, -1.5)], heading_xy=(1, 0))
assert [s["clock"] for s in steps] == [12, 3]
assert steps[0]["say"] == "Straight ahead, then about 2 metres." and steps[1]["metres"] == 1.5
+
+
+def test_distances_are_said_the_way_a_person_says_them():
+ say = lambda metres: to_clock_steps([(0, 0), (metres, 0)], heading_xy=(1, 0))[0]["say"] # noqa: E731
+ assert say(0.7) == "Straight ahead, then about 70 centimetres."
+ assert say(0.96) == "Straight ahead, then about 1 metre." # not "100 centimetres"
+ assert say(1.02) == "Straight ahead, then about 1 metre." # not "1 metres"
+ assert say(1.5) == "Straight ahead, then about 1.5 metres."
From 5ff417a214c1c1e94d4196a821d99818b3fb2282 Mon Sep 17 00:00:00 2001
From: Rachit
Date: Tue, 3 Nov 2026 03:01:43 +0100
Subject: [PATCH 09/30] feat(demo): the no-keys demo on the full living-room
scan, with walks to each thing
samples/replay/flat is re-recorded from scan bb449f4c (real Muse Spark Open calls) and now holds the floor
geometry routes need: the point cloud thinned to 5 cm, positions only, and the walk's camera poses only.
make demo-walks runs the replayed Guide and the phone's route planner from where the scan began and writes
the website's walks.json; make demo starts the API and the website together.
Co-Authored-By: Claude Opus 5.5
---
Makefile | 15 +-
apps/web/public/real-scan/flat/walks.json | 293 +++
samples/replay/flat/guide.jsonl | 39 +-
samples/replay/flat/home_cloud.npy | Bin 0 -> 406592 bytes
samples/replay/flat/meta.json | 34 +-
samples/replay/flat/poses.jsonl | 162 ++
samples/replay/flat/scan-result.json | 2188 +++++++++++++++----
services/api/src/recall_api/infra/demo.py | 108 +-
services/api/tests/unit/test_demo_replay.py | 50 +
9 files changed, 2380 insertions(+), 509 deletions(-)
create mode 100644 apps/web/public/real-scan/flat/walks.json
create mode 100644 samples/replay/flat/home_cloud.npy
create mode 100644 samples/replay/flat/poses.jsonl
diff --git a/Makefile b/Makefile
index 0d77d43..f42e605 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ BIND ?= 0.0.0.0
WEB := apps/web
REDOCLY := npx -y @redocly/cli@2.72.1
-.PHONY: help setup dev worker robot-export place-index regression viewer public-scan live demo record probes diagrams pair rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval
+.PHONY: help setup dev worker robot-export place-index regression viewer public-scan live demo record probes diagrams pair rerun demo-walks test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval
help:
@grep -E '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | awk -F':.*?## ' '{printf " %-18s %s\n", $$1, $$2}'
@@ -59,11 +59,20 @@ viewer: ## Export a home's newest processed scan for /homes//viewer: make
live: ## The API with real Meta model calls (keys from .env), and the scan worker
( cd $(API) && RECALL_BIND_HOST=$(BIND) uv run recall-api ) & $(MAKE) worker ; wait
-demo: ## No keys: the API on a memory built from a recorded real scan; the Guide's replies are replayed (labelled recorded)
- cd $(API) && uv run python -m recall_api.infra.demo serve $(if $(PORT),--port $(PORT))
+demo: ## Judges, start here. No keys: Recall's API on a recorded real scan and the website's demo page (localhost:3000/demo), labelled recorded
+ @if command -v pnpm >/dev/null 2>&1 && [ -d $(WEB)/node_modules ]; then \
+ ( cd $(API) && uv run python -m recall_api.infra.demo serve $(if $(PORT),--port $(PORT)) ) & ( cd $(WEB) && PORT=3000 pnpm dev ) ; wait; \
+ else \
+ echo "No website here (run make setup for it): the API only. The same demo page is online at https://recall-ten-khaki.vercel.app/demo"; \
+ cd $(API) && uv run python -m recall_api.infra.demo serve $(if $(PORT),--port $(PORT)); \
+ fi
+
+demo-walks: ## Rebuild the demo page's walks (public/real-scan/flat/walks.json) from samples/replay/ with Recall's own code; no keys
+ cd $(API) && uv run python -m recall_api.infra.demo walks --viewer "$(CURDIR)/$(WEB)/public/real-scan/flat/meta.json" --out "$(CURDIR)/$(WEB)/public/real-scan/flat/walks.json"
record: ## Re-record samples/replay/ with real Muse Spark Open calls: make record RESULT= ROOM="Living room" AT=
cd $(API) && uv run python -m recall_api.infra.demo record "$(abspath $(RESULT))" --room "$(ROOM)" --captured-at "$(AT)"
+ $(MAKE) demo-walks
diagrams: ## Redraw docs/diagrams/*.svg from the Mermaid sources (mermaid-cli; set PUPPETEER_EXECUTABLE_PATH to use a local Chrome)
@for f in docs/diagrams/*.mmd; do \
diff --git a/apps/web/public/real-scan/flat/walks.json b/apps/web/public/real-scan/flat/walks.json
new file mode 100644
index 0000000..01ea07e
--- /dev/null
+++ b/apps/web/public/real-scan/flat/walks.json
@@ -0,0 +1,293 @@
+{
+ "label": "Recorded output of a real run on 2026-11-03: one living-room scan from an iPhone 16 Pro, processed by MapAnything and SAM 3.1, and Muse Spark Open's replies to the questions below.",
+ "recorded_at": "2026-11-03T01:47:26+00:00",
+ "captured_at": "2026-11-01T18:11:32+00:00",
+ "room": "Living room",
+ "frame": "the viewer's: home frame (floor z = 0, z up, metres), x/y centred on the room as in meta.json",
+ "walks": [
+ {
+ "question": "Where are my keys?",
+ "answer": "Your brown keys are near the desk in the living room at knee height, seen on Sunday at 19:11.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "label": "keys",
+ "target": [
+ 1.28,
+ 1.835,
+ 0.658
+ ],
+ "mode": "route",
+ "steps": [
+ {
+ "clock": 11,
+ "metres": 1.1,
+ "say": "Turn to 11 o'clock, then about 1.1 metres."
+ },
+ {
+ "clock": 1,
+ "metres": 0.6,
+ "say": "Turn to 1 o'clock, then about 60 centimetres."
+ },
+ {
+ "clock": 11,
+ "metres": 0.9,
+ "say": "Turn to 11 o'clock, then about 90 centimetres."
+ },
+ {
+ "clock": 1,
+ "metres": 0.9,
+ "say": "Turn to 1 o'clock, then about 90 centimetres."
+ }
+ ],
+ "arrive": "Your keys should be in front of you, near the desk, at knee height.",
+ "path": [
+ [
+ 1.261,
+ -1.467
+ ],
+ [
+ 0.861,
+ -0.467
+ ],
+ [
+ 1.061,
+ 0.133
+ ],
+ [
+ 0.961,
+ 1.033
+ ],
+ [
+ 1.261,
+ 1.833
+ ]
+ ]
+ },
+ {
+ "question": "Where did I leave my glasses?",
+ "answer": "Your glasses were near the desk in the Living room at knee height, seen on Sunday at 19:11.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "label": "glasses",
+ "target": [
+ 0.548,
+ 2.563,
+ 0.651
+ ],
+ "mode": "route",
+ "steps": [
+ {
+ "clock": 11,
+ "metres": 1.1,
+ "say": "Turn to 11 o'clock, then about 1.1 metres."
+ },
+ {
+ "clock": 1,
+ "metres": 0.6,
+ "say": "Turn to 1 o'clock, then about 60 centimetres."
+ },
+ {
+ "clock": 11,
+ "metres": 1.2,
+ "say": "Turn to 11 o'clock, then about 1.2 metres."
+ },
+ {
+ "clock": 11,
+ "metres": 1.1,
+ "say": "Turn to 11 o'clock, then about 1.1 metres."
+ }
+ ],
+ "arrive": "Your glasses should be in front of you, near the desk, at knee height.",
+ "path": [
+ [
+ 1.261,
+ -1.467
+ ],
+ [
+ 0.861,
+ -0.467
+ ],
+ [
+ 1.061,
+ 0.133
+ ],
+ [
+ 0.961,
+ 1.333
+ ],
+ [
+ 0.161,
+ 2.133
+ ]
+ ]
+ },
+ {
+ "question": "Where is my laptop?",
+ "answer": "I found two laptops in the living room on Sunday at 19:11, a grey one near the table at waist height and a black one near the desk at waist height. Which one are you looking for?",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "asks_which": true
+ },
+ {
+ "question": "Where is my wallet?",
+ "answer": "Your wallet is near the desk in the living room at knee height, seen on Sunday at 19:11.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "label": "wallet",
+ "target": [
+ 0.643,
+ 2.253,
+ 0.664
+ ],
+ "mode": "route",
+ "steps": [
+ {
+ "clock": 11,
+ "metres": 1.1,
+ "say": "Turn to 11 o'clock, then about 1.1 metres."
+ },
+ {
+ "clock": 1,
+ "metres": 1.2,
+ "say": "Turn to 1 o'clock, then about 1.2 metres."
+ },
+ {
+ "clock": 11,
+ "metres": 0.6,
+ "say": "Turn to 11 o'clock, then about 60 centimetres."
+ },
+ {
+ "clock": 11,
+ "metres": 0.4,
+ "say": "Turn to 11 o'clock, then about 40 centimetres."
+ },
+ {
+ "clock": 2,
+ "metres": 0.4,
+ "say": "Turn to 2 o'clock, then about 40 centimetres."
+ }
+ ],
+ "arrive": "Your wallet should be in front of you, near the desk, at knee height.",
+ "path": [
+ [
+ 1.261,
+ -1.467
+ ],
+ [
+ 0.861,
+ -0.467
+ ],
+ [
+ 1.061,
+ 0.733
+ ],
+ [
+ 0.961,
+ 1.333
+ ],
+ [
+ 0.661,
+ 1.633
+ ],
+ [
+ 0.761,
+ 2.033
+ ]
+ ]
+ },
+ {
+ "question": "Where is the remote?",
+ "answer": "I found two remotes in the Living room seen on Sunday at 19:11, a grey one near the table at waist height and a white one on the desk at waist height, which one did you mean?",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "asks_which": true
+ },
+ {
+ "question": "Where is my bag?",
+ "answer": "Your grey bag is near the chair in the living room at waist height, seen on Sunday at 19:11.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "label": "bag",
+ "target": [
+ -1.848,
+ 2.577,
+ 0.975
+ ],
+ "mode": "route",
+ "steps": [
+ {
+ "clock": 11,
+ "metres": 1.1,
+ "say": "Turn to 11 o'clock, then about 1.1 metres."
+ },
+ {
+ "clock": 1,
+ "metres": 0.6,
+ "say": "Turn to 1 o'clock, then about 60 centimetres."
+ },
+ {
+ "clock": 11,
+ "metres": 1.2,
+ "say": "Turn to 11 o'clock, then about 1.2 metres."
+ },
+ {
+ "clock": 11,
+ "metres": 1.6,
+ "say": "Turn to 11 o'clock, then about 1.6 metres."
+ },
+ {
+ "clock": 10,
+ "metres": 1.3,
+ "say": "Turn to 10 o'clock, then about 1.3 metres."
+ }
+ ],
+ "arrive": "Your bag should be in front of you, near the chair, at waist height.",
+ "path": [
+ [
+ 1.261,
+ -1.467
+ ],
+ [
+ 0.861,
+ -0.467
+ ],
+ [
+ 1.061,
+ 0.133
+ ],
+ [
+ 0.961,
+ 1.333
+ ],
+ [
+ -0.139,
+ 2.433
+ ],
+ [
+ -1.339,
+ 2.033
+ ]
+ ]
+ },
+ {
+ "question": "Have you seen a bottle?",
+ "answer": "I saw a grey bottle on the cabinet on shelf 1 at waist height seen on Sunday at 19:11, and two more grey bottles near the desk at waist height seen on Sunday at 19:11. Let me know which one you need and I can guide you to it.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "asks_which": true
+ },
+ {
+ "question": "Where's my book?",
+ "answer": "Your book was seen near the table at waist height in the Living room, and two others near the chair at knee height and the sofa at waist height, all seen on Sunday at 19:11 \u2014 can you describe which book you mean by colour or name?",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)",
+ "asks_which": true
+ },
+ {
+ "question": "Where is my umbrella?",
+ "answer": "I haven't seen that yet. Scan the room it might be in, and I'll remember it.",
+ "grounded": true,
+ "model": "replay:muse-spark-open (recorded)"
+ }
+ ]
+}
diff --git a/samples/replay/flat/guide.jsonl b/samples/replay/flat/guide.jsonl
index 8da62d1..affb4ff 100644
--- a/samples/replay/flat/guide.jsonl
+++ b/samples/replay/flat/guide.jsonl
@@ -1,18 +1,21 @@
-{"question": "Where are my keys?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-17f0f934043c4c39b89f71c1da8168e9", "name": "find_object", "arguments": {"name": "keys"}}]}}
-{"question": "Where are my keys?", "step": 1, "reply": {"text": "Your keys are in the Living room near the desk at waist height, seen yesterday at 04:21.", "tool_calls": []}}
-{"question": "Where did I leave my glasses?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-7613d07eef0b4830a2869fe18f562ba4", "name": "find_object", "arguments": {"name": "glasses"}}]}}
-{"question": "Where did I leave my glasses?", "step": 1, "reply": {"text": "Your grey glasses were near the desk in the living room at waist height, seen yesterday at 04:21.", "tool_calls": []}}
-{"question": "Where is my laptop?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-9148a321cb374e91942537b78bd58c3e", "name": "find_object", "arguments": {"name": "laptop"}}]}}
-{"question": "Where is my laptop?", "step": 1, "reply": {"text": "Your laptop was near the desk in the living room at waist height, seen yesterday at 04:21.", "tool_calls": []}}
-{"question": "Where is my bag?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-caeed23def3443b69bf398bc8e170435", "name": "find_object", "arguments": {"name": "bag"}}]}}
-{"question": "Where is my bag?", "step": 1, "reply": {"text": "I remember two bags from yesterday at 04:21, a grey bag near the desk at knee height and a white bag on the cabinet on shelf level 1 at waist height in the Living room, which one did you mean?", "tool_calls": []}}
-{"question": "Have you seen a bottle?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-d4b41c061c374ac39ab892b6650dd1b9", "name": "find_object", "arguments": {"name": "bottle"}}]}}
-{"question": "Have you seen a bottle?", "step": 1, "reply": {"text": "I saw a grey bottle in the Living room near the desk at waist height, seen yesterday at 04:21. I also saw a grey bottle on the desk in the same area at the same time.", "tool_calls": []}}
-{"question": "Where's my book?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-6c38bc73533845ffad9585adac2c042d", "name": "find_object", "arguments": {"name": "book"}}]}}
-{"question": "Where's my book?", "step": 1, "reply": {"text": "I last saw two books in the Living room seen yesterday at 04:21 \u2014 a red one on the desk at waist height, and a grey one near the shelf at waist height. Which book did you mean?", "tool_calls": []}}
-{"question": "Where is the laptop", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-27b2630358ce45f5bf25e88ae4436044", "name": "find_object", "arguments": {"name": "laptop"}}]}}
-{"question": "Where is the laptop", "step": 1, "reply": {"text": "Your laptop was near the desk in the living room at waist height, seen yesterday at 04:21.", "tool_calls": []}}
-{"question": "Where did I put my reading glasses?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-be6b68b48de7426e86eaa08bf225375b", "name": "find_object", "arguments": {"name": "reading glasses"}}]}}
-{"question": "Where did I put my reading glasses?", "step": 1, "reply": {"text": "Your reading glasses were near the desk in the Living room at waist height, seen yesterday at 04:21.", "tool_calls": []}}
-{"question": "Where is my umbrella?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-6d2d35d244144e068698c1efb969a016", "name": "find_object", "arguments": {"name": "umbrella"}}]}}
-{"question": "Where is my umbrella?", "step": 1, "reply": {"text": "I haven't seen your umbrella in my recent scans, so I'm not sure where it is right now. Try scanning around the entryway or where you usually leave it and I can check again.", "tool_calls": []}}
+{"question": "Where are my keys?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-e18b747ed50147108020c8ce38bd680a", "name": "find_object", "arguments": {"name": "keys"}}]}}
+{"question": "Where are my keys?", "step": 1, "reply": {"text": "Your brown keys are near the desk in the living room at knee height, seen on Sunday at 19:11.", "tool_calls": []}}
+{"question": "Where did I leave my glasses?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-ac51479c9e744db7baf31c01eda2dbd7", "name": "find_object", "arguments": {"name": "glasses"}}]}}
+{"question": "Where did I leave my glasses?", "step": 1, "reply": {"text": "Your glasses were near the desk in the Living room at knee height, seen on Sunday at 19:11.", "tool_calls": []}}
+{"question": "Where is my laptop?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-a4ead53696744bd7a6b614de29fbdd2f", "name": "find_object", "arguments": {"name": "laptop"}}]}}
+{"question": "Where is my laptop?", "step": 1, "reply": {"text": "I found two laptops in the living room on Sunday at 19:11, a grey one near the table at waist height and a black one near the desk at waist height. Which one are you looking for?", "tool_calls": []}}
+{"question": "Where is my wallet?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-90cb2bbec28a4ad8b7fc8cd5fd29a4c0", "name": "find_object", "arguments": {"name": "wallet"}}]}}
+{"question": "Where is my wallet?", "step": 1, "reply": {"text": "Your wallet is near the desk in the living room at knee height, seen on Sunday at 19:11.", "tool_calls": []}}
+{"question": "Where is the remote?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-25ae49b6121443b0ac7474b32469a341", "name": "find_object", "arguments": {"name": "remote"}}]}}
+{"question": "Where is the remote?", "step": 1, "reply": {"text": "I found two remotes in the Living room seen on Sunday at 19:11, a grey one near the table at waist height and a white one on the desk at waist height, which one did you mean?", "tool_calls": []}}
+{"question": "Where is my bag?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-664577b31b284219a689859c7e3bfc8c", "name": "find_object", "arguments": {"name": "bag"}}]}}
+{"question": "Where is my bag?", "step": 1, "reply": {"text": "Your grey bag is near the chair in the living room at waist height, seen on Sunday at 19:11.", "tool_calls": []}}
+{"question": "Have you seen a bottle?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-fa84b39bafa9482884e713c5479db01f", "name": "find_object", "arguments": {"name": "bottle"}}]}}
+{"question": "Have you seen a bottle?", "step": 1, "reply": {"text": "Yes, I have seen 3 grey bottles. One was on the cabinet at waist height and two were near the desk at waist height, all seen on Sunday at 19:11. Which one are you looking for?", "tool_calls": []}}
+{"question": "Have you seen a bottle?", "step": 2, "reply": {"text": "I saw a grey bottle on the cabinet on shelf 1 at waist height seen on Sunday at 19:11, and two more grey bottles near the desk at waist height seen on Sunday at 19:11. Let me know which one you need and I can guide you to it.", "tool_calls": []}}
+{"question": "Where's my book?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-4532fea656c144c2b26737ee44388593", "name": "find_object", "arguments": {"name": "book"}}]}}
+{"question": "Where's my book?", "step": 1, "reply": {"text": "I found 3 books in the Living room seen on Sunday at 19:11 \u2014 a grey one near the table at waist height, a grey one near the chair at knee height, and a beige one near the sofa at waist height. Which one are you looking for?", "tool_calls": []}}
+{"question": "Where's my book?", "step": 2, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-b1ecc28bf3a140f69140ef1c5448d1dd", "name": "find_object", "arguments": {"name": "book"}}]}}
+{"question": "Where's my book?", "step": 3, "reply": {"text": "Your book was seen near the table at waist height in the Living room, and two others near the chair at knee height and the sofa at waist height, all seen on Sunday at 19:11 \u2014 can you describe which book you mean by colour or name?", "tool_calls": []}}
+{"question": "Where is my umbrella?", "step": 0, "reply": {"text": null, "tool_calls": [{"id": "chatcmpl-tool-05820fcdd41449538dea322747b3e562", "name": "find_object", "arguments": {"name": "umbrella"}}]}}
+{"question": "Where is my umbrella?", "step": 1, "reply": {"text": "I haven't seen an umbrella \u2014 try scanning the room so I can look again.", "tool_calls": []}}
diff --git a/samples/replay/flat/home_cloud.npy b/samples/replay/flat/home_cloud.npy
new file mode 100644
index 0000000000000000000000000000000000000000..a65486dc5e89bfda627b5b9db9e651b2be9ecfb5
GIT binary patch
literal 406592
zcmbST^?y{y+r^8P7Wfv|*ltKRo0WU#`pn$o7D#Xi5W}TVN^y6G1QL=U#YtGQn=S56
z61=5QC{oa_wLp4a>ZE-~|B~8huZ>GmBW$PPR2{gsS)FY6scK_
zFOo%jsDF~`YX|iraxE=gFD}>Art0P8s#a2cDaN$
zPt!1t))?kBtr0L(Ya@S9AE{mC&*~wyxBOn6s}7J~sNL1k@=MjJ&XJ$050q5-fjU^p
zkZ-E>l~wWub&Z@M7pT9;i{$-kwA@9Wrw)><%e~ZJ<*!m3b((xgs;+L3*GTUbue?;c
zrTFCD(n2L$wn`DoTKNE3Bzxsmq?f!~9!Y}a%W^ciE$x&UIVqizKS-(;YWzjE$g43H
zpy~H=N$sZgqg+<2ucgYT)fjo2SXFAT^n^xJ)MnBg<-Yt(a={uJ)_hW10S~D0cqM
zD4kJG$pO+prG#>ie3oy>1>}r;UXCIqAva$BMsuiJ$u`V9l|*g2UTEhb`U`QIR8+Yy
z^;T!ePr#w8{vP`~U~QLHUr)gP50#R#puN+2W4uNgP3Fpi@>Tpzx+XsmQ>CqPsx(n+
zLf62{6={-OP8+852KH0NlkIY*vQ!!izn7BT+IH+vPK#8Ql85pFB_8{%P@0iAd9~7#
zw3iJf5#Fq$q(~zX8)?|>wXqEN!YE|2KueO}s+#->+(v0}^0!)^yiEE@+ipBS{Jhap
zq+!|)a8_XzlitcHA>;91x73E^AP-pYG2htn}1RdN`17tYCAAFs4s_o
zj;L!3Il!C0XouB(Qa|kxqCH0YDi!i?XS9R9Q}t9t-8A*RSVKusUXYXUV=ZYH;wMb<
zAO>1uO{~6G8USn5mR6}F<<{8Wp{Gb=v@YsiX`t3XX(IiBn(~~S)g=0goI%W6@$5(a
zfHVlP`a#r{naWmSg|tJC0_V4|$pozq*!I(+<+5S}WNV?^ig+pq%nv}*Pf|-rDbSLU
zUr&_~F}JlzH)u#FU%Jh!LZ(ItvxK0s>Wi6ZfYua=!~6Ls>9_(Xu3{5L2hZ)X*i^8(_#xP
z_TQS%(g|`Mp0#0z`C0+FhRnT!Jrj+1DHBm@0o!P82-Xi$PJrhE$nt6y%w%Y_lzeg(
zF;?jPhiU|NIHdlJr`F5I$Z55oJeSN>N6FiT-f}*eT*3)*5<6Em&Xd2i54saOPg93s
zpD4L0`B5Dw+lf`32(}gEf8{r@RegOZKK&{skb#&lLEb6T>z!s&6?d5Zpm{GqMYULtZ9YBi+;h^Ql^oYn#SZ(-$ga#~B&UW1!et05gm>J_Je-dm`jT5UE!k#r~@ISq27nKhpc%<
zGIpq=2a-K%g4_XGBpR_;kGh2QueFuXbQ^Se4lPPc7q$7)dvP&xXE5xrT293JN7(Zv
zJhBw)=W4A;1+AT2LM*BdLk4}&9w>2wq>PlK-RGqjav*W(0dxUYw$m0DS^!u~f1oZa
z)KOgxoT_Hw(^%DqPaW!B;A2GlXKk?ZNj$3dkl%}%-k+8ry55g|1B|7uNuYjEPk=5?
z{Q#a#RSK{^0B6KWwY&U9G~nwfQV&)-3SH_dbKHtNLM}oKL}TH6cl{XoqOFo&i5xRu
zMXE>817Na2K92S6k-wj{VEG%7qs}A>>y%}9R@bZ0!l%w7i$7?Sr4J&5zlvf0ol&Uc
zS3Qa}(BEi#vCk*zBX)jm6n=(}pWi=eZ4Cz|-Q1lCxMR#;=z
zK8ZiVkA-~Px&>HUZ%u+=?N-F9A5uF27ppx;D6IWnY^p!j%i!7Rhy+!CtGySS!>U0<
zf%h#$fM;`s-l#2~#3oo1Ohj;Ki9Nfh--ug~^@H6Hra(Yq`q#KPPpDk-Q`wTqtRfZ=?7IC
ztdyp<1-GS2S@E3OO&%ongH=C^Z4spfu4-~;*(y%cN6;Z+Uww!13!YtQyc65$M-Y=#
zucx%f4kzS6Zc`a7hl^A6o^(7`YQ|f!wLVYlNSNM6?ja1u`SlBSwi;9M=~w-g*an(@
z5nDi`4v3k{Qg`grSLy})PKp=CArohcee^^6a^PhBgV;*{syc)?#D6LAH+>YXC#E2-
zLc|5gYnwP$FGVB8(fV^^CHC*4e*ymkYALZ2vgUhnt=@~S0rT4WV0V@>U9OFldB$(p
z`K^|X`F!o2*jb;V#tZ9_3k9xpxhKxP4LYYK#0=~oC{9L3JH!ckJL8Sm9iCl=r{*CN
zdg=AlaB#2~pTv0mf$~Dw2>X{1S0VzP;v#r|GWaBEiy_ORUBXk5>J#8yL-n2u)soNF7|n(
zrUSRBKHxBQ5BBV;eiaAm^_9w`zaAym!p`TU`q;U<6h!*qlqw>w(%TwVagF}3URuls
z^KfwsV$vmgbdN5IoAgxujW}G-mnXWJx?Co(UQ^g}tlmMcN=E3rNs>@Oi;?qOveHKm
z7I$HN196?+O^+1U>ki#1E{0V;b(?WIQ`q+RN_!hqn#>8pdDM
zPS|s@x(-%7BYQEwT0VgJCi0)q>oO|yDt$XCaCuR06Qske2pcS&W}oTL(n)ri{sKHk
zhe=1-ZW@i1-PJ*`LuGX+_Iaet!aiKN0SlYbzv5=(;Cpc>c&;QB*&Vu{M6lc7Nq~P6
z6Elm2B>kE8LP*zpYeld_ckP;ZT3;_c693RAORvRq*r6ynrSBty+%wfK@_wSSyYw#-
z15GmSt(7F4chh09
zrsOOzpLBq&N{T-Hu{uvYsV|XNh<_q~KZy5m66Cu|tCQvGu66QUIYK(cKGEi|Y75#*
z+RnlucPFb3iM!Y3w+o1^jz}^kq-bOBb-O
z)G3{1dvR+x#d2w+beyfk%yG7a#z`I)VvLoxuqpa@X(Q{b&yuoPJ$k*?d4a_UGjXni}*9*`3LC?8$qihx_Zz$z~ NeW`AN*yVj)lmYaFm^(&E!Af2CYZP%0qS(+TCMb;2pLO
zb>JqV*
zB_x?fh-bk{cIFcg{opWNUzGOpa_4(_p!`~_348V+3!vRW$QnV9kZY{4WB&)yRY3km
zls+M+SwrI$@T>j_99HTz5GMs1ZUih!3jv={?GOoOkI+2^!>8k+>27k8?4`4uhl!sK
zcFuOisvYDF&KgR0xu}>yZ_`xq1#EkWTt2IKElKus%}c)AK6?1@!>vqYl+
z+65U!AtglHMtZY9(1E*HH}x?5HA6Z?>LYTJz`VTF4R%{V+K_3eyj{s8nk=NkkG&j~
zBn$gB;JE9w+EV@`?1S}Ml0EDtO@kDMn7Paz;?8oJ?bC;o(~#PP%%u5ZS2B;9LagJR
zG(!F=oWxTd#gd5L;iL$ELVqR2IJ|(UE(MQYWs~(2z>fMXBC-f|E*L&m){|OnCnEL-
z<|Ez6F1nDsB0;R{%AUeFdXwH2e?r7Ih9zIqpGZ-Dj81{>;q)-M&T50tHTGPaf-@*h
zor)+;QD$S$|0xTB1#%i@I>?#8&*<@1(u%G;R~gNaS3CD8{pC1^Rh}q!7iJ+ZiiwZF
z{HZtt8QO}xWN&d_E6QKd1n}QO6QPR_ar-aJ)AGp;HdjMjpzhfa6^Wz<8B3>%jbPO=
zZo~k+@7&I^tT7_tDOUm#&v?qDT`U2k|=7*6d?42#pw0mE$#&qCEJry?Wq!&h!Q}2m59H_>U=B%;Wi!k;CZ$T$f
z3wbL3O6!u_;$V77e1SUY5X+F^)GdS}->bPJNE&6X17J8UV574F-aLL3s-c3F78?K`
zihI~|%3;0n;PZ|}(01f4tA$3iJ7`t%Rvb=i3Tfnj%w2z)W2+n`M+-Y(hnM07M63jV
z^@Jbau@^?6+!ICyQ8nRxbEqbI!
zViWR}-WEg1AGA36EUu!}$Zuj2EenjNXT&U^MSL$#rbFC`j&*W7IaW9g4$lPvweE}f
z5)o^IM_Pi(dp6IA2A_DNEBIK9p1@Cf3VF)Ds_9_0ANQa}EK2S|1h$B@Bz4$3F^Jrv
zkHl9vn^w9?!sh=tzY5W)Gr_QGIJ(v%ydrH*nz64^OW3MB(8UtT3voAYty{zqbfWvA
zxP-oPzC+(K$~g%6HE_jdXG^t>+*x>xXfG}HK_z)CmgTQ$AohG^Siv?AH_b0>KW=dC
z5D72EGxVK!M?6M9it(g6`*&48QkSh*7w7mOw~~7y>Tbi=HTg68My$(k;r3UHAHi}fR!@Uu=avtMrvoT9hDnvS>~7vm1SFxOsdU13`;bc6i^EG2CdwV~7Gf5v!9B4y_tGa~b)HOLh(934-ihDC<6p&ZxrsI9pF_TgDtjoF5ogl!WSh{EUJ#!NNi>UOa=yAQqM*(7)&o;jsw
zVh7~lL$Nv2Nl|eOJ>*Jo`>}IJp&uguqwo+_WjSK(g^`7wj~h?LYJ4*8jMaH}eI0fi
ztXu_lm&%A<`c`}{EJrRp7ADd~u1x0*wW-`rn1EZxXQ2dtK~>Sk?^6SKhc*QsqbtFA
zj*$)fv^DMn>*=oxW#L^xO`fkULB9M?!R?#n$vZLLARQ6=vQTNK7=!v;Qal8keiV}F
zX=f>M8SUXra1>Wt%bnflP!qZfYheG+LOq-*E>YmQbSZohXsiH!)w6&H^sT_j`W~@9
zba@6Qv$dCyo&f1Jc%b$zsl>ag3&nn{HNMffORtMX#6ReL(I+gWpT#7h3t}irXijH}
zJ%x6(irCeC4|2N*yKn}*5d>stX;I~W)2fiZ828XHPXEiJkLq)1cK~r)gQu!-!T|b8
z{9TwvsbifmlD2T{aE(ST9Cyut7h>H-*ex0h|30NJ1)U#-MjGFU(^EtamJ^+bt03%L
zf!cwujR&w{NA;;#mD_PA?aTI%Qep{qPE3GD-iw`t-c%G494h>h(@qM2PL#MdXg1{Vpa7n_-tW
z!dSY-)kT<0qg=g&>GZ5K(a~Eqmi{Q5h12n*2FriM+sg4O#Py&w`7`8D$Vl
zK}MCr=+f&z`awMy<8(a~I9Yc9yXgY3k*)!4dQ0Fl?UgVFHip)
zw;+vSw@+#lFg%ExvI`NCAO`bi%1&V#!*`v+3slZx!UX!zwO)8cd03)jz3P$=xL$x;
z5BCJ*>|(bE6}y8Fjr@ug+M#a832jifUkQ}I(MpI0->Q`drf9lIcz2aSN`(5GFqJu#
zlR{s1M(lx8CQsb$?n}oCJ6**PfuTYSe?VIcQG7UUkKN*ESD__u0WF%LO1~8tYJO4d
z8K{*JbyVr9z!WtER;s0T73#2U;(T{^+ErXAlxJ@mCJ0yXf;GW$NtNY2u3Cu0B=-(*
zZY8wh0W=CbS3@4P;4h5?;4$O5!1-Q85#wFd3Zl#(Ddpki{YoFymJRaH!X(y0ju$A)
zK%9T2PeP-F2UH6kX>TKo@?n>R$m!)ii!&tzT5#F|mfvnfL)Jp06J(7xx&agZlND^N
z!Tc31N?3tYpslc!WhyO&1#FnoMwpG0C0dxlmdibbIjjxN;;E?EeTC_4hcpP|anfXA
zBCAAt37y$gvA=r&Js>8!<7t36z}*`+nhwJEtU>rl`v_T<_qkeu)p1ux#8P+nU2v-^
zv`6mLhu2;ktzhAbVAuxn+y!_*Pk`l9^;9q)sjUXXD9tUTvoxivFb|bHN|?gzQcJ{h
zIVoBg#kPRM6l7VVFp`ZHhqz}WW_r1QLQS0No=HXbLiYkX#P!)-K<@@+I!7YE4!Zh-
zNus+fZV;#4z4>n10JcgrV!)~sIJD=HdMxmz))VsuT5sTLZ3VbxsT+V3)ogI74%3Ucq#
z_3nQBANo7cM|WW+1^(*DXKIDg+h~1&R&6Mxe^E0av9wwY5^E^o!e;hC8sa`o^Ta;x
zbb8xWR;a~pG%PB7%U)QMotcn$z_k+D(bY{+uO_+s@Q0Aq2l1Q@DRGFs&OAhm0e(?C
z1Myo3p%YH$NwD@3<%N4E%TkI9+t_Y78~IyN+AKKvzvO#i6{{*0MFdVK7GW713VpBO
zZr;VchVFH}a@S*RLPyzMiYBKycYwoCS3mfDmunmPzya9*U|y&ixar`Vx0v
zRNxG-lITjX3a2qb7v5Rz4ki(5q2=!?i9%bxNcqdX2Oaw}WKDB<0V?4xQbX9l0>!@W
zcXY2f$$fxc2}*Gm_Qt=vPUHN|cilyPCAo*84h(>-adZkS96)CRtI}jhd~T#eV!pA*
zy_MBd?z;W#3f@!IVLtRumDzStac`v^T(#WG=ndx#_rLUM$Y^_}@}s=c`51b2gcd!~
z|Mug#M&X`wjQP+d$woh8{#>UWjn
zkDzZ5;_46V9H9M+J@2YFfk)KG*m;8b8asDa7r2kHB1$@T-XN>+@)qowj$Z4KD;c=Y
zH4fG7kZU5>%)eIoSv7(dglf{8=D0}cYmT)NbQ6byqnNmNais@6;!|G?qjZhkUKkZvR8KPcCDf}
zasC!|nA4omsQ>Goy@C5&)6lu)xn|*H`4i)0I>$Yn=ct9aL`{bD&gvHERb9CVY$9ET
zrcUX$y9ZAsm4%UfrMrSKh!1s_6cTt-_fc0-wpZNmdPnz(!(Ef;FkzIvxk}|E$1cd-
z;GB+g;kt7p>QW|TJ)l01pyxcM?A*tpT5X(lPfyzL^x@(aiJ?Bd#j!SIZFlZ|MD6I!dyC=~CxiRG}!>GJcLmyVBv)t*)8K&^^F?
z`oTgw>nDKS^fK=0e2-e$Jq>lfGURr{HZZPB~I7Hw=XE{`0#g&Qvgn>gc^qtS0kUkGL)5E|5?U-vW
zU#OLZ+*`_bz`05lShbQ;71l10o4CjFc#`Dm${I>j+#deiJ=VRJFLDpWngZ8Z=QKJ&
z{M$Jd??~S{7vStmv<;W-@Mbim|LP1xm00Rjf#qEp{2mR31x8XkFph3^EkNb*74p#z
z0`I6NfQQt=d~Ys)1MLc=8o*ML1t^oMt~6RJB-$|q`8e7cgFVxogZN|mz`2HBqpzLo
zP#?ZJ*CKkWLf;m&E+j_KV92UQBcc0qW1DL+&YJD6Md)z$xfXJpng@wbmCuN-?ZoCD
z$DiUI*Dm^}xWbu>>a^0ekXgev<3ww1-Q(KGYGzz@=Fq;*MB8iVXgH=KTe~_kkU^82
zGr?rJa|KwP0h1JZ8LY<9TCU}|%>{$c0>cJAy$k_-EQSueuQvwn)0+cR^tQmRdS|S0
z>Twvq)cS&Rp5_JTCCWC~s;W}xwRHKQYc`KT$3ByPkWL~u$4S-Pe*U|=5a+ndBQno8
zH#zUpQ)0UFAL!ECnNEKa_ThB4S`ut7#U(d#_>u2D9Vd|mo1N=%rfdi2mh=Ry7e-ry
z?K~|8Oyad}K#P_D{HzWF?pH?ulhrX`*h!rNhBeg9VA5XlB8KhKPH@{o4!P#=IP%mP
z&nihzv1XlXlkNC1li4G(9d+^x$#X5?jamTQ8r-LmM!(HQ9iY?1|js6v!
zVB4$6@?iT6dV@-ivZ!nw9f8P&63*XoTP)+;%8%3EA=hdg0p8dD0PfS80zQ?jXFD)6vi8NmgpbY*+%bU=L&R;
z(~GxHvz<9O>lZn<@MBtr)58~Q0nqDL$pR@6QoLg)?%G`)xp+I&31?8KIMuP0))h86
zN3v(ZiMBSXDEF}6gY~SAwUELcyU{ncbR0zA7y~>;;~e|ZO`URV<2#LO;4s^G42(D4
zgGs1S26n!$|KjxV*4kL`3D6b;@2Kg(L+X0qQgth^le!-~!_@-te4<<~V=pSkNND=n
z_|vfy@o*iOqCW$6(LVtr^>WS~{GB!k@$-{96xdOn;`Ad6W&s~6tD!|@Wgj9rPuhp*
zogk$;hx5vk*EyCSAp4w?_(XEoIfs`d362^pM{MILgWhk5a~u076nnDg){f3~?7Wr2
z7YU9Y4j=v2($-GEd4N3z80I*DGf@VC
z=T3Cm)qwlevCec}Ui!&73g>cLM^AQ<+;Xnw39k3fEdJh^#x;lEX7a`m>3(=lv%S3*)
zv)@B)Ot9l!C@lx+_h@@)&1hF(IPD#I8ue-d@R^YUR=*pofpZNHxFs4pfdR%Jz~lN2
zV2U2*IDuN~2HRk*3Gjp35x8581*WOHBge8lGCQ&byG6#JCU*(#u70V
z+Xp@gwv)gg?G3;u#;zdu6YcGQBkX-KGX)Gk>KW=?>c-mJ{IXXnB}p
z4<9WxaSZ3dlIZBkYf3IhN4|qJadhEbNDoI6|0oWP9K_C&WscFjsf#)OWXYkj;{@Al
zZESl13-_=UMI^VePXzOLdnPc|{x7P^Z}yvrtE2WSyg$viUq)7*iadvkQPOb+^}oF1
zH1Dlg!Q_n;j?A4Q3CLEPWOrmCs_R5HLtHg?4CB3A-5rzo9%nhnHCVD-75jeFeYuZDvmbAP525(4*
z04vay@XHrzM&wN1K-y{V&Nh-`_BiGy1(CmS$@z=@CcP~7vLBjoUK_A-(1Gm^mL6S3_0t>nR~8{~Xg
zwYfas-iXg2$>86Uc^jV39&uEShT&v
zTg^7M&p3q}*`MMaW-I$Mya|lLI981XMyiQma#Q)q{s`UU6xiyMv<#W$mg4QBfL-jp
zd5F}}9?R=Ut?bQkLM6d^Tg5bc7w&bX+B@+R&i(c%UabCVdpW)<_-A__8x|68&t_YL
zL+p(ivFr{nVqT@E!;6{6=LW)6KG{*`Ca_rMhTt?dK|!SrI;5WB5JS
zG*mpL#xT3dn*0UQN}i|EVnPxv0H
zmhB@>OvUyIy=4rn*T(3AaggD%?WWVkI9n@v+0x3^h|aLowsw{q$f*s_BeP0{mo&%G
zI^o65UNjiHwWK2Sji4=If$DTH@SU*=xXaj%`B}y(tcf!Iz&O~rjqxYFgsr%_U;o}#
z%uLp;z>d0MD{9u%TLRx|?Sc7PKU)!Vg*Fa|zl4gI0}CYCPSaQ6iMD)tK0MyGk-oHa
zw58B#7SlG0F1Kv8zE{HKy4Fu}i2T*ET@I4xT4tiUuCWw$p`R>!fiJ*(uJ$>+gxO20
zV=G~XYGJ@nYHQ$bwTG>wxm4{79I6h%Oh};&%ds_khXdPy)q=w1Wtz#9NJjQY!tkzkc
z0e4vnd$22(y1)`v8Fi9{R!6S22n_^QY`6pd>d>&XiEn|z%bKz3TcBMnXDe;~t5mgp
zV=hsst&|y{Ft9CBh4
zwnn_F^e)`N%Sc6R5qvB8&gSOriQA^|qvDUy>t7+>#`$1(S6d6d+%>`0gx7X$wng)r
zVLfb*Swu)j+bz~2M6(sJd%+>L^{iTOU0W)v5?skPo<#@02=Bqlhnx-nAG26|;g9JO
zYfkuW`pEjfa62Mupd|+SPO}UKrdy@}{g!1oBkovsquVKG{R5FU#Hzv$SFN}U)2PsL
z<~?}6h&f)m8UCJIr7PioAy3bQ@5kG@-QfcNTgVT$bGL9NT;!MBZ^I4V*!e76=YfuI
zZ6?2I`_9&ke+jQ-Yt4s-Rk6kJ_15R%lwtD7_7;cDj3v}WiJW=~o?Jc4&4-&-zXO;1Z*
z^9CJjsc#;kDL@}Rf|*71s-=!OmR7LV#QLh(p*fX-;WPpJRHYu^E8{Wnn2{V>(>$jy
z2jcBzXf5Q+6JQ6uM8leqAyP#vSy>K!^Tq
zXkExk4y|K~S_Z~}+E(CK^-*X&bC3Egw7!|DhBU-_wOzwH=51wK!&>H2Wi7Ch@(~PI
z$zOqOA?7jzXLDVeuzEY7K_m3yAJ^m
zs~0h|M7?VXF}tg?u>U)0zV!polFnH-^IFnsYj0jj`pv4~9ee#yJAdl_V0CcGEn=MF
z`Xw}y*Ki$7*1(DxeK7M&`ZSF7^bEvvyYxNi
zN$xB=kEo7WR_K>%%Zh>F_OwC^j7clB!0WVjcqe~gSwCRbvdO@pWtqV7sfD`qOf7;6
z&~#}7Sm|;~sCkF3nC~(#(nWLK<_T&Pb(sffb!azNiL->6Rh0#nh9-V34~&t2vV3Q5
zk``NT@JwmBzUbmW4P|7FxREecB331TQDewbaF1?zNTxbPo9z
zEB{L@#89!R)y~U11nc*#oTZL+6WwR2XiY(6zqE{`OD&fy33Lmv5xr>HXR*^iE%#Rt
zM0?V5(Og2iXS$IcY3WY02aQUPG%5Wi&2BcNQA_POFMge5H{TlJbzF$LpMlAGPF;!H(_8%=yo)KM2?l*P@}&>ooIX&GUu%*Tcdw|rnhi86iJfKGCk0)|MOe
zH%pY|I9+dX{wA6osP#7iJXfxAgJ%n1bsD$Q4W5=2ZZprAo++3MjO!VKnP@c4Ag0x5
zlTOSp`k$#1v?%WNK!fwrH7*Q>;k3WVY0t
zX~YcCmMkLX6ZK_r(LAS)tSFfK)U7LJbFVsJB{9>~Vk-rlW#53gLp`7AG)t;kz}w2q
zOqaP&i2*KEB2%eZO_{ZXnR#-^5(AObWvOO9lXfi$H{4|dOTP>ysSPC5r!?R%e%QJF0IAq
zxk@J2=3kx7Cf4Ca9iB~((C_7MY6m9rrq*UZ>fY1}-N=p=wYbM#Vr5;P6P~fWHew@k
zc_p3~az675>mGb8^BSuZq-7pqor8{KY-Ya&HOpAcT)~wyMzMjxiRtldMex3~#;in0
z`7}EV2y%)E0445zLS4Up7XYxsEYs=u}>PL@W?6KN+)-Q`VaTT5PM1l24r
zGO8l~`(}JLx?1eXXOtj0aLE{@zAPt)W9(Y8RSuBXB`3&r<*r#bP>XhD1;Kh(vs7S_
z>~?sA@m+R*V2$i4Kx_62V8`ssz((2a%@$P5j>ZXM%Wj7gq$0*Ijk{UV=I=&6w3}^Y
zLDP=LBxoIK!~wtP>o>s;+Q>~U%(nQ^NGr3cQXkrNls~Pj&UN8$;6nGtbwBVmuA%Gx
zhwKi)_+4a`byfM@h;u9V8Z2ac)-R}{C$c60-)1dGEiIev2R6ul4E~Pn5@4=ocY_T{
zcC0y0ZwTzFSIX{ghU=fRx|y%FX<5z9hFV-!Q`G-0U@}=5w<+2*l_s0on_new)+yBR
zTUjsYb?je+U8n!c4gm&c7rt4F%#H?<&AT|>Fu(!}wdm}&X1vxVyA?Xdi0l^T0kvLs
zb91~}F}s=BS^b*T%yg=^A?ucs2I-ZRDOs`R5jl0!DLzd~*;s|26&G&&A8#O*$exey
zn2KiqhI6W3wjUVj>23Z=1y4Wo2o3i1MdVZhCewNJ%hkuB`}4idIo~SR8N0&DRaPw
z(~(%&kdE*qnGNVq7+0YEfp3gN;A5jB@PyGEE9V*lw1_k6;MoSox4{`KOu9oT>KRol2w)8BF{Ee;kCm0dIrLG6FgIa8J?BE-#xp5
zr#x4He|vBfr2l#bfbE~qah`Dw`ojP^Xl#_Ld9SrQ}DG3gfJhgypJvvzJ_rw6-dwwyi>E*WkY(CdMc!r_cZ-uotX-7SY
zI6KaIdYLg8_cVjF-+^D%-#k6cU1|z2MV$ofs`m4AH>;_Qvrn*nfnnL}*u&=w@0=IEe;N}{9Y-r=`CaiC;D?5^H2(FO*Ejt+e
zCF=u?33-xrhbD&{$||7QA-z43>>3^Bk&$t8JTbrwPjBG#oC&DsGjk>(7iQ#4MjTGc
znPM)cV=yz8j=-mJv{=pnGfDdlh5_0u;Ai#jmcHge^(=6ldJ5Q4J+!5d8L6({(%XEd
zYz8h+767lxDO)C(7o|&{m*|C#d+zY1Qnu$f?R{Zn4Dei~*XVH1Bj6lQ
zF=)3o$A-@T&zx!IDf(B=3}n`yKp#DwGZWc;9NM*{hjJ#O=idjts?u#3KQ(;7d?O3E
zz}N(>C;znNEl-nfct+ub
z-R6l!Mr3+g^1Eb+$HpI!_MUpU<%MKdp|3(pWq&ljg!r@e8Y=iNK|LJkSp!_**#mr&
zqk(^A?@aV;)x9&o_J7`KVEesyD%h6A_?b~0SYUhxlLf%(VEY)_H8Aew%mUkE(D$vD
zgZbmy(Ji~s72NhT;JGB(zmF{eI3^bGi03XN);Z
z+YL)Dk*<3#Gb!+-Cyzx0Zt>)>U4ipGOWEk4ex7mcr=Z^6#W<;3VV`4~;+=@6ns~>X
zahmBJi+)T2K2~MmZq@D`gL_AWcbwVfKXj^fF#l4i4_vJL0IaQ4^-ec0%GI%3vRoP1
zNWPcT&y1JSbBddDq>EbuVV`qb0(hwKatq-h_HsFGd8T!brw7{@lFn*}i@V>*hPA_wI^4sJ|P7dDF?#~&=3&`voyyYMxp-~^#+?-Z?f^C;4i=GM2
z@TAa&gua~t`Cv)*7I0m#?RURf_Ey~u4Da;kUUQ!{S>^+(-^gDvL}-M5fbFxf_fX_-3g6M?-}6FzQQ*Z
zbA1Z*TI1`CH>}S;&-ZV2bVm#{P#keUonk-UCbo+x^;DU#jWRCiqg&BhB|MGFxaf
zeGBnsU2TQ;B#+YrTXz!2s?kpP8{}iLV
zj9bMv7`udMM0-b92X7S5clP!6=7Yl;fm>8ygg2C11M7LK@iu`gJd2D{)&Q>+e*xK_(&5vqBjQ6Rb7~`8PeA-nF$z6f&v`+4F^B=|Kd(6Y6lD@6{5J~s8;iKJ+
zykA&wpv`-mEezP_>1h07o$rY>dRpsyU(tW4!&@8-`}#~+X}GU7Fx58{cm>k&_nNso
zajrei-DBdL8Q>z!?=gqbi@CeeG2Y1CZaV0BU{!h=%-z_*C|3HSBpR@aZdzP!J)t@l@!7N~i9vHt~yd0ViQpiu8x
z+&G5%t|0ct`|bjZ`kR_pXt+Pqyi8@k-@HT{`Hz~%X*2(ER1y=om{R{S^gk5y_&aO=
z5xgzbfJugknLr~F_(c!L%5K18xB&(G594h^Wxv<_O{?P1K`-~ce~Zc0?=iNi-(q}A
zDGyw#l!L_Oa%sQM{8=szY%Uk|Z#QpBU*V&X(%RhNW|%ZGw~bjz8j{=C{6-4OeZlj|
z65j}(L#FtW_)0R_*Nu0;xHG>jPWE->^?^On4^Q+Z^0Ds8utu1Bj;}wTRXG7Q$$!$^O$Pxp=>T9d?FSr7I|AqE
zn*VpaInn*Oc)O^>J{C;@K2l}i;s0=vTDXP*x1v%JjIS#qaIxZoWvVN7|4!4C?TCjo
zDFF8DB$f70Gb5#Q&@@7LBbNBW_@sadzUsVUU{~*F+CP|juh1F6R&O4i73|(J
zPd^hll+-k1wK
zVa&(OY-15t#u&5x=gc5uI>ul1Nf@I`1ma)m0fu3G8n=njz&z-?#3W$;i}{08)!)l3
zDt+T;=52C4x3ZZ>>g4w30QP#n}MyL`aeXW7t_}Ac@>rnqS=;HD}1vd3pz-h7=*%YTOLVg8l3o!no
z7UIGGFio8gjdJBAf4b?Es{5;(`1gbSU-@p*C$~R8C639Rz?--x!L*9{R7LFS_f$YA&RAfFpqapyR#gzl@Ief&Y>jMsNH7#(Dj(|DyTaxCYEO
z{>983Gc
zwle1yEl-~Rl57k8*P(D8(J%b$<#^q&6&x~99R5&_z6
zj6bS(fqT_E{=CAs;#kvJy$?PT@D!@VE#OV%CSqo>a@YSCzVo^VEq2RyV3T(8E!g3h
zbkl#qoF!ev4(+9I|6#s@)by|AnWTb$K3`0VgB5-u06ZiX1KPxI{8PE=F6W=cQ(cw(
z3;1H^5B_C*mA$rqC2t-6o3AB%5IELnV_yQBc_ZldkUqY@pxrnADmV?T{#J;kGyW#X
z@k{;=&{2Yp=cxSq1M~kly6U*7mNqP+*e%#fvq%VRo!vb#yJs-4_1c}FVt41Yy(Ws-
zVuFbq=bSx@_u56MsMw#~-TgiA&-0t#?C#9_yfyF4iQR?vbL7dQ_H<%&JVl-~;M1u>
zQM;9(b3DV9$7tXyJ_4xnk-#;@*b_ZZMSngV_?oNixF{~4^Pz%$y^RTKY@cdd5V#3mY7Uu}^_zHw@`t|(Mom)Ip`yKHmZxa7
zvipNIV>bY=*y|z74RpLj{!ehcf@N*s_|a?ym(yl~c2o0P`!w+}^T0k>yv*FR_Yf~I
z$L$7DV*1!|&!=w3Z!Vj+!oSF#xd*q@k
z_E)xBh>nWLuFD;7acy@ouohVe{3OhEyc4s8A&%GLP16T^Q&GW;QQ|=+&t6U3#moll
z5PhoMY3mtfu}9gq2UfGcu?7b&%sN75g&Gcb6Mkv}4#xkI0Ssf?+2>DpGDUffX(=WKy7i)7jC=bJ9eq-pu<>p4LsqAWfd^RvH=_k
z@V)2@{49o;(}3Si8yp|Rxu&(SLQm6N$2Bp|q;rfE&q5F1Y#X45JMjNq(D^ZA27J4o
zaUYN|{A-tNWt-&L&)ar|CfKXnssx4G?^;U)re>`pvqOhxO(Ok}$KLVpH1+Ih$fa7l
z0=x*1xhw_L$?1_}Ov-T1q$Xa$FLFO!w`_
zZA+N~`##$!=CXY+qWe62+mFe&Z?_#V7T9;&ni#Lz_t*jq*YN2I{WW}Zv+lP2knN-P
z0sO%`?y=pC^Z8fyW44D;U+kA`|AyU%^|KXm)cE8vc
z@YsLZSHQLG2k7UM`8`;lHGhWo;?19cXHB2bKf&}6tnpSC#|5#bX}M!IRzq)x-8N5G
z)*;(QXzMz1Y?+ZS?5k`k0XLD&-UgWLYph9u^RnWJJ}faS2IpGghyURxhR4{)V8zL|
zTOo6Rvoy`-8aYeQ9+pDKA@MD9&@n|+nNWvf>&=8a9MD*}!)3D>qA<%GJ@3e~jg5Y6
z?`_*0c-5X@n;2-bkGJj*G};GSI|q%Ly~}J
zi`!YGIcC(e3D8Us+^&g$yg5m1zc9
z4qQq80nQ}LoE2$rvI0^ipuYl*Mo$ImN0tEJ2}|+mIpH7RN@2OP5*;tB1ojtJ;kV(!
zGT=jgDNyB?Ijg{Lmjk=;tAMfmO5h)EDewX}1{$_=Bf)VAHxAgBn*{Cw+(fj$u@k}C
zU5u;Paqx{^XnRvFI|;vfWSQdhp(`zOo#kj1%RFa!y2HE#@+6oS0pFYELCz_r@%UzA
zQ?m1q*vORWs3gu~=0R8a+D(pEw#?Ai_8PVhf!FQdtc3w{va*H#$e-1UHFFc1ks#q^
zlI^TcPZJsK-oyR`GWZ3PqoeGOIvpmYh{-p~;Rx^!Tz22hpf&@5$0>6(Y{4;?QY0_-&u`*
z5N^OO8!R`0qb-jhTa4vSk^H6!&J1L{hmQA<=dt6d?N;;+N1^Rt{d~s-+mZT@><6qD
z11{QAtOWr!`(x50B+7n)LjetjLC`-CPP=ZF;|{_Bj&2-s!J(f9~i?IwSRHt
zU6trQwg}g;KESSMSEfiAiS$KZA~md1F#k{PZ8E8X5eQ*#CHY?E%2+xLUZ6ZJ|5$!e9>sD`2<%3+Ldn(
zJja<_epKe7!D1z+2M*>et}65w%et!4e_0D~Eo%aHMo%>wfu8F0nT2!JgqJqQTvaV?
zF;{PMTg>vt)DG=MrUZ;>Fx_z;MO;sE+QivRg!7rL7o&AP!Lv?AXl9?z;(TH2t8MA@
z62FEQI;Pw9`9HGvvHlD&+EY<=jL5pcPt`<1SGar4-Vd5N>lh84^mf&tx5-ej=ueU{
zV*=XsXjC!QDaJ2Ca*@V_F<`eunC1$g(foAaJFYk6U%`!l{2|;#$l1~|1;6#TOhfy;
zdAh3(jWKm~m7zZwo$I;yKW2rqi8zWG>8v1*X8Jl`+A{Tt&bPJ^x+Ld&+coV_=Vx1C
z>+J((q48n1SUD{xbJt4GY?ooYE1Kr@f`UG
z8OEX=Mwba*z{x^M;9$X9($EN@loSqcFNyD~ytfnqzCKbU?Ocped>Qn-=1KuibFaW?
z8RsQ6qJ?Y`9xulAKz|wy45XQsB22NAkwR&;V%%XagIUG{!)a4J!dKLmhrx`q}a%c{+JO!@SzjBrq
z({;0)DYowEvl_S2+Uh}m9^+z5-X{}#T7UA8|$-aGHO4K5>{
zGr+|l>1Zv!3G8@5%+ExwISaMcwpvF-4yq0t>&0pg*k
zNY@`*P|9d$ysb;4yN>0f*s;1{kLK2N&}0hRw5ZLt!>9XAKGGF&
z0n^`AUp!?@f+oX_!y#c?Ju;ZMM^}Vrw9{M>q7pXPRZE<(ey%glw!KllV}_LtFx&qV
zM9oFJ9edJKj$MeLAxY#!
zNHv)24bSP#4#synSd#D^vn3fUDqDsFFPMh`n-$|#Q!+lCXBsJKX?If+ysoUN2mItD
z<0qXFXEJ4^z2bQ0Bcfn1li~^#qYcTfkG9Jx3!T2U)s2QZzgd(0FFX2LTlxRbK8099
zIr}=Syao1jWc#y@RbVmP8G-t3q-2IykCZI%>fw@!W|0v=Y@Ms_ay>SFw|%CbTD8)Zc`i4n4fG%z)mGTIK^IOrsG&
zSxhx)4Wh$anksf+uDW#M!zMReMlmPqCa_iGJXe%hDs`&srp;Pukn4o4am_K#yVjG?
zjA+gDe`p^>wufr%-GsrK3$XPqL`qHQWInij5mrF*AN)$m3a?u&*}!X^BvPJF1AgK*
z0MBtLkmMno0!y7_*B04}T?6b^+!M)e$8U)x9lxz<*@4lr&FR3l=A!mT(+c>fi21DJ}70
zQdDV)6IX~^vd
zxT_dfkwesRi9uaGM=DR}Gp!{%{H?l_AmW|Tkl}){
zs$>zvjMaf=gD-m4>+3+O^}0HeAimO8mtw?88%s;I#mO7a!U}U1&vdP`{aa_2bDuRK
z;EtoV^<3axdo)Q73%7sfJ88~3niqR!QPw&H?Oi2VkY^^)O8b$+K(V;pxEL!EY^>=U
zL6In3DC9^jX&9die9z?pPjlzME1b*0cYd=M@tvb=KCHHe&A~XmfdV~a$;EdLS@O`6
zYPl*kqk)#|QVV*mB7@D%q1-(1AK#5db>ukalU_XZfky#;<}i_p#9
z1-lmP9r%9(_9a+HnC`*{vY4xg({;>j=^R$ranfP26C;A}JRC8#7IlUez_q#rsjYZd
z+Y0T^v2COd;`itZ(rsJmbxE#x+vx_goD-~{{f9XFS_k_VIBt>t!FTLmgt{8Wewy!~
zIpwGcJtR8!Ko9L*MOElE*yRR!0H)K(L$L0Jc5~FCMToOm^w%QqA+u2^!XV)@`2OI(
zfvKA>C;x@qQeJLNkD1HMZ4hx4<#zO~=_=OWB}`|jqo^2r183_KrJmxos9{oHF(ZDnV)jY7$aV9jWum}LktT^&W~4M2I07v4
zjAMbl4O66EBGFBidW%=J(}9m-X8@PQ%#{+wvYOdos@XhFY9;R5>MgCX^qWG+en7rsKk*GIw67PYYQpUU`Nx{mj+dy2YD0T{fNimdt%Y_J(aBxuNZtVK&Ku=!
zv(_wSSM53}
zNh}qYCJhr)V^XDL(G@jX3KtJuEDRaz
zSc}#0oZ~h)j&hEGO$WP-$p1R{PaR?aeiRtFGb%S;?m}A&R!ANsGzH%ud~+~;%(swx
zP=#*=T+g=$yKa0Rxi8IT6Cm??mX|4A4PG2w!fNC=I)(L>gAiW{;PT4c1*82iwU%3A
z6}ThaL9LaCO!61ART?Sg8n;QqMU63C8Y)^0JEb9FbA1MSR_Q#_EU|VOS(+?%-;^kQ
zx0T;oQ5tBwSvSe~o1`L_b{Dp4hB-TeOOneUTqtHaM_K?6lUBghBmrC|pxp%-wFpJh
z5*SVX0wbr;1zeT_yW>oz2iie`0LhoL_2k;9*gMMcw2fsnKE?l|0Oy$ogJ~9XMaqI7
z=14Qe8Zj58dE)8dd})UGaO)zehFE4>St;JutlkipYHjb|&vnE)#{Y(60;wB%+A)~)
z3cZdMYNN(z58++N0}86vWTzGK_j3KgYBkJN9xTd8i&2LQvWT3J06)1-`arf*WFRn^
z3rjS`
zl6~o2%OZ>uZ06;9bUbrSS}W>dm%mXrUWDd@jrr0v@rL0dqE9dszzV7Qo6;Qdy6%=V
zSIpGiMZ1^w9yCxc?je4a7E^?0qFy0p+ic54B=%09F5R;Y-CS9UwcV*Z!FiiJ3p(xC
zi`sIyvk6$2l13r>^_EKDBz!DbIDaZQvEf
zr^`c8W$gx@1tufgXUfBA2ksy+7?_M*XAz!Ycgut6PPPNM4`I!6HTux934T7$ybatt
zn0H~8x27Gy>83r00KB28qBa!ERviIhKx{h6oG
zQlz1hyhXg6c1da`y3_xb&fD&8t|Zm4jcYi~waz-N;biA|VhYZ8P*h+4v%llVYtA4Z
z?_f9T2Mr8y-G$w~rLNG!$YMRD$pdHr*$DI_TjhZiPnpSsP+ji;i)F%2-~eGav=J=q
zgC2hHSXGkXS5IKYy*8vCP9R471
z2FU{UCWrB9OX9$%^@$T$i^#AMsuaj}QpkpE%LEVF1BKi2IP3`S0|U8-z?W8{n>Xp(Dy
zRabww6Yp*fIq#@Rs)r>x-y)hyN)eF1le8NV)JJjx|0|M&oB-nO(7^HJJg^Tr2W&}R
zK@uzfMjlTi_z%F3+-Klv;6%EBE1^tA9ko@~(NuPW9D~eo2ic>u`5iL+Af}CcRx~pf
zdB0ejG07QXw$Uu_5$glDh_MEBFEWO&d;m|%Tm_?C@*RwR@FkQ9@Cq;BLEcB1NayiofL-_s$|NwY0$v}vYQR&R
zuQHW(<7z2WXarXW_=l~lOrsulC)TOy>=xOd{>7%sb!jB*6;F?t-^!h^NAps8(mjlq
z(v4b~PBQjoyG(MDn7i|;blt{p+bgZM4Nk2f9k8BkG|Clgo!W4mb0Rq&Qef}L=V|)6
zmKIC?2mUrol3=5f%4p=|(#jar59O3`s2^%V@_K?VWd6ZNDl>4-W>jX;i)^M`iw3bf
z5Rd<}>;x{cJdt_oHhU=@XtcQ;6
zL*sDSAqE>(%DLjZt=FXEw$9snNabwDmX(**TPxQc;Hqi;%YU?U04WR^?mSNLv^b*X
zCYdg6D3*38&QU8u<_n}Iq|GLN%2-?%sEziPVq8h;pgo<`Lw`TuINF8O1&X8rFq-%)
zf_Nq!8KLp(B)h+V1KR^Fr4e8%)|ccj9edQ
zrza5gv6k1e7UzvI$^caFO@Q-Ff|5w5Gd5)iy!1c0D(%B$%74U;%skmGrs)5cC2@mp
z3GhtJYFQB{M5M{b#k*m<E
ztPuoFo;{3D)g(C&A>w?bJFw+M$%`duE2$+&R|j5v|GL|YM2f3Iq|JA
z4RM!a*a$g;fG5P6`c3kG;#}Q!`CqZ4HXZE`u{+UDiOG<jR0JIY&GYvPaGmuk7w3
zQ}OoBIZ`d#mwMjPKx;~!VXpTiC19d6o_q-D@9G3Amy*tc(JW~!Y+d9N2se`C*8)b1@zM_VJz1fySks1C|
z7j~brf}Ue5E5URcTLT!+zCqk2S-h26w4Nmm8OF!F1-V6KUcwKDGMD87F++D%zAS#$
zUI)|Xu{VLsV(!R!;*zLC@-#7Z_fDz4?RSF((mQM8jWeak*8a#RqP1SF0WMtQ2rO`P
z7A9)?xZ=RtN7@U#C@Jv35y)NnWFpv|CNsfqFPRPBOCfWC6Ug7dzGN3ztm88wc_;p`
zvYb}qosjiw{b|ms
zWI<@YBTks6DJ89i0y%&3M+zSA
zHS`sC8F-#6P*PCu-B8wIpL-h^$lV7%V;?H(aI$n+S&H-a0Hp?fW2viDrb{iCVcEv!
ztFY-8QvvKd$&?T4YfX8u(Ne}&`Gd-~l=57>ZY-s|z*<~Jc_wNM<$;s+6_h99Ze0cC
zvDiUZQF#R0RZ$*@|BI~-jEeD7?u$dC8Y)l48{s~RTa@=4mRpGb?y4nOtc~=3Qk->{
zu7PB>_R^J>s#^Os9Oc?aIs{F2{uF$|`nrnF>gP(4&_k}Y9C|nd?zKoREb>~o2)rO%
zh6WZDw|ff(XlsN*Wfe4Y4;om@KL-BAe^JtCJMKF$fcpvj!IOz!X$)*POY(!v{_+#ZoC|HcNj~r}DS-SNi*XgXhW2!F19t3B9zgOi
z@)%f~d{a_k(^8PIIqw6E<14r~;*7Tn@GNi>UC7mTZ@`to$%>ZFWJfBEv1?9LLMUac
zxtCF$xrTc=%`;U&`)|`*WRh;oGvv(HOh?$-%=A{gXwRU&3htoT8m{2>*1cDxpVq$V
z`;p&bH_nsRSie@B>ikzw!;U-7@Clm1u8+v)UQ(Z8pZo*Nl@h^=m2qW?Jc9hE$XDoG
zB-J5p8KDO7C+`OdPw@31?OI;vPRGvI2>i)ffX7*#dke1jtx*i9CB`ZKG?pEz)Wtb`
zta~*bV;+zEpkW3o73eQxSEUs7Gxh|U4Sj*L_5BoYnyKp#7M-<&V6_LaiNI?yNx;yU
zVZh|5v0(Q*e2P+<^68_LZ{m`@5z0ET&fYr8G|_)=s8rkP)(1<~t+Vtqu!?yal#y-|
zn}4!v0vQxM#<>npg$#B*L^V|)^#Ie?SdT7|FR)i02?RTD5(NAqXuwet8pAG2gmAR`
z2s-yxD)C0(THfN`PJ8hJFb1VaIxN`2y@Q?s?nDil03675bnnJ4tSj&syGzkw7uE^m
z%rbX!ug4ywgL@tQU`l{a)|goLQW|J7yXVtPW<4}klUWQceKAf}%F!9dNlJMdVw?(W
zXqW+9t)GSdrMg*48G1@P39;EF!$&$#=KBwK4I&qUy(AWPoQ<0K2ElnCIJTD`!f?*;km@O^<_xIykc;5f*=
z7aS9T1G(Yu4Adv1(7&3!p|qg0*;Ps%+?6p7zR}9u%smVK$+*W;fpIJ0)SJmr8q({=
zRHZ7_8rLaRXfMM?;AH(~c<)Btex(u}7(P)sD+Xi?Rt}4!G6qR4tY7sS=_jdVSSaXr|4b{B?d@*PY0rJ4b*Q{+0lHxf*z!^3ZoJSiHhp#i!*LpU(C6CQYg*r53o(hRJ}
zlje}E4`~6!m0$3B%qIg+@FT!tEk72#dht&ck@jb=DhzGP9);vl>_#w6GcUksuS^r%
zt8kt70c=`=DO95AJEI$n`e=_T)#&5cW58N5r+{sua+KRyQBEhjWPG(
z)iKdIh;i@Cu2MZ~B2HGzT1Oi==^GhgXe`|!R)bM`g7?qsrAK)C;&dt6x~9IDw2n*)
z8sv&0x{xu>7-2MOfW3&{l2RpDe5RBMyoBssbaJu(8eEO
zgFBP96gIo};Xa&n_W}BZ-{C%pDsDIQvyRVzW_s}lp@$$|f@bb=3h*>116OjY+ktBQ
z7<3iNMYucST6RS@fmhr|dSmY
zjj>Sw3A&oC`=sD*3hg&&?0W1E;K>*-cYXSARB7Pa@N({YG$s8r;$mCIJc+YzHH?-b
ztVssF^glAu5ROQ%ZLmT=Ep#^N0r8D1DgDN7Wty}OddQJJ1KY@}pog{4z(SG=oKDh!
z!^j3m*blfD*KpI}&kxSP>OMqhUr4Klt`
zzKRk1ZliKoyDv$qY+Vl)SBX{MQF=;7#Lks!SoQTvNmIzF-~p~`WJ@Tn=J5SA!(8~^
z5V9TF^*rR5TEIx5%!QwVo
z4tNsyA9jTmfPJ~j;1$GGL4Pl{o_d0YTgs?Av6_@pw_|_drEb9$oTsP|dN79&vn81A
z?nL_9*wGzF8yVYzV*^7w;97k66U%(go5=UxX>T3erHbG-RTy6gk{@r0bhdO;b|34nNG7&VzM3xi8k8Yp_%f
zc@3rm$vZG@LB4`x6!B7JT9T9mz7o7u1?SH`svB2Y%K!%nMKL28IsnR`iBxaU$mt1XB=!zvPB4)V02>(QmbxQmw%`B`_^IgdjcsB?$M+cc(oB~fmeV~8*9AM7k5hAL2CG%`
zaHUuWoX_q=^tECm;l2GWG58g==wSUY;4ymA9F37XnM=id77CAyTM>Fh59fqcXQgnxX&%nCWZsZx0Cg_
zp_@L}9ddJYh3+71dO~{;9IMA2avNxR%n{(CsQ=tX+9Gp4vgK(*qU3_dj**qOE~a-ClK&;3KSgM3GrYTC^QATXMA(8lllJY
z4V)GZQg6{&$gZPc$FeGFJhq}b5!Y3N)MfM&{4s`STW%cC0lbCt1e5v?Ee&5?Ltk22fbV%rA|i2sB^lllZyAB{!!08b7bnakF=K*x
zD6-9KQ+IU_Ep2MAZa}^bR)^9+V|}$Z?P>6Z9=d7Esc|%8-&I)inIR2~dc_qWDpxiv
zE0rc6qC+Kra$8eY`XMxg#|HD0!x;HmvB$QAm1`^KZC6Q}(oHNNJ-}-v=?z}vNFVU(
z174SL^`QuBk)dhJi*CKU_UXqfJTbUfA8H&ZHr*_$cgHL)Gqdjy?po
z4vY%{$NyqNVe6cz@@gpkv~LkA)xQks5>7d^Go=aSLs(C#Em_-mh$~9ytjU+UgWYhY
z78q?*ngbPOh!C>bEqgcTP`
z5wyfKN=?UEalDG_1xAXz6k-(BG4zbVrjA5LvLH^c>5cI4fAj{m7hRy!t36P)>eQ~Z
zgI2G0rtf0)>R)tWj9%?P>qHyWcJyPY0oquau7l^V*i~MwL$B=D$??|u#z0wXH5sqM
z`p*sPr4;;MNF%wLH94hFIzwJIN|rhhsj;`@5xn5xABtC>1n6q39FC}Os2s9gA@Rx$
z;4Nh;G&Tg8Di8WOhnjE{eB%_E1I-9z9x$5Bhga7oi=m~@LXm#D@f(4S`K_?{8!laa
zhBLIC(AXkwH>^L9OGLdy*}>{&Tp=9<&yQk6sn
z_N3L~nnD{}W2|6$Jpmkybm3p;=z!Y)Yuj4wi&-vjwm
z_;Ko1TnpG#6#J%4sF&uL(h!e6rfrDU1I%FLu0_VKkam}@tC~cwYrCm^>F%g*h|R^}
z-5~jeZ5`B(bX*3b>gd+}ho$x8Z(Qx&L+pX0B|`inM@l9#LX(75vWBL#lp^Rf$*#9N
zubGJ|sOXF+7S_pAT7bnGu&|R9UL#
zAqT+gpl}F`mI|9#u5Z-Uei?Gtkd%>VST3qhho}GmVG?Sck~4^(!G4`l-t2Lnkk}0_fx{cUAog
zjTHj>aWmDmbSOIqYf~?1=>Q#RQIMr4n;yc$TQQG-KaIDc`Mt(@(8Ch_3~>LVoeu8$
zxOwVmdS=HoFnyIVNgYT%nZpp*!TUR^Mw)t{n;c^eG2TQrt82`Y){^PE0x6B?it)Af
zGJM9Z9pO65M`$)92J?vVM`41CTP
zfY&Ad3Q*>+f?W!K9oUT@f>>L|rl~thYIumA*Rh@DKji%STxl|C8$Lv;Nwcz^_>+dCG;7D=>tgDbh;2YsStQIPCM;6`B7WvpF1@6Yef}{$#ag!1^a9zeqshzj8gcF8`Gj@0jOPO_2YgQp
zjlZFd`o?d__9cwz>QdU-xLaLG3k`?V`Ba;6Q2m?E%-pL^r-S!z#EdWYFNO^35B#lm
zp+^o3SIzX+fvs{c>ulp@xx2NAF-X2i^u`=%8vZ}&vNW4S>zm7u$%B+$uI2pCu($Bo
zOW^pJN+cJ2kAUL~tjmS!E99}8uytSZu*g@*Q{Y?SIjng>c%^>9&hAZ-uL@qCU$`sG
z%kz_(fIqM==#Sdz1e=Jww2xg5nWwSaP+v8LW>S%d^O4gom|tPnP{aHM8K%Hg!Sjm#
zVQP84(k)CS&r5pWZ~{8i8jhoSnYzaTrqvJb!TL4-;09#8od-SgP-{nHy4(tDP;2=I
z{<-in?7QBOCyj&MZ%HGFHX={zPa-4BNYO%btUmj2+K?xG#TwLBo&;NOmQMm3DNSMP
z`${=z;1$?)Ca-}B
zq2r;4Kj6CsOg~|rD((4#eNcI@?nNqsbxYz4rtgGMXnq@4gv+@A#M)VQKh}}WYyj+4
zt>T>!7|bg!wSr&7FwY3lKc@5NC3_O8)}ampKmJU3_^Lm;gDTp#GUMwjWsJvZra
zod!L;E(ARuZ9`8!?G;zobAkRF>+3m5kH=K;WaAo#kLL)z9rZ=sNN;59ltZl=<5bxT
zti$C@{7YgQ+Pwmr$_EG+=p()5bsBG_1g!i}xefbzAtH`}(MPPsI!N-0J%UWQ8a7%*
z%d-y&U1^F*PA{6qepe3=PA|d%-}moThF7?
z($VET>9j{iY0nn=_rA|KU3qb!k(>gr`zQ@2RSf0j$@pLIE%FyqG3X-JzO$h%WU)AQ
z)3<1s8wL
zjp*Kt2wH}_Q}R)<$Flda>V2~4Js0T^eMir8`a&ys3hDef&U2MEi8G^LiqUz_({@eb
zJf~>I=s-^vt+}VJXAiBPnITuV8jU}tL1d$$F64i%oi7~~s%U(qvwVOi$)(|g!g8b&
z$oz>)bFAHul>eY%tGf?N81L^5>Mh~Vx
zOjhEc$vkC+?J{|)>?zjg5B%d>QA{C+L7&;AJtQnedH~IQZ{P*4A8!{um<~c-nm>NU2`KCyZoenj{LWi*?okyX*$fXDA4yLi^0QT&h=P34Z
zX87}KV_Q!S*3BV^$8#|w;Q42wMRT(XsgSuNLOv5)D(Ajdr3?9C&)7ycnv
zKs_bFI>A^&Zb9DYV&v|`7F$*3$krIFcf$KHU*#MmC+<72ND_S2mlVnQMi^EktuPrq
zA~y-X)R^=3EXA2gS-Kb_V8`6!s{LWax?r(aj@K!#2BW^v&d}C2&ukM
zM^i@X$3F^diHI&bU!~Cb17!qiNWpy!njeigT@7}>ur5!4y?3}cWJ$W*-vI^V3b%B{&utpzggiw(r5b>pr{rv-ma9c3Tn
z9Ho3m6hBf1K-w+t@?sux%UM`tG`!aXIe*~Zn|~nfA|cJ=9siQs=qVL{n%e?g!EHi4
zFqqp6Z?BK`FWf=o=b4PtQ8S`=pTz`!Sc}TxAF8n=!K4=*!xf`k|K$#M_UATCC8Xoi1;P&C0AVLkQ{4W>?*WUmkbFMf#u_1W3N*Zkc3^`%qj8?8hv$5>
z5cFTR48=O=F(-L;(LLr~hz^$_7K}1X9g*ug8M{K0)r|L~(G$*kR>wSktG|mMn)Ewd@MS$ShM$Ihu?&
z8(^vWW}FL=S0)`sd&kU1q|Y$U#az9NxTl?j>z2z&q)?lN{{Mm#LfG*oE+lFjfymHZ3tS|P_%KK>w|2PAxsr%b$8F@EH%o*8rw+tM=^
znJQd1k(uTe@bh`F7e``EBjvwHYus&^gnv)mA`c~fwcCLoVmCvE`>~JZjd(xyH0c6=
zCai%n1-5&sM1i~N{+}%uae5w6a2+$^{Ya>IB|Hhzf~1>=!4I!6qp&6(!~Lj3$qwUo
zc{qtQZjk$tkGOYn5Lv8iBiAIpVb`QiIQ!Zl|3pr>sFcUZoO>`<$!*0PkAq`Xk_SC+
z67r$vzQPrFY^ZPzdiE9y;jv%&As8**k__~-jFX2EZ?tjWn|U*OUK?>IIXPq4gdWzg
z2EMu@u8Ojo)Cg@ZAI5&CmeLoDRQG9kU=C(k0{lX|l5<#>O~7~9&lY0OwL%yQznaNr
zW3>0$&G3^)ab=WLaw%XtJf~!6OZlfT8CzA?;nZAAI04`Xsoho#B)pr0|y
zdW>v!H$xWkxDP;PAN7qb2V;J~Nz`rZ5F^M}#5L}P&i0NM_>w>^?+rZ3wUxV(IBp!Q
zJdzzH4jWD8^mL#qwg&B)1@nD&aDEc9MU+2vmOCA072tRO`mO*(KJ5TPqfCy-0KF?Zly6?G$&{e)|f
zE*IS!&`w?>bPT&Fo#Fk%5|y5?)C=VdcoBCcSgN#o5K-xs{RsY0D*FjMup-*MkV8h0
zuH00Nc8X1dhAYD2MsnMC+%7~!G&dXC_>KGLdype|cAyhU#Vq(cV#``sA;Fv`>qr3n
z`Kz!ubQmhJt61^B6x$*dK5SDz!y;bUS8yGtPWE*~N1g0K+!Zh%>x;y#!I;aq|Ku?w
z26OcwUokRAj#yU6QHYT=xe4iM-ih`4kEyKEo^bl|z;(Knffv
z(2|Xo{lF2H(aIF^Utk+KO4t#aCk@~)hH&n4m@8hr0QtMBPZ2MbJYO+m*=z-GCDLVI
zgO8cOC4=z5CBz0yD#qR<9__|tC;V+DcL?K$;~r@(xxmy>nvhN*xVA3bheht7GVxK1
zysMLX9pevyCtSdu?w*MMfAf^V8mIHrL{-oRb5$q*Vg*U(*P~9G#;<@~M&k*o1QN%e
z$NEx<&p=N)o||Y!TIdyJ6S)|jt7PIz#1N%F`4jkDnFua+cXe1qR1X)$i01-k(Py8<
z&g-x20$lsfLcaZvmt?%9k>3RUOvQ6pB5B2+f_1{VQc5h5nCgm_u()$rM;b9lk^keH
zQym9U*Py_dFX~TRDaO)B~jt;yTOy6QkkEEOb)d^PiXlZT!T@#%vXl@>li+Xg(?X
zGMouwmG!y8#iW%E_DnkE>**UQ92+SCZ{j7%EN8W2n4r?DMDa08t
zN}*(Au%Gfv$PQ^PpXaTiekz5YhpWe+XS3%O#xZ6efX@tpwzrVs+2?TvH43B6BoooL
zk!#2p5BW;USc3OaDHdYWJyDt=OL*Lmp$8u|09AorEvgErT2%k+P|HBVN}jD^E_Tp2
zFn&mO8m^4b#Av<99AF*t2VQ|UnJ7u{mIturP3{>qu$O%)*Ct!ciHeE5(+^dQ#HPQl
z%p?&pqm($ZIe5F=M5qy3TJ^(>xOxbktXE$`CquK3;>2TWb~ZJVG8pqIUsD-Os`0N;
z!(24`Dz(WBQxe7;YaFUD_#gRU$}QOOKGvNguNVUsiE1h&->3eBHCfMjNIub189AY8
z_I7A^F~*N9Mn94U`74sjN_
z$qL?ArhB1GCes7zDgO#jLL_%l#aK%H1I#S06E3S^;yE(j+1(GL?RFo4&t$0gpqYzm
zZP;S0=NI-wyF3A~!qDu!xH_4Z?V*Fn>g*%5CfSILQH7LM;z@1(H)cuZyrAulh^aWD
zLEZ}|+gKm?RVrIrX-G=2Hl+@E&!mFa8LiQsN?L|&kw5TRAxX*^avvF~8LX42PK1yB
zQd?s7%^p9@o1C2veeTGX5zAXJZ$q*n`!KCcHUYm2B^8#mp4jrQD!?J~0^XW1V{xKP16V4KPlB!7M5QhH!w&@><3}iMAo&m_f%M=9BFlX@B`Uv#d!}1T
zBeFw(M`=L3boJae(l7QVW_cZ;b63Jsu7})xA>jx$5?bAX^|}CZ9)`3%U=clw^@6-W
z7WyXmn(xEgH|pZt&B)Q1f#6sWWkh>GgN<^3zG1Mpx)K$IQN0E^H>kZaS|!hC@YQ=7
z<6i*0j_#!zoHI9t2aW+AL;f_78o&VZFS{D!zhdq{&M4*v=6zL-^h3KUiTgiCXB{8K(Z+GyokDS!OR~9K&Q@+^
zvG@~D7A`~6o|xtT(|_5+
zNCG6OLl$Vm5#xQezG(T9Itn&?uH1#z-K0-KaYtBrZ^>fUIsf5&a#&spAh8s5&pZZL{t)!|)^{2`XH9`MR
z*{t@`zbQ#-4gIT5(jF3KVNCx*Ym2p8U=J+{x>WV_
zRf=d$Nq?oFW+f@;w}$>N7@<9egxTsloOx9JigJRQAm3NUDplorN}SSP$*q;bRqxd2
zh7U*IX+fNM8q896Yk58A)UL44d-br959ONbX=xv98-w$g7@5jVcu`)@dFT>S2B|BQ
zZc0(@gV9fUtM)fyl$Y?#1Ild`b1SY*fQ@gf;hLsY*P4?)N--_d=&QU?pXw)-JL(gR
z&vW&do}DAPr>9avTa2U6)YE|-S}GM4w-#nplU2_Wr7G&5=o6(F<%Xvs$~*NWrMPxl*?~42DdkZU4~-ruE0uD%
zO3~X$Rh5p4fR@{6b{w^6JLNd76FDupJg=3JsJFtC$7)?kYiXo1T=^y*mYXXN_ELly~Af_;qO|
zR-1&@)+*z0<|U=N=bLy!Ij2O+yR;YRSucWiuP*Pl4;8~~)m8iN(sV0}#`YBn`O>H>&Nj|GJQX7JH)o@>s
z+bcx=OMBsI1SWX?lAKyArKx;XtL|wj|BW_VpxjzHiTc(`W1MqMUN05XVwGQ|T>4n7
z(Q@fj{RHO!&Ig`sloLE>rQCWREr;LKzfrSEy(#6@XR3MSyn1akRn|>vQC~(pPn)@1qe<87wp?h$qq0dr(mUO+FIuo?xOhezS{DyLSWh?ki
zE+Gxl$B>PvpDP!Y2J5}CTUkLHpk#|<^*N*fjy5Ajq*&cb3WKFcNz{BW%A%LuMky&y
z|JC>bHT{i>IBGGfqJ>L(6_k(Yx5Pw!A*mz{(G&FQsOh3xq;>jV`dF!}Zj+yhyI{kI
z;x_d7PTZ<9@>SfTN0D+kv!?z5H3dm7DMinRJ^tfbzN;~vkkRb*AmWB*xwC8Afnej>dP1kV#X5EJKQ}uYgSdPqEX~-O-l}9;3
z%aQh)S_3?!hJl^cLa@M4e85+T@EQAJya=qyTLsKO_ZdoLqnwp;eV#v^uzxR-m<%6#AKjp?sgz
zm$vAqJQiuaKH3v5CFvDCRpe7756za>)2cLF$)u-=P1!?}NgcGXmej>n%ZXJ1IW
zXVRDCEzVg>-h<1@CwVJnIDZqZjNPftv;rxhtfQxmB1$5iXcSl0(EeEcCD6J?dDN6Q
zYAYM*Q@xI|mLAf>mEY(}-G!R&dMQ|VyVB2{OB<#vkOxxy+D_g}E75=DJ+u^kEa!ZS
z_L00BI=(OY3`SL7Uo<5b6=_2DH?9kSD0y|iYHzU(a*>YX%X#rS)vEk
zgR+Arsr%(fI$b>gc2f__5!9ycm21)mo`V>Lm7ZgAHTr|+2lqX7lyXWs2k*KFeVdV-
zaapY&lf&p4WuIJ`Hb5q7d4}`0GF_=GD;Uisat6HaD9(Rp9D|ik8Fz72cl|!Diq=2MsZ`eD