diff --git a/.gitignore b/.gitignore index 31e8e67..bc305d9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,14 @@ __pycache__/ scans/ *.mov *.mp4 +# the one exception: a 3-second synthetic test pattern (ffmpeg testsrc) the demo page tests play; no home in it +!apps/web/e2e/fixtures/test-clip.mp4 # the sample room's photo colours stay local (the viewer hides "Photo" when the file is missing) # your own scan for /viewer (make viewer); it never leaves your computer apps/web/public/local-scan/ +# the real iPhone screen recordings, cut for the demo page on this computer only (make demo-clips); they show the home +apps/web/public/local-video/ .vscode diff --git a/CLAUDE.md b/CLAUDE.md index 75f96ec..5d2b037 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,7 +106,8 @@ domain ← application ← adapters (inbound + outbound) - **API `domain/ports.py`** (one repository per aggregate): `HomeRepository`, `ScanRepository`, `TrackedObjectRepository`. - **API `application/ports.py`** (needs of use cases): `LlmProvider`, `VisionProvider` (Muse Spark 1.1, tie-break only), `Transcriber`, `BlobStore`, `PipelineTrigger`, `Clock`, `Segmenter` (SAM 3.1 over HTTP on the Meta API, for the Verifier), `ArrivalVerifier`, - `PromptScreener` (Muse Spark Open or Glimmer, after schema validation). + `PromptScreener` (Muse Spark Open or Glimmer, after schema validation), `PlaceMatcher` (classic ORB + PnP photo + matching against each scan's `place_index.npz`, no model; `POST /homes/{id}/locate`, ADR-039). - **Pipeline `application/ports.py`**: `Reconstructor`, `Segmenter`, `EpisodeSource` (reads from R2 or disk), `ResultSink` (posts to the API). **torch and local models live only in the pipeline**; the API reaches vision models over HTTP only. diff --git a/Makefile b/Makefile index 8d09d48..b959cc4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ BIND ?= 0.0.0.0 WEB := apps/web REDOCLY := npx -y @redocly/cli@2.72.1 -.PHONY: help setup dev worker robot-export regression viewer public-scan live demo record probes diagrams pair rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval +.PHONY: help setup dev worker robot-export place-index regression viewer public-scan live demo record probes diagrams pair rerun demo-walks demo-clips test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval help: @grep -E '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | awk -F':.*?## ' '{printf " %-18s %s\n", $$1, $$2}' @@ -30,6 +30,10 @@ ROBOT_HOME = $(or $(HOME_ID),$(if $(filter command line,$(origin HOME)),$(HOME), robot-export: ## One room's memory for a robot (Nav2 map, things, furniture): make robot-export [HOME=flat] [ROOM=...] -> exports/robot// cd $(API) && uv run python -m recall_api.infra.robot_export --home "$(ROBOT_HOME)" $(if $(ROOM),--room "$(ROOM)",) --out "$(CURDIR)/exports/robot/$(ROBOT_HOME)" +place-index: ## Build the photo matcher's index (ORB + LiDAR depth, no model) for every processed scan on this Mac + cd $(PIPE) && for e in $(CURDIR)/data/blobs/scans/*/episode; do \ + [ -f "$$e/result.json" ] && uv run python -m recall_pipeline.adapters.place_index "$$e"; done; true + regression: ## Re-run every processed scan now (no posting; cached SAM answers) and compare with the last run cd $(API) && uv run python -c "from recall_api.infra import jobs, regression; print('\n'.join(regression.run(jobs.settings, jobs.run_pipeline)))" @@ -55,11 +59,18 @@ viewer: ## Export a home's newest processed scan for /homes//viewer: make live: ## The API with real Meta model calls (keys from .env), and the scan worker ( cd $(API) && RECALL_BIND_HOST=$(BIND) uv run recall-api ) & $(MAKE) worker ; wait -demo: ## No keys: the API on a memory built from a recorded real scan; the Guide's replies are replayed (labelled recorded) - cd $(API) && uv run python -m recall_api.infra.demo serve $(if $(PORT),--port $(PORT)) +demo: ## Judges, start here. No keys: Recall's API on a recorded real scan and the website's demo page (localhost:3000/demo), told step by step + @python3 scripts/demo.py $(if $(PORT),--port $(PORT)) $(if $(WEB_PORT),--web-port $(WEB_PORT)) + +demo-clips: ## Cut the real iPhone recordings (docs/private) into clips for the demo page, on this computer only (gitignored) + @python3 scripts/demo_clips.py + +demo-walks: ## Rebuild the demo page's walks (public/real-scan/flat/walks.json) from samples/replay/ with Recall's own code; no keys + cd $(API) && uv run python -m recall_api.infra.demo walks --viewer "$(CURDIR)/$(WEB)/public/real-scan/flat/meta.json" --out "$(CURDIR)/$(WEB)/public/real-scan/flat/walks.json" record: ## Re-record samples/replay/ with real Muse Spark Open calls: make record RESULT= ROOM="Living room" AT= cd $(API) && uv run python -m recall_api.infra.demo record "$(abspath $(RESULT))" --room "$(ROOM)" --captured-at "$(AT)" + $(MAKE) demo-walks diagrams: ## Redraw docs/diagrams/*.svg from the Mermaid sources (mermaid-cli; set PUPPETEER_EXECUTABLE_PATH to use a local Chrome) @for f in docs/diagrams/*.mmd; do \ diff --git a/README.md b/README.md index 8b51e9a..cbaf427 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,14 @@ Indie track · Theme: Build for Your World

- Try it (no keys) · + Live site: recall-ten-khaki.vercel.app + (also at recall.onica.ai) · + Try the demo: ask, then walk · + Explore a real room +

+ +

+ Try it (no keys) · Architecture · API · Privacy · @@ -71,11 +78,24 @@ Indie track · Theme: Build for Your World

### For judges, in one minute +**Start with the demo.** Nothing to install: open +**[recall-ten-khaki.vercel.app/demo](https://recall-ten-khaki.vercel.app/demo)**, pick "Where are my keys?", and +follow the walk to them in the author's real scanned living room. To run the same thing from this repo, with no keys, +no GPU and no account: + +```bash +git clone https://github.com/coder-RT/recall.git && cd recall && make setup && make demo # then open localhost:3000/demo +``` + +`make demo` starts Recall's API on a memory built from that real scan and the website beside it. The answers are Muse +Spark Open's real replies, recorded once and replayed through Recall's own code, and labelled as recorded; the routes +come from the planner the phone uses, on the scan's floor. + | Criterion | Where to look | |---|---| | **Impact** (30 %) | [Why I built it](#why-i-built-it): a true story · [What it is](#what-it-is): built first for blind and low-vision people, for anyone who misplaces things, voice first · [Privacy](docs/privacy.md): the home stays on your own computer · the [tester session plan](evals/usability/) | | **Use of Meta AI tooling** (25 %) | [Only Meta models, each with one job](#how-it-works-the-meta-models): MapAnything, SAM 3.1, Muse Spark Open (Muse Glimmer on your own Mac), Muse Voice Transcribe, Muse Spark 1.1 · [Recall's three agents](#recalls-three-agents) · [Sharing with robots](#sharing-with-robots) over A2A | -| **Demo** (25 %) | The screenshots below are real output · `make demo` replays a real run with no keys · [How to run it](#how-to-run-it) | +| **Demo** (25 %) | [The demo page](https://recall-ten-khaki.vercel.app/demo) and `make demo`: a real run, no keys · the screenshots below are real output · [How to run it](#how-to-run-it) | | **Technical depth** (20 %) | [Architecture](docs/architecture.md) · [28 decisions](docs/decisions.md) · [Guardrails](#guardrails): geometry decides the facts, the model only phrases them · [Algorithms](#algorithms) · tests on every part ([Results so far](#results-so-far)) | --- @@ -167,7 +187,7 @@ floor, furniture and walls as cells, and the things as dots: no photos of the ho | iPhone app (Swift 6, SwiftUI, ARKit; VoiceOver first) | `apps/ios` (build with XcodeGen) | Works on an iPhone with LiDAR, iOS 26 | | The API on your computer: the Guide, the arrival check, memory, privacy, robots | `services/api` | Works (`make live`) | | The pipeline: 3D reconstruction, furniture and things, the room model | `services/pipeline` | Works (`make worker`) | -| A recorded real run you can try with no keys | `make demo` | Works; the recording is added after a privacy review | +| A recorded real run you can try with no keys: ask, then walk | `make demo`, [`/demo`](https://recall-ten-khaki.vercel.app/demo) | Works | | Website and 3D viewer of your own scan | `apps/web` | Works (`pnpm dev`) | | API spec (OpenAPI 3.1, Swagger UI) and the A2A Agent Card | `services/api/resources/` | See [API](#api) | | Architecture, decisions, privacy note, measurements | [`docs/`](docs) | [architecture](docs/architecture.md) · [decisions](docs/decisions.md) · [privacy](docs/privacy.md) · [verification](docs/verification.md) | @@ -186,7 +206,7 @@ services/ api/ FastAPI: domain/ → application/ → adapters/ + infra/ ; resources/ = the OpenAPI spec (source of truth) pipeline/ the scan pipeline, same layers: MapAnything, SAM 3.1, planes, merging, the room model docs/ architecture, privacy, verification, Meta Muse (docs/muse.md) -evals/ questions, usability plan +evals/ questions, usability plan, results and the replay scripts ``` - **Hexagonal (ports and adapters) in every service**: domain ← application ← adapters and infra. @@ -356,7 +376,7 @@ runs its software or its models; every model Recall itself calls is a Meta model | Muse Glimmer | Guide agent alternative; local privacy mode | Modal / Fireworks; LM Studio on the Mac | same adapter | | Muse Spark (hosted) | Guide fallback | Meta Model API | same adapter | | Muse Spark 1.1 (multimodal) | Verifier tie-break; may only downgrade a result, never confirm | Meta Model API | `services/api/.../adapters/outbound/vision/spark_vision_meta_api.py` | -| SAM 3.1 | Objects and furniture in every frame, by text prompt; the arrival check | Meta Model API (optional self-hosted on Modal) | `services/pipeline/.../sam31_meta_api.py`, `services/api/.../sam31_meta_api.py` | +| SAM 3.1 | Objects and furniture by text prompt, on a sample of each scan (40 photos × 24 prompts on a large scan); the arrival check | Meta Model API (optional self-hosted on Modal) | `services/pipeline/.../sam31_meta_api.py`, `services/api/.../sam31_meta_api.py` | | MapAnything (`facebook/map-anything-apache`) | Metric 3D reconstruction from the frames, depth and poses | Mac (Apple GPU); Modal if too slow | `services/pipeline/.../mapanything_local.py` | | Muse Voice Transcribe | Speech to text with turn detection | Meta Model API | `services/api/.../voice/muse_transcribe.py`, `apps/web/app/api/voice/transcribe/` | | Muse Spark Open (or Muse Glimmer) as a screen | Checks the object name a robot or outside app sends the home memory, after the schema; strict JSON verdict, fails closed | Baseten (LM Studio for Glimmer) | `services/api/.../safety/meta_model_screen.py` | @@ -372,14 +392,18 @@ runs its software or its models; every model Recall itself calls is a Meta model 1. The phone sends a spoken question to the Mac, and Muse Voice Transcribe turns it into text (typed questions skip this). 2. The phone says what it is looking for at once ("Looking for your keys."). 3. The Guide calls `find_object`, which returns each match with its room, landmark, relation ("on the desk"), shelf level, height band and when it was seen; one call usually answers. The answer is logged with the sightings behind it, and **Why?** reads them out. -4. **Take me there**: `localize` from the phone's ARKit pose, `plan_route` over an occupancy grid, clock-face steps spoken as you walk. +4. **Take me there**: the phone recognises the room (ARKit's match of the saved room map, helped by a photo match on + the Mac about once a second, trusted only when two fixes agree, and spoken hints such as "point the phone at the + TV"; landmark directions if neither matches), then `localize` from the phone's pose, `plan_route` over an + occupancy grid, clock-face steps spoken as you walk. 5. On arrival the Verifier checks the camera frame; if the thing is nearby, memory moves it. **OS and browser features (not model calls)** | Feature | Used for | Where | |---|---|---| -| LiDAR depth, ARKit tracking, ARWorldMap | Capture in metres; relocalisation for live guidance | iPhone | +| LiDAR depth, ARKit tracking, ARWorldMap | Capture in metres; relocalisation for live guidance; the map grows on each visit | iPhone | +| OpenCV ORB features + PnP (classic geometry, not a model) | Recognising the room from one camera photo, beside ARKit | Mac (`services/api/.../vision/orb_place_matcher.py`) | | ARKit poses, compass (magnetic north, no location) | Speed and turn rate for scan coaching come from the ARKit poses; the scan learns where north lies in the room, so directions can say "north-east" | iPhone | | Haptics | Turn and arrival cues | iPhone (Apple Watch planned) | | VoiceOver, Dynamic Type | The app is usable without looking | iPhone | @@ -423,6 +447,7 @@ routes match it. While `make demo` or `make live` runs, **Swagger UI** is at `ht | `DELETE /scans/{scan_id}` | Erase a scan from your computer, with what only it found | | `GET /homes/{home_id}/objects`, `GET /homes/{home_id}/room`, `GET /homes/{home_id}/room-map` | What Recall found, the room model, and each room's map for Find | | `POST /guidance/sessions`, `POST /guidance/sessions/{session_id}/verify` | Directions, and the arrival check | +| `POST /homes/{home_id}/locate` | Where am I? One camera photo matched against the room scans: the room, the pose, what to point at | | `GET /privacy`, `GET /audit` | What is sent where right now; who did what | | `GET /homes/{home_id}/robot-export`, `/a2a/home-memory` | For robots: the Nav2 map, and the Home Memory agent ([Agent Card](services/api/resources/agents/home-memory.card.json)) | @@ -445,6 +470,15 @@ Every route except `/health` needs a key: the owner's device key, or a robot's o 4. *Not found is a fixed sentence,* never the model's: "I haven't seen that yet." 5. *Tested:* `tests/unit/test_grounding.py` (an invented place or number is caught) and `tests/unit/test_ask_where_is.py` (an invented place is corrected, then replaced by the tool facts; a place with no sighting behind it is never said). - **Every answer is traceable.** Sightings are append-only. Every answer is logged with its model and tool steps and the sightings it rests on, and **Why?** on the phone (or `GET /answers/{answer_id}/evidence`, or asking the Guide "why?") leads back to the camera's view, the scan and the models that made it. An answer that names a place with no sighting behind it is never said. +- **Recognising the room from anywhere, never on one photo's word.** ARKit's own match of the saved room map is + the main signal. While it hasn't matched, the phone sends about one photo a second to the Mac, which matches it + against every room's scan photos with classic ORB features, the scan's own LiDAR depth and PnP (OpenCV, no model; + `adapters/outbound/vision/orb_place_matcher.py`). The phone uses a photo pose only when **two fixes in a row agree** + within 25 cm and 10°, and ARKit's own match replaces it the moment it comes. The Mac also says which room the photo + shows ("You seem to be in Bedroom, not Living room") and what to point at ("the TV"). Each time ARKit recognises + the room, the map it grew is kept on the phone, so the next visit matches from there too. Measured on the real + living-room scans: held-out photos 1.2 cm median error; photos from other sessions fix 59% of the time, 89% of + them right; 22 to 35 ms per match ([evals/results.md](evals/results.md)). - **The Verifier can only downgrade.** The vision model may turn a result into "uncertain"; only SAM plus geometry can confirm. - **Two checks on the door.** A robot or outside app can send only one object name (letters, digits, spaces and hyphens, at most 40 characters, `services/api/resources/agents/find-object.input.schema.json`); free text is refused and never reaches a model or a tool. Then Muse Spark Open (or Glimmer) must answer `{"verdict": "safe", ...}` in strict JSON; anything else, a timeout or "injection" refuses. Measured: 21 ordinary names allowed, 18 instruction-like names refused, none wrong. - **Privacy and security.** See [Security and privacy](#security-and-privacy): what stays on the Mac, what is sent where, and how the home's memory is protected. @@ -463,9 +497,9 @@ Every route except `/health` needs a key: the owner's device key, or a robot's o | What | Sent to | When | |---|---|---| | Scan photos, about 50 per room (no depth, no sound) | Meta Model API: SAM 3.1 | While the Mac processes a scan | - | A spoken question (a few seconds of audio; only the words are kept) | Meta Model API: Muse Voice Transcribe | When you ask by voice | - | The question as text and what memory says (no photos) | Baseten: Muse Spark Open, or Muse Glimmer on your Mac in private mode | Each time you ask | - | One camera photo | Meta Model API: SAM 3.1, and Muse Spark 1.1 only when SAM is unsure | On arrival | + | A spoken question (a few seconds of audio; only the words are kept) | Meta Model API: Muse Voice Transcribe | When you ask by voice. In the AR view the microphone opens by itself after Recall asks a question, with a tap on the phone, and closes when you stop talking | + | The question, or your reply after a find, as text and what memory says (no photos) | Baseten: Muse Spark Open, or Muse Glimmer on your Mac in private mode | Each time you ask | + | One camera photo | Meta Model API: SAM 3.1, and Muse Spark 1.1 when SAM is unsure or the thing seems to have moved | On arrival | | The room map, things and furniture (no photos, names, notes or anyone's position) | Robots and apps you gave the robot key | Only while Robot Access is on (off by default) | - **Your controls:** delete a scan and it is erased from the iPhone and the Mac; forget a thing; Robot Access off by @@ -476,6 +510,9 @@ Every route except `/health` needs a key: the owner's device key, or a robot's o approved by the author: - points and surfaces tinted by height, the room model, the drawings, labels and robot files; - no photos, video, depth images, keys or paths; guard tests fail if a photo file or photo colours ever appear. + - the no-keys demo (`samples/replay/flat/`) holds the same scan's room model and labels, its point cloud thinned to + one point per 5 cm (positions only, no colours) and the path the phone walked (camera poses only, no times or + compass readings), with the recorded answers; a test checks there is nothing else. **Security: how the home's memory is protected** @@ -547,7 +584,9 @@ erDiagram - **Indexes:** scans, landmarks, objects and rejections by home; observations and set-asides by object; audit events by time and by action; the answer log by home and time. - **The job queue** is a separate SQLite file (`data/jobs.db`, Huey) that holds only queued scan jobs. -- **Not in SQL:** the scan files (`data/blobs` on the Mac, or Cloudflare R2). On the iPhone there is no database: +- **Not in SQL:** the scan files (`data/blobs` on the Mac, or Cloudflare R2), and each scan's `place_index.npz` beside + them (ORB descriptors of the keyframes and their 3D points, for the photo matcher; built by the pipeline, or + `make place-index`). On the iPhone there is no database: scans are folders in Application Support, the device key is in the Keychain, and the Mac's address is in UserDefaults. @@ -560,8 +599,12 @@ erDiagram - **Room model:** RANSAC plane fitting for the floor and the walls, deterministic and seeded; furniture from SAM 3.1 text prompts, lifted into 3D boxes. - **Flat surfaces:** a height histogram of the point cloud, peaks grouped into connected patches on a 5 cm grid, thin lines rejected by fill ratio. This is what finds desk tops, seats and shelf levels. - **Spatial relations:** "on" (within 5 cm of a surface and inside its footprint), left or right of a landmark's front, nearest landmark within 1.5 m, and five height bands (floor, knee, waist, chest, head). -- **What changed:** a rescan matches each kind of thing with the room's last scan by optimal assignment on 3D distance; moved more than 30 cm, new, or gone is decided by geometry, and things outside the rescan's view are left alone. +- **What changed:** a rescan matches each kind of thing with the room's last scan by optimal assignment on 3D distance; moved more than 30 cm, new, or gone is decided by geometry, and things outside the rescan's view are left alone. Unit-tested; no real rescan has lined up with its earlier scan yet, so each real scan so far made a fresh map of the room. - **Navigation:** an occupancy grid at 10 cm from the point cloud, obstacles widened by a 0.3 m body radius and paths kept to the middle of free space, A* to a reachable cell near the object, simplified to turns and spoken as clock-face steps. A gap too narrow gives "no safe path" and landmark directions instead. +- **Recognising the room:** after processing, each scan's keyframes become ORB features lifted to 3D with the phone's + LiDAR depth and ARKit pose (`place_index.npz`). One camera photo is matched against them (ratio test, PnP with + RANSAC; at least 30 inliers within 4 px), giving the room and the camera's pose in the room map's world. The phone + trusts two fixes in a row that agree within 25 cm and 10°, and ARKit's own match replaces them. - **Arrival check:** SAM 3.1's box in one camera frame, lifted to 3D with the LiDAR depth; within 30 cm of the remembered place is confirmed. - **Viewer:** voxel downsampling to about 150k points; a vertex shader that gathers every point from a scattered shell to its real position, floor first. @@ -575,14 +618,18 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima - **Without a model:** LiDAR depth back-projected with the ARKit poses put all 10 objects of the living room within **2 to 7 cm** of the MapAnything run, in **0.2 s**. - **Answer time:** the Guide on Muse Spark Open answers in **2.3 s** median, **2.7 s** p90 (10 real questions, one tool call each); the phone speaks an acknowledgement at once. - **Arrival check:** on the real keys, SAM 3.1 plus the LiDAR depth put them **3 cm** from the remembered place, in **4.9 s**. +- **Recognising the room from a photo** ([evals/results.md](evals/results.md)): held-out photos of the living room + came out **1.2 cm** (median) and **0.55°** from ARKit's own poses; photos from other sessions gave a fix **59 %** of + the time and **89 %** of those fixes were right, so the phone waits for two that agree; **22 to 35 ms** per match on + the M5 Pro. On the way: none of five rescans of that room was recognised by ARKit alone, the problem this fixes. - **Robot screening:** **21** ordinary object names allowed and **18** instruction-like names refused, none wrong. - **Cost, measured:** a room scan is **456** SAM 3.1 calls and **48 MB** on disk; a question is **2** Muse Spark Open calls, about **2,460 + 261 tokens**. - **Flat surfaces:** the geometry-only detector found **12 level patches** in the room; the two largest sit at the heights of the two measured desks. - **Viewer:** about 150k points, the gather animation, photo and confidence colouring, and the floor grid, height bands and surfaces as layers, on a phone and a laptop. -- **Tests (2 November):** API 202 (including Schemathesis against the spec), pipeline 59 (including a synthetic room run through the whole pipeline), website 190 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, keyboard-only and 200 % text usability, security headers); on the iPhone, 182 unit tests and 31 UI tests: accessibility audits and usability tasks done with VoiceOver labels at default and largest text; the Watch app builds. +- **Tests (3 November):** API 218 (including Schemathesis against the spec), pipeline 64 (including a synthetic room run through the whole pipeline), website 203 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, keyboard-only and 200 % text usability, security headers); on the iPhone, 197 unit tests and 34 UI tests (2 skipped): accessibility audits and usability tasks done with VoiceOver labels at default and largest text; the Watch app builds. - **Usability, honestly:** the checks above are automated. In person, the author's screen-off run (VoiceOver Screen Curtain: the screen stays black) is planned, and a session with blind and low-vision testers is planned; its plan is ready in `evals/usability/`. -**Planned evaluation:** median position error against hand-measured positions, questions answered correctly, Verifier precision, and time to find an object by touch versus with Recall, scored without AI; the dated table will live in `evals/results.md`. The session plan for a blind or low-vision tester is in `evals/usability/`. +**Planned evaluation:** median position error against hand-measured positions, questions answered correctly, Verifier precision, and time to find an object by touch versus with Recall, scored without AI. The dated table is [`evals/results.md`](evals/results.md); recognising the room from a photo is its first entry. The session plan for a blind or low-vision tester is in `evals/usability/`. ## Installation @@ -593,7 +640,7 @@ offers the same guide as a download (`/INSTALL.md`). | Path | You get | Needs | Time | |---|---|---|---| | **A. Website** | The landing page and the real 3D scan at `/homes/flat/viewer` | Node 22 | 5 min | -| **B. Demo, no keys** | `make demo`: Recall answering "where are my keys?" from a recorded real scan, labelled as recorded | uv (Python 3.12 comes with it) | 10 min | +| **B. Demo, no keys** (judges: start here) | `make demo`: the demo page at `localhost:3000/demo` (ask, then walk, in a real scanned room) and the API answering from that scan, labelled as recorded | uv (Python 3.12 comes with it), Node 22 | 10 min | | **C. Live** | Your own room: `make live`, the iPhone app (Xcode + XcodeGen), pairing by QR, the Watch | A Mac with Apple silicon, your own Meta Model API and Baseten keys, an iPhone with LiDAR on iOS 26 | 45 min | The short version: @@ -601,7 +648,7 @@ The short version: ```bash git clone https://github.com/coder-RT/recall.git && cd recall make setup # Python services (uv), the web app (pnpm), generated models, the database -make demo # B: no keys; then: curl -s -X POST localhost:8000/ask -H "Authorization: Bearer demo" ... +make demo # B: no keys; open localhost:3000/demo, or curl -s -X POST localhost:8000/ask -H "Authorization: Bearer demo" ... cd apps/web && pnpm dev # A: http://localhost:3000 and /homes/flat/viewer ``` @@ -654,7 +701,18 @@ Watch app all run. ### 3. Backend, pipeline and demo - `make setup`: Python 3.12 with uv, both services, the web app. -- `make demo`: no keys, no GPU. Builds a memory from a recorded real scan (`samples/replay/`, geometry and labels only, no photos) through the real ingest code, then serves the API on `localhost:8000` with Muse Spark Open's recorded replies; every answer is labelled `replay:muse-spark-open (recorded)`, and the command prints questions to try. `make record` re-records it with real calls. +- `make demo` (**judges: start here**): no keys, no GPU. It says each step as it goes (tools, the recorded scan, the API, the website), then prints **Ready** with the page to open; the servers' own output goes to `data/demo/`, and `Ctrl+C` stops both. + - It builds a memory from a recorded real scan through the real ingest code. The scan is `samples/replay/`: the room model, labels, a thinned point cloud and the path walked, with no photos. + - It serves the API on `localhost:8000` with Muse Spark Open's recorded replies. Every answer is labelled `replay:muse-spark-open (recorded)`, and the command prints questions to try. + - It starts the website, with the demo page at `localhost:3000/demo`. Pick one of 13 questions, read or hear the answer, and GSAP plays the walk: + - the route draws on the floor of the 3D room, every point and the architect's room side by side on a wide screen; + - "you" move along it, and each clock-face step comes in as the route reaches it; + - a phone beside it shows the step being spoken. It is a simulated screen in the app's layout, labelled as such; + - when Recall answers "which one?", each match it found is offered with its own walk; + - the same page is online at [recall-ten-khaki.vercel.app/demo](https://recall-ten-khaki.vercel.app/demo). + - On the author's own Mac, `make demo-clips` cuts the real iPhone screen recordings into clips: finding the watch, and the scan. The page plays them on the phone, and the route follows the video. They show the home, so they stay on that computer (gitignored) and are not on the hosted site. + - `make demo-walks` rebuilds that page's walks (`apps/web/public/real-scan/flat/walks.json`): the replayed Guide's answer, then the phone's own route planner from where the scan began. + - `make record` re-records the answers with real calls, and then the walks. - `make live`: the API and the scan worker (`make worker` alone starts the worker: the warm pipeline server plus the job queue), with real Meta model calls using the keys from `.env`. The minimum is one Meta Model API key from [dev.meta.ai](https://dev.meta.ai) with `LLM_PROVIDER=meta`; the demo video uses Muse Spark Open on Baseten (`LLM_PROVIDER=baseten_spark_open`); the local privacy mode uses Muse Glimmer in LM Studio (`LLM_PROVIDER=lmstudio`). - **Where the API listens:** `make live` and `make dev` listen on your network (`BIND=0.0.0.0`) so the phone can reach the Mac; behind a Cloudflare Tunnel use `make live BIND=127.0.0.1`. Run on its own (`uv run recall-api`), it listens on 127.0.0.1 only. Every route except `/health` and the API docs needs the device key, the calls that reach a model are rate-limited, and bodies are size-checked. - **Home Memory agent (A2A) with `make live`:** its check runs on Muse Spark Open with the Baseten key (`SCREEN_PROVIDER=baseten_spark_open`, the default) or on Glimmer in LM Studio (`SCREEN_PROVIDER=lmstudio`); no other key is needed. @@ -706,8 +764,11 @@ Planned next, not built or not working yet (updated before submission; removed i - The Apple Watch app on a real Ultra 4 (built and unit-tested, in preview) and Meta AI glasses (planned). - A physical robot running the ROS 2 example (the example and the robot package are in the repo). +- A real rescan that lines up with the room's last scan and reports what moved (built and unit-tested; the author's six scans each started fresh). - The tape-measured 20 cm accuracy check with the phone's depth and poses. - Quantitative evaluations and the Verifier precision figure. +- Timing room recognition on the iPhone from five start spots, and whether a grown room map matches from a spot that + failed before (the photo matcher is measured on the Mac only so far). - The usability session with blind and low-vision testers (plan ready in `evals/usability/`), and the author's screen-off run. - Daily spend caps on model calls (the API and the website's voice route are rate-limited per key or visitor; there is no daily cap yet). - The 3-minute demo video. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index a95adfa..9b68a86 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -95,6 +95,7 @@ Remove any row for a tool or library that isn't actually in the repo at submissi | What | Licence | |---|---| | Atkinson Hyperlegible Next (via `next/font/google`, self-hosted at build) | SIL Open Font License 1.1 | +| Inter Tight, Space Mono (demo video graphics only, from `@fontsource`) | SIL Open Font License 1.1 | ## Audio diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/AskInAR.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/AskInAR.swift new file mode 100644 index 0000000..d10873a --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/AskInAR.swift @@ -0,0 +1,32 @@ +import Foundation +import RecallDomain + +/// A question asked with the AR view open: the Guide answers from the Mac's memory as it does on Find, then the answer +/// is matched to the labels in this room. One thing here: walk there. Several alike: ask which. Anywhere else: just +/// the answer, because walking there needs that room's map. +public struct AskInAR: Sendable { + public enum Outcome: Sendable, Equatable { + case walk(Thing, say: String) + case which(say: String) + case answer(String) + } + + private let ask: AskWhereIs + + public init(ask: AskWhereIs) { self.ask = ask } + + public func callAsFunction(_ question: String, home: String, known: [Thing], labels: [ArLabel], + about: String? = nil) async throws -> Outcome { + let result = try await ask(question, home: home, known: known, about: about) + let text = result.answer.text + switch ARPick.settle(thingId: result.answer.thingId, noun: result.thing?.label, question: question, labels: labels) { + case .walk(let label): + guard let thing = known.first(where: { $0.id == label.id }) else { return .answer(text) } + return .walk(thing, say: text) + case .which: + return .which(say: text + " Double tap the one you mean, or say which.") + case .elsewhere: + return .answer(text) + } + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift index 65a3235..45728a3 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/GuideToThing.swift @@ -30,6 +30,7 @@ public final class GuideToThing { public var replanAfterMetres: Double = 1.5 // off the route by more than this → a new route public var maxReplans = 3 public var faceTimeout: Double = 30 // seconds to turn towards a compass bearing + public var keepsSession = false // guiding inside the AR view: the camera stays on when the guide ends public init() {} } @@ -45,15 +46,20 @@ public final class GuideToThing { private let haptics: any HapticsOutput private let clock: any Clock private let compass: (any CompassSensor)? + private let photos: (places: any PlaceLocating, scan: String)? private let settings: Settings private var cancelled = false + /// The route just walked, to retrace it; a walk that needed a new route goes back along the last one. + private var walked: (path: [SIMD2], floorY: Double?)? + /// `photos`: the Mac's photo matcher and the scan whose room map this is, to recognise the room alongside ARKit. public init(home: String, thing: Thing, roomMap: Data?, locator: any Locator, guide: any GuideService, speech: any SpeechOutput, haptics: any HapticsOutput, clock: any Clock, - compass: (any CompassSensor)? = nil, settings: Settings = .init()) { + compass: (any CompassSensor)? = nil, photos: (places: any PlaceLocating, scan: String)? = nil, + settings: Settings = .init()) { self.home = home; self.thing = thing; self.roomMap = roomMap; self.locator = locator; self.guide = guide self.speech = speech; self.haptics = haptics; self.clock = clock; self.compass = compass - self.settings = settings + self.photos = photos; self.settings = settings } /// The compass heading when it can be trusted (else directions leave out "north-east"). @@ -87,8 +93,11 @@ public final class GuideToThing { return } locator.showPath(route.waypoints, floorY: route.floorY) // arrows on the floor, leading the way + walked = (route.waypoints, route.floorY) if let first = route.steps.first { speech.say(first.say, interrupt: true) } - switch await follow(route) { + let arrive = route.arriveSay ?? "You should be at your \(thing.title)." + let end = await follow(route.waypoints) { [self] in await check(session: route.sessionId, arrive: arrive) } + switch end { case .done: return case .offRoute where replans < settings.maxReplans: replans += 1 @@ -100,21 +109,39 @@ public final class GuideToThing { } } + /// Whether there is a walked route to go back along. + public var canWalkBack: Bool { (walked?.path.count ?? 0) > 1 } + + /// Back the way the person came, on the phone alone: that path was clear a moment ago, so it needs no new route. + public func walkBack() async { + guard canWalkBack, let walked else { return } + let path = Array(walked.path.reversed()) + locator.mark(nil) + locator.showPath(path, floorY: walked.floorY) + speech.say("Going back.", interrupt: true) + _ = await follow(path, replanning: false) { [self] in finish(.arrived("You're back.")) } + } + public func stop() { cancelled = true locator.mark(nil) locator.showPath([], floorY: nil) - locator.stop() + release() compass?.stop() speech.stop() } + /// Lets the camera go, unless the AR view that started this guide still needs it. + private func release() { + if !settings.keepsSession { locator.stop() } + } + /// Says the current instruction again (the Repeat button and the magic tap). public func repeatLast() { switch phase { case .locating: speech.say("Still finding where you are. Turn slowly.", interrupt: true) case .planning: speech.say("Planning the way.", interrupt: true) - case .checking: speech.say("Checking with the camera. Hold the phone towards it.", interrupt: true) + case .checking: speech.say("Checking with the camera. Hold the phone out in front of you.", interrupt: true) case .guiding(let cue, _): speech.say(Directions.say(cue, heading: heading), interrupt: true) case .landmarks(let t), .arrived(let t), .failed(let t): speech.say(t, interrupt: true) } @@ -122,8 +149,14 @@ public final class GuideToThing { // MARK: steps - /// Waits for tracking to turn normal (ARKit recognised the room) or the timeout. + /// Waits for tracking to turn normal (ARKit recognised the room, or two photo fixes agreed) or the timeout. private func locate() async -> TrackingState { + let assist = photos.map { + PhotoAssist(home: home, scan: $0.scan, room: thing.room, locator: locator, places: $0.places, speech: speech, + clock: clock) + } + let helping = assist.map { a in Task { await a.run() } } + defer { assist?.cancel(); helping?.cancel() } let started = clock.now() while !cancelled, clock.now().timeIntervalSince(started) < settings.locateTimeout { if locator.reading()?.tracking == .normal { return .normal } @@ -146,12 +179,13 @@ public final class GuideToThing { private enum FollowEnd { case done, offRoute } - private func follow(_ route: Route) async -> FollowEnd { - var follower = RouteFollower(waypoints: route.waypoints) + /// `replanning` false: off the path it keeps pointing at the next waypoint, because no new route is coming. + private func follow(_ waypoints: [SIMD2], replanning: Bool = true, + arrived: () async -> Void) async -> FollowEnd { + var follower = RouteFollower(waypoints: waypoints) var throttle = CueThrottle() var lastTick = Date.distantPast var warnedWeak = false - let arrive = route.arriveSay ?? "You should be at your \(thing.title)." while !cancelled { try? await clock.sleep(seconds: settings.tick) guard !cancelled, let reading = locator.reading() else { continue } @@ -171,10 +205,10 @@ public final class GuideToThing { if cue.arrived { locator.showPath([], floorY: nil) haptics.success() - await check(session: route.sessionId, arrive: arrive) + await arrived() return .done } - if cue.offRouteMetres > settings.replanAfterMetres { return .offRoute } + if replanning, cue.offRouteMetres > settings.replanAfterMetres { return .offRoute } let now = clock.now() if cue.clock == 12, now.timeIntervalSince(lastTick) > 1.5 { // a tick while facing the right way haptics.tap() @@ -189,7 +223,7 @@ public final class GuideToThing { /// last word. Without a frame or an answer the arrival stands, and says so. private func check(session: String, arrive: String) async { phase = .checking - speech.say(arrive + " Hold the phone towards it. I'm checking with the camera.", interrupt: true) + speech.say(arrive + " Hold the phone out in front of you. I'm checking with the camera.", interrupt: true) try? await clock.sleep(seconds: settings.aimSeconds) guard !cancelled else { return } guard let frame = locator.arrivalFrame() else { @@ -231,7 +265,7 @@ public final class GuideToThing { facedSince = nil } } - locator.stop() + release() } private func finish(_ end: Phase) { @@ -240,6 +274,6 @@ public final class GuideToThing { case .landmarks(let t), .arrived(let t), .failed(let t): speech.say(t, interrupt: true) default: break } - locator.stop() + release() } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift new file mode 100644 index 0000000..3f78e38 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/PhotoAssist.swift @@ -0,0 +1,78 @@ +import Foundation +import RecallDomain + +/// Helps ARKit recognise the room, never replaces it: while ARKit hasn't matched the saved room map, about once a +/// second one camera photo goes to the Mac's photo matcher (classic features + PnP, no model). From its answers: +/// - the room is somewhere else → said once ("You seem to be in Bedroom, not Living room…"); +/// - how to move → said every few seconds ("Keep turning slowly…"), never where to look: someone who can't see the +/// room can't point at a shelf, and furniture is only named once the room is recognised; +/// - two fixes in a row that agree → the locator takes the correction and tracking reads normal at once. +/// The moment ARKit matches by itself it stops asking. No Mac, or an error: nothing is said and ARKit carries on. +@MainActor +public final class PhotoAssist { + public struct Settings: Sendable { + public var firstAfter: Double = 1.0 // ARKit often matches within a second: then nothing is sent + public var every: Double = 1.0 // the Mac answers in about 35 ms; one photo a second is plenty + public var maxAsks = 40 + public var hintEvery: Double = 8 // seconds between "keep turning" lines + public init() {} + } + + private let home: String + private let scan: String + private let room: String? + private let locator: any Locator + private let places: any PlaceLocating + private let speech: any SpeechOutput + private let clock: any Clock + private let settings: Settings + private var cancelled = false + + /// Whether a photo fix (not ARKit) located the phone. + public private(set) var fixed = false + + public init(home: String, scan: String, room: String?, locator: any Locator, places: any PlaceLocating, + speech: any SpeechOutput, clock: any Clock, settings: Settings = .init()) { + self.home = home; self.scan = scan; self.room = room; self.locator = locator; self.places = places + self.speech = speech; self.clock = clock; self.settings = settings + } + + public func cancel() { cancelled = true } + + private var located: Bool { locator.reading()?.tracking == .normal } + + /// Until the room is recognised (by ARKit or two agreeing fixes), `cancel()` or the ask limit. + public func run() async { + try? await clock.sleep(seconds: settings.firstAfter) + var agreement = FixAgreement() + var turn = 0 + var lastHint = clock.now() // the opening line has just said how to move + var toldElsewhere = false + for _ in 0..= settings.hintEvery { + speech.say(LocateCoaching.keepTurning(turn: turn), interrupt: false) + turn += 1 + lastHint = clock.now() + } + if let pose = fix.pose { + let session = Pose(columnMajor: frame.pose) + if let agreed = agreement.add(MapCorrection.between(map: pose, session: session), session: session) { + locator.correct(mapFromSession: Pose(columnMajor: MapCorrection.columnMajor(agreed))) + fixed = true + return + } + } + } + try? await clock.sleep(seconds: settings.every) + } + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/ReadFoundReply.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/ReadFoundReply.swift new file mode 100644 index 0000000..73a8402 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/ReadFoundReply.swift @@ -0,0 +1,29 @@ +import Foundation +import RecallDomain + +/// Understands a spoken reply after a find: the Meta model on the Mac first, the phone's own word rules if it can't be +/// reached, can't tell or takes too long, so "go back, or done?" works with the Mac asleep and nobody is left waiting. +public struct ReadFoundReply: Sendable { + private let reader: (any ReplyReading)? + private let patience: Double + + public init(reader: (any ReplyReading)?, patience: Double = 6) { + self.reader = reader + self.patience = patience + } + + public func callAsFunction(_ said: String, about thing: String) async -> FoundReply { + guard let reader else { return FoundReply.read(said) } + let understood = await withTaskGroup(of: FoundReply?.self) { group in + group.addTask { try? await reader.read(reply: said, about: thing) } + group.addTask { [patience] in + try? await Task.sleep(for: .seconds(patience)) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + return first + } + return understood ?? FoundReply.read(said) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift index 6e2a75e..e2e9c43 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift @@ -13,13 +13,29 @@ public final class ARLocator: Locator { /// What the camera view draws over the feed: the arrow above the thing. public let scene = SCNScene() private var arrow: SCNNode? + /// Everything placed in the room map's world hangs here: the session's own world when ARKit matched the map, + /// or turned by a photo fix's correction until it does. + private let mapRoot = SCNNode() private let track = SCNNode() private let tags = SCNNode() private var shown: [ArLabel] = [] + /// Each label's node, by label id: for finding the one under a finger and for hiding the others on a walk. + private var labelNodes: [String: SCNNode] = [:] + + /// T(map ← session) from two agreeing photo fixes, until ARKit recognises the room itself. + private var correction: simd_double4x4? + private var watching: Task? + /// ARKit itself matched the saved map in this run (not a photo fix): the map may grow from here. + public private(set) var recognisedByARKit = false + /// Handed a grown room map (ARKit's map of this run, which keeps the saved map's world) once ARKit has matched + /// and mapped what it sees; the caller saves it beside the scan's own so the next visit matches from here too. + public var onMapGrown: ((Data) -> Void)? + private var growing: Task? public init() { - scene.rootNode.addChildNode(track) - scene.rootNode.addChildNode(tags) + scene.rootNode.addChildNode(mapRoot) + mapRoot.addChildNode(track) + mapRoot.addChildNode(tags) } public static var isSupported: Bool { ARWorldTrackingConfiguration.isSupported } @@ -43,6 +59,7 @@ public final class ARLocator: Locator { } guard let roomMap else { if running, !runningWithMap { return true } + forgetRun() session.run(config, options: [.resetTracking, .removeExistingAnchors]) running = true; runningWithMap = false return true @@ -54,6 +71,7 @@ public final class ARLocator: Locator { // TODO(verify) on the iPhone how quickly tracking returns to normal after a resume. session.run(config, options: []) } else { + forgetRun() config.initialWorldMap = current.map session.run(config, options: [.resetTracking, .removeExistingAnchors]) } @@ -64,6 +82,7 @@ public final class ARLocator: Locator { return false } current = (roomMap, map) + forgetRun() config.initialWorldMap = map session.run(config, options: [.resetTracking, .removeExistingAnchors]) running = true; runningWithMap = true @@ -74,6 +93,79 @@ public final class ARLocator: Locator { public func stop() { session.pause() running = false + growing?.cancel() + } + + /// A fresh ARKit run: no photo correction, nothing recognised yet. + private func forgetRun() { + correct(mapFromSession: nil) + recognisedByARKit = false + growing?.cancel() + growing = nil + } + + // MARK: Photo fixes and ARKit's own match + + public func correct(mapFromSession: Pose?) { + correction = mapFromSession.map(MapCorrection.matrix) + let inverse = correction.map { Self.float($0.inverse) } ?? matrix_identity_float4x4 + mapRoot.simdTransform = inverse // the scene renders in the session's world: map → session + watching?.cancel() + guard correction != nil else { return } + watching = Task { [weak self] in // ARKit's own match replaces the photo fix the moment it comes + while !Task.isCancelled { + try? await Task.sleep(for: .milliseconds(250)) + guard let self, !Task.isCancelled else { return } + _ = self.reading() + if self.correction == nil { return } + } + } + } + + /// The camera in the room map's world, and whether the room is recognised (by ARKit or a photo fix). + private func mapCamera() -> (transform: simd_float4x4, tracking: TrackingState)? { + guard let camera = session.currentFrame?.camera else { return nil } + let raw = CaptureSession.tracking(camera.trackingState).0 + if raw == .normal, runningWithMap { + if correction != nil { correct(mapFromSession: nil) } // the session's world is now the map's + if !recognisedByARKit { recognisedByARKit = true; growMapSoon() } + } + guard let correction else { return (camera.transform, raw) } + return (Self.float(correction * Self.double(camera.transform)), .normal) + } + + /// A few seconds after ARKit matched, once it has mapped what the camera sees, the run's map (saved map plus what + /// this visit added, in the same world) goes to `onMapGrown`. At most once a run; never from a photo fix alone. + private func growMapSoon() { + guard onMapGrown != nil, growing == nil else { return } + growing = Task { [weak self] in + try? await Task.sleep(for: .seconds(6)) + guard let self, !Task.isCancelled, self.running, self.recognisedByARKit, + self.session.currentFrame?.worldMappingStatus == .mapped else { return } + guard let data = await self.currentMapData(), data.count < 50_000_000 else { return } + guard !Task.isCancelled else { return } + self.onMapGrown?(data) + } + } + + static func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4(columns: (SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), + SIMD4(m.columns.3))) + } + + static func double(_ m: simd_float4x4) -> simd_double4x4 { + simd_double4x4(columns: (SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), + SIMD4(m.columns.3))) + } + + private func currentMapData() async -> Data? { + await withCheckedContinuation { cont in + session.getCurrentWorldMap { map, _ in + cont.resume(returning: map.flatMap { + try? NSKeyedArchiver.archivedData(withRootObject: $0, requiringSecureCoding: true) + }) + } + } } private let encoder = FrameEncoder() @@ -84,7 +176,9 @@ public final class ARLocator: Locator { // ARKit's intrinsics are for the full camera image; the JPEG is scaled to a 1024-pixel long edge let s = Double(size.width) / Double(frame.camera.imageResolution.width) let k = frame.camera.intrinsics - let pose = frame.camera.transform.columnMajor.map(Double.init) + // in the room map's world once a photo fix corrected it (the Mac lifts the arrival photo in that world) + let camera = correction.map { Self.float($0 * Self.double(frame.camera.transform)) } ?? frame.camera.transform + let pose = camera.columnMajor.map(Double.init) guard let (mm, w, h) = Self.millimetres(depth) else { return nil } return ArrivalFrame(jpeg: jpeg, depthMM: mm, depthSize: (w, h), imageSize: (Int(size.width), Int(size.height)), intrinsics: (Double(k.columns.0.x) * s, Double(k.columns.1.y) * s, @@ -123,7 +217,7 @@ public final class ARLocator: Locator { up.timingMode = .easeInEaseOut node.runAction(.repeatForever(.sequence([up, up.reversed()]))) } - scene.rootNode.addChildNode(node) + mapRoot.addChildNode(node) arrow = node } @@ -139,7 +233,7 @@ public final class ARLocator: Locator { let node = SCNNode() node.simdPosition = SIMD3(Float(p.x), Float(p.y), Float(p.z)) - let height: Float = 0.35 + let height = Self.tagHeight let width = Float(item.tag.size.width / item.tag.size.height) * 0.09 let plane = SCNPlane(width: CGFloat(width), height: 0.09) let face = SCNMaterial() @@ -176,17 +270,44 @@ public final class ARLocator: Locator { node.runAction(.sequence([.wait(duration: 0.12 * Double(i)), .fadeIn(duration: 0.35)])) } tags.addChildNode(node) + labelNodes[item.label.id] = node } } public func clearLabels() { tags.childNodes.forEach { $0.removeFromParentNode() } shown = [] + labelNodes = [:] + } + + /// On a walk only the thing's label stays, so the route reads; nil brings them all back. + public func focus(label id: String?) { + for (key, node) in labelNodes { node.isHidden = id != nil && key != id } + } + + /// The label under a finger: the nearest to `point` of the tag, its line and the thing itself, as drawn on screen + /// (a tag is small from across a room, so a generous reach). Nil when none is close. + public func label(at point: CGPoint, in view: SCNSceneRenderer, reach: CGFloat = 70) -> ArLabel? { + func distance(from a: SCNVector3, to b: SCNVector3) -> CGFloat? { + guard a.z < 1 || b.z < 1 else { return nil } // behind the camera + let (ax, ay, bx, by) = (CGFloat(a.x), CGFloat(a.y), CGFloat(b.x), CGFloat(b.y)) + let (dx, dy) = (bx - ax, by - ay) + let length = dx * dx + dy * dy + let t = length > 0 ? min(1, max(0, ((point.x - ax) * dx + (point.y - ay) * dy) / length)) : 0 + return hypot(point.x - (ax + t * dx), point.y - (ay + t * dy)) + } + let scored = shown.compactMap { label -> (ArLabel, CGFloat)? in + guard let node = labelNodes[label.id], !node.isHidden else { return nil } + let thing = view.projectPoint(node.worldPosition) + let tag = view.projectPoint(node.convertPosition(SCNVector3(0, Self.tagHeight, 0), to: nil)) + return distance(from: tag, to: thing).map { (label, $0) } + } + return scored.filter { $0.1 <= reach }.min { $0.1 < $1.1 }?.0 } /// The label the camera points at most directly (within 25°), for "what's this?". public func labelInView() -> ArLabel? { - guard let cam = session.currentFrame?.camera.transform else { return nil } + guard let cam = mapCamera()?.transform else { return nil } let eye = SIMD3(Double(cam.columns.3.x), Double(cam.columns.3.y), Double(cam.columns.3.z)) let fwd = -SIMD3(Double(cam.columns.2.x), Double(cam.columns.2.y), Double(cam.columns.2.z)) let scored = shown.map { label -> (ArLabel, Double) in @@ -196,6 +317,9 @@ public final class ARLocator: Locator { return scored.filter { $0.1 < 25 }.min { $0.1 < $1.1 }?.0 } + /// How far above its thing a tag floats, metres. + private static let tagHeight: Float = 0.35 + public static func colour(_ mark: ArLabel.Mark) -> UIColor { switch mark { case .told: UIColor(red: 0x42 / 255, green: 0xb7 / 255, blue: 0x2a / 255, alpha: 1) // cta green @@ -216,7 +340,7 @@ public final class ARLocator: Locator { let chevrons = ChevronLayout.place(path: path) guard !chevrons.isEmpty else { return } // without the floor from the Mac, assume the phone is held about 1.4 m above it - let y = Float(floorY ?? Double((session.currentFrame?.camera.transform.columns.3.y ?? 1.4) - 1.4)) + 0.01 + let y = Float(floorY ?? Double((mapCamera()?.transform.columns.3.y ?? 1.4) - 1.4)) + 0.01 let still = UIAccessibility.isReduceMotionEnabled for (i, c) in chevrons.enumerated() { let node = Self.chevronNode() @@ -296,10 +420,9 @@ public final class ARLocator: Locator { } public func reading() -> (pose: Pose, tracking: TrackingState)? { - guard let camera = session.currentFrame?.camera else { return nil } - let m = camera.transform + guard let (m, tracking) = mapCamera() else { return nil } let pose = Pose(columnMajor: [m.columns.0, m.columns.1, m.columns.2, m.columns.3] .flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] }) - return (pose, CaptureSession.tracking(camera.trackingState).0) + return (pose, tracking) } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift index d0eaa56..33710bd 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift @@ -89,7 +89,7 @@ public struct ScanSignals: Sendable { case .moveCloser: "Too far. Move closer to the walls." case .moreLight: "It's too dark to see well. Turn on a light, or move closer." case .tiltDown: "Tilt the phone down a little so I can see the floor." - case .lowerPhone: "Lower the phone. Point it at the walls, not the ceiling." + case .lowerPhone: "Lower the phone. Hold it level with your chest." case .keepMoving: "Keep moving slowly. I haven't seen anything new for a while." } } @@ -161,7 +161,7 @@ public struct ScanGuide: Sendable { public var spoken: String { switch self { - case .recognise: "Scanning again. Point the phone slowly around the room so I can recognise it." + case .recognise: "Scanning again. Hold the phone at chest height and turn slowly so I can recognise the room." case .start: "Stand by the door with the phone at chest height, camera facing the room." case .turnAround: "Turn slowly all the way round." case .alongWalls: "Now walk along the walls, about a metre away, with the phone pointing at the wall." diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift index beb419e..8068e84 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift @@ -12,7 +12,7 @@ public enum ScanTips { tips.append("Walls were seen on \(walls) of 12 sides. Walk the whole way round, about a metre from each wall.") } if let seen = m.zonesSeen, let total = m.zonesTotal, total > 0, Double(seen) / Double(total) < 0.6 { - tips.append("Height zones seen \(seen) of \(total). Point the phone at knee height and up at shelves too, not only straight ahead.") + tips.append("Height zones seen \(seen) of \(total). Sweep the phone from knee height to above your head, not only straight ahead.") } if m.directionsCovered < 10 { tips.append("\(m.directionsCovered) of 12 directions were seen. Turn slowly all the way round at the start.") diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/SpeedCoach.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/SpeedCoach.swift index 797dd65..6a5f5b3 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/SpeedCoach.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/SpeedCoach.swift @@ -62,7 +62,7 @@ public struct SpeedCoach: Sendable { case .walkingFast: return "Slow down. Walk more slowly." case .turningFast: return "Slow down. Turn more slowly." case .trackingExcessiveMotion: return "Slow down. The phone is moving too fast to follow." - case .trackingFewFeatures: return "Point at something with more detail, and turn on a light." + case .trackingFewFeatures: return "The camera can't see much here. Turn a little, and turn on a light if you can." } } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Guidance/FoundReply.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Guidance/FoundReply.swift new file mode 100644 index 0000000..6bd414a --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Guidance/FoundReply.swift @@ -0,0 +1,47 @@ +import Foundation + +/// What the person meant after "Do you want to go back, or are you done?". A Meta model on the Mac reads the reply +/// (`ReadFoundReply`); `read` is the fallback for when the Mac can't be reached, so the walk never stalls on it. Plain +/// word rules, because a wrong guess here only costs a sentence; anything else is a new question for the Guide. +public enum FoundReply: Sendable, Equatable { + case goBack, done, question + + public static func read(_ said: String) -> FoundReply { + let words = said.lowercased().replacingOccurrences(of: "'", with: "").replacingOccurrences(of: "\u{2019}", with: "") + .split(whereSeparator: { !$0.isLetter }).map(String.init) + let has = { (list: Set) in words.contains(where: list.contains) } + if has(["where", "wheres", "find", "looking", "locate"]) { return .question } + let refused = has(["dont", "not", "never"]) || (words.contains("no") && words.contains("need")) + if has(["back", "return"]) { return refused ? .done : .goBack } + if has(["yes", "yeah", "yep", "yup", "sure"]) { return .goBack } + if refused || has(["no", "nope", "done", "thanks", "thank", "nothing", "finished", "stop", "ok", "okay", "bye", + "goodbye", "fine"]) { return .done } + return .question + } +} + +/// Roughly how long a line takes to say: under VoiceOver the app can't hear when VoiceOver has finished, and the +/// microphone mustn't open over it. +public enum SpokenLength { + public static func seconds(_ line: String) -> Double { + Double(line.split(separator: " ").count) * 0.42 + 0.6 + } +} + +/// Ends a recording when the person has finished talking: the microphone's level, nothing more (no speech recognition). +public struct SilenceGate: Sendable { + public enum Verdict: Sendable, Equatable { case listen, finished, nothingSaid } + + public var speechLevel = -34.0 // dB: louder than this is someone talking + public var quietAfter = 1.2 // seconds of quiet after speech that end the turn + public var giveUpAfter = 8.0 // seconds with no speech at all + private var lastSpeech: Double? + + public init() {} + + public mutating func hear(level: Double, at seconds: Double) -> Verdict { + if level > speechLevel { lastSpeech = seconds } + if let lastSpeech { return seconds - lastSpeech >= quietAfter ? .finished : .listen } + return seconds >= giveUpAfter ? .nothingSaid : .listen + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift new file mode 100644 index 0000000..69149b4 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Locate/PhotoFix.swift @@ -0,0 +1,81 @@ +import Foundation +import simd + +/// What the Mac's photo matcher said about one camera photo (classic features + PnP against the scans; no model). +public struct PlaceFix: Sendable, Equatable { + /// The room the photo was taken in, when one room clearly matched (it may not be the room being looked for). + public let room: String? + /// The camera's pose in the room map's world, when that room matched well enough. + public let pose: Pose? + public let inliers: Int + /// One-of-a-kind furniture in that room ("tv", "cabinet"). + public let hint: [String] + + public init(room: String?, pose: Pose?, inliers: Int, hint: [String]) { + self.room = room; self.pose = pose; self.inliers = inliers; self.hint = hint + } +} + +/// From a photo fix to the phone's own poses: T(map ← session) = map pose × session pose⁻¹, for the same photo. +public enum MapCorrection { + public static func between(map: Pose, session: Pose) -> simd_double4x4 { matrix(map) * matrix(session).inverse } + + /// A session pose in the room map's world. + public static func apply(_ T: simd_double4x4, to pose: Pose) -> Pose { Pose(columnMajor: columnMajor(T * matrix(pose))) } + + public static func matrix(_ p: Pose) -> simd_double4x4 { + let c = p.columnMajor + return simd_double4x4(columns: (SIMD4(c[0], c[1], c[2], c[3]), SIMD4(c[4], c[5], c[6], c[7]), + SIMD4(c[8], c[9], c[10], c[11]), SIMD4(c[12], c[13], c[14], c[15]))) + } + + public static func columnMajor(_ m: simd_double4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] } + } + + /// How far apart two transforms are: metres between their origins, degrees between their rotations. + public static func distance(_ a: simd_double4x4, _ b: simd_double4x4) -> (metres: Double, degrees: Double) { + let ta = SIMD3(a.columns.3.x, a.columns.3.y, a.columns.3.z), tb = SIMD3(b.columns.3.x, b.columns.3.y, b.columns.3.z) + let ra = simd_double3x3(SIMD3(a.columns.0.x, a.columns.0.y, a.columns.0.z), SIMD3(a.columns.1.x, a.columns.1.y, a.columns.1.z), + SIMD3(a.columns.2.x, a.columns.2.y, a.columns.2.z)) + let rb = simd_double3x3(SIMD3(b.columns.0.x, b.columns.0.y, b.columns.0.z), SIMD3(b.columns.1.x, b.columns.1.y, b.columns.1.z), + SIMD3(b.columns.2.x, b.columns.2.y, b.columns.2.z)) + let d = ra.transpose * rb + let cosine = max(-1, min(1, (d.columns.0.x + d.columns.1.y + d.columns.2.z - 1) / 2)) + return (simd_distance(ta, tb), acos(cosine) * 180 / .pi) + } +} + +/// One photo fix can be wrong (on real scans about 1 in 9 from another session), so the phone trusts photo fixes +/// only when two in a row agree on the correction: within 25 cm and 10°. Two wrong fixes rarely agree. +public struct FixAgreement: Sendable { + public var maxMetres = 0.25 + public var maxDegrees = 10.0 + private var last: (correction: simd_double4x4, session: Pose)? + + public init() {} + + /// Adds a fix's correction and the session pose its photo had; the agreed correction once two in a row agree. + public mutating func add(_ correction: simd_double4x4, session: Pose) -> simd_double4x4? { + defer { last = (correction, session) } + // a session pose that never changed means ARKit isn't tracking: a correction from it would hold nothing + guard let last, last.session != session else { return nil } + let d = MapCorrection.distance(last.correction, correction) + return d.metres <= maxMetres && d.degrees <= maxDegrees ? correction : nil + } +} + +/// The spoken lines while the room is being recognised. +public enum LocateCoaching { + /// While nothing is recognised yet: what to do with your body, never where to look (`turn` counts the lines said). + public static func keepTurning(turn: Int) -> String { + turn == 0 ? "Keep turning slowly, with the phone at chest height." : "Keep turning. Tilt the phone a little up, then down." + } + + public static func elsewhere(found: String, wanted: String) -> String { + "You seem to be in \(found), not \(wanted). Walk to \(wanted) and I'll keep looking." + } + + /// A furniture label as said aloud: short ones are initials ("tv" → "TV"). + public static func spoken(_ label: String) -> String { label.count <= 2 ? label.uppercased() : label } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift index 461248b..8bd9afc 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift @@ -55,3 +55,28 @@ public struct ArLabels: Sendable, Equatable { self.scanId = scanId; self.room = room; self.floorY = floorY; self.labels = labels } } + +/// What a spoken question settles on among the labels of the room the phone is in. Decided from the labels and the +/// words said, never from how the model phrased its answer: "where's my laptop?" with two laptops here asks which one, +/// "the black laptop" walks to it. +public enum ARPick: Sendable, Equatable { + case walk(ArLabel) + /// Several alike and the question doesn't say which. + case which([ArLabel]) + /// Not one of the things labelled here. + case elsewhere + + /// `thingId`: the thing the answer is about; `noun`: what it is ("laptop"), so alike things can be found. + public static func settle(thingId: String?, noun: String?, question: String, labels: [ArLabel]) -> ARPick { + guard let label = labels.first(where: { $0.id == thingId }) else { return .elsewhere } + guard let noun = noun?.lowercased(), !noun.isEmpty else { return .walk(label) } + let alike = labels.filter { $0.title.lowercased().hasSuffix(noun) } + guard alike.count > 1 else { return .walk(label) } + let said = Set(question.lowercased().split(whereSeparator: { !$0.isLetter && !$0.isNumber }).map(String.init)) + let named = alike.filter { other in + let words = other.title.lowercased().split(separator: " ").map(String.init).filter { !noun.contains($0) } + return !words.isEmpty && words.allSatisfy(said.contains) + } + return named.count == 1 ? .walk(named[0]) : .which(named.isEmpty ? alike : named) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift index 6236884..07abe39 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift @@ -63,6 +63,8 @@ public struct AuditEntry: Sendable, Equatable, Identifiable { return "The camera checked you'd reached a thing" case "guide.answered": return refused ? "Your question couldn't reach the Guide's model" : "Your question went to the Guide's model" + case "reply.read": + return refused ? "Your reply couldn't reach the Guide's model" : "Your reply went to the Guide's model" case "voice.transcribed": return refused ? "Your voice couldn't reach Muse Voice Transcribe" : "Your voice went to Muse Voice Transcribe (not kept)" diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift index 055cf17..3e8fe33 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift @@ -53,6 +53,11 @@ public protocol Transcriber: Sendable { func transcribe(wav: Data) async throws -> String } +/// A Meta model on the Mac reads a spoken reply after a find; nil when it couldn't tell. +public protocol ReplyReading: Sendable { + func read(reply: String, about thing: String) async throws -> FoundReply? +} + /// The Mac's processing status of an uploaded scan. public protocol ScanProcessing: Sendable { func status(scan: String) async throws -> ScanReport @@ -112,7 +117,10 @@ public protocol ScanLibrary: Sendable { public protocol EpisodeFiles: Sendable { func directory(scan: String) -> URL func manifest(scan: String) -> Manifest? + /// The scan's room map: the one ARKit grew from later visits when there is one, else the one the scan saved. func roomMap(scan: String) -> Data? + /// Keeps a grown room map (the scan's own stays as uploaded). + func saveGrownRoomMap(scan: String, _ data: Data) func delete(scan: String) } @@ -151,12 +159,19 @@ public protocol Clock: Sendable { } /// Records a spoken question. `finished` is called once, with the recording, when the user stops or time runs out. +/// `endOnSilence`: it also ends by itself when the person stops talking, with nil if they never spoke. @MainActor public protocol VoiceInput: AnyObject { var isRecording: Bool { get } - func start(maxSeconds: Double, finished: @escaping @MainActor (Data?) -> Void) async -> Bool + func start(maxSeconds: Double, endOnSilence: Bool, finished: @escaping @MainActor (Data?) -> Void) async -> Bool func stop() } +extension VoiceInput { + public func start(maxSeconds: Double, finished: @escaping @MainActor (Data?) -> Void) async -> Bool { + await start(maxSeconds: maxSeconds, endOnSilence: false, finished: finished) + } +} + /// The phone's place in the room for guidance: relocalises in a saved room map, then reports the camera's pose. @MainActor public protocol Locator: AnyObject { func start(roomMap: Data?) -> Bool // false when the saved map can't be read @@ -169,6 +184,16 @@ public protocol Clock: Sendable { func arrivalFrame() -> ArrivalFrame? /// Lays arrows on the floor along the route (ARKit x, z), at the floor's height when known; empty clears them. func showPath(_ path: [SIMD2], floorY: Double?) + /// A photo fix's correction, T(map ← session) as a column-major 4 x 4: from now on `reading()` and + /// `arrivalFrame()` answer in the room map's world and tracking reads normal. nil drops it; so does ARKit's own + /// match of the room (its world then is the map's). + func correct(mapFromSession: Pose?) +} + +/// The Mac's photo matcher: which room a camera photo shows, and the camera's pose in a room map's world. +public protocol PlaceLocating: Sendable { + /// `scan`: the scan whose room map the phone runs; `room`: the room being looked for. + func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix } /// The phone's compass (magnetic north; no location is read). diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift index 9b5dea1..afa3e09 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift @@ -43,7 +43,8 @@ public enum PrivacyNote { "On your iPhone: your scans until you delete them, the room map used to find your way, and the Mac's " + "address. The device key is in the Keychain.", "Between them: your home network, or an encrypted tunnel you set up (Tailscale or Cloudflare Tunnel). " - + "Every request needs the device key.", + + "Every request needs the device key. While Recall finds the room, about one camera photo a second goes " + + "to your Mac (only there), to be matched against your scans; it is not stored.", ] public static let never = [ diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift index c0906be..7702560 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift @@ -70,6 +70,38 @@ public struct RecallAPI: Sendable { } } +// MARK: PlaceLocating + +extension RecallAPI: PlaceLocating { + /// Where am I? One photo to the Mac's matcher (LocateRequest.yaml). A short timeout: an answer that comes late is + /// worth less than the next photo. + public func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix { + let boundary = "recall-\(UUID().uuidString)" + var body = Data() + func part(_ name: String, filename: String? = nil, type: String? = nil, _ data: Data) { + body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"\(name)\"".utf8)) + if let filename { body.append(Data("; filename=\"\(filename)\"".utf8)) } + body.append(Data("\r\n".utf8)) + if let type { body.append(Data("Content-Type: \(type)\r\n".utf8)) } + body.append(Data("\r\n".utf8)) + body.append(data) + body.append(Data("\r\n".utf8)) + } + part("image", filename: "now.jpg", type: "image/jpeg", frame.jpeg) + part("pose", frame.poseJSON) + part("scan_id", Data(scan.utf8)) + if let room { part("room", Data(room.utf8)) } + body.append(Data("--\(boundary)--\r\n".utf8)) + let path = "homes/\(home.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? home)/locate" + var req = request(path, method: "POST", contentType: "multipart/form-data; boundary=\(boundary)", body: body) + req.timeoutInterval = 4 + let json = try await send(req) + let pose = (json["pose"] as? [Double]).flatMap { $0.count == 16 ? Pose(columnMajor: $0) : nil } + return PlaceFix(room: json["room"] as? String, pose: pose, inliers: json["inliers"] as? Int ?? 0, + hint: json["hint"] as? [String] ?? []) + } +} + // MARK: HomeMemory extension RecallAPI: HomeMemory { @@ -103,6 +135,20 @@ extension RecallAPI: HomeMemory { // MARK: GuideService, Transcriber +extension RecallAPI: ReplyReading { + /// nil for "unknown": the Mac's model couldn't tell, so the phone decides. + public func read(reply: String, about thing: String) async throws -> FoundReply? { + let r = try await call { try await client.readReply(body: .json(.init(said: String(reply.prefix(300)), thing: String(thing.prefix(80))))) } + guard case .ok(let ok) = r else { return nil } + return switch try ok.body.json.intent.rawValue { + case "go_back": .goBack + case "done": .done + case "question": .question + default: nil + } + } +} + extension RecallAPI: GuideService, Transcriber { public func ask(home: String, question: String, about: String?) async throws -> Answer { let r = try await call { try await client.ask(body: .json(.init(homeId: home, question: question, about: about))) } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json index 13b9233..b617047 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json +++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json @@ -846,6 +846,58 @@ } } }, + "/homes/{home_id}/locate": { + "post": { + "tags": [ + "homes" + ], + "operationId": "locateByPhoto", + "summary": "Where am I? One camera photo matched against the room scans (classic features + PnP, no model)", + "description": "Asked by the phone while ARKit hasn't yet recognised the saved room map, alongside it (never instead of it). Answers the room the photo was taken in when one room clearly wins, the camera's pose in the requested map's world when that room matched well enough, and what to point the phone at. The photo is matched in memory and not stored.\n", + "parameters": [ + { + "$ref": "#/components/parameters/home_id" + } + ], + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/LocateRequest" + } + } + } + }, + "responses": { + "200": { + "description": "What the photo matched", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlaceFix" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "422": { + "$ref": "#/components/responses/Invalid" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + } + } + } + }, "/homes/{home_id}/robot-package": { "get": { "tags": [ @@ -1344,6 +1396,43 @@ } } } + }, + "/reply": { + "post": { + "tags": [ + "voice" + ], + "operationId": "readReply", + "summary": "What a spoken reply means after a find, read by Muse Spark Open as a strict JSON verdict", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplyRequest" + } + } + } + }, + "responses": { + "200": { + "description": "What the person wants next", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplyIntent" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + } + } + } } }, "components": { @@ -2505,6 +2594,7 @@ "arrival.verified", "guide.answered", "voice.transcribed", + "reply.read", "audit.pruned", "rate.limited" ] @@ -3120,6 +3210,118 @@ } } }, + "LocateRequest": { + "type": "object", + "required": [ + "image", + "pose", + "scan_id" + ], + "properties": { + "image": { + "type": "string", + "format": "binary", + "description": "JPEG of the camera now (any size; intrinsics are for this image)" + }, + "pose": { + "type": "string", + "description": "JSON: {\"pose\": [16 numbers, ARKit camera-to-world, column-major, in the phone's session (not yet the map's)], \"intrinsics\": {\"fx\", \"fy\", \"cx\", \"cy\"} for the JPEG, \"image_size\": [w, h]}. Only the intrinsics are used on the Mac; the phone keeps the session pose to turn the answer into its own correction.\n" + }, + "scan_id": { + "type": "string", + "maxLength": 64, + "description": "The scan whose room map the phone runs: the pose is answered in its world" + }, + "room": { + "type": "string", + "maxLength": 60, + "description": "The room the phone is trying to recognise (for the hint)" + } + } + }, + "PlaceFix": { + "type": "object", + "required": [ + "room", + "scan_id", + "score", + "runner_up_room", + "runner_up_score", + "pose", + "inliers", + "reproj_px", + "hint", + "ms" + ], + "properties": { + "room": { + "type": [ + "string", + "null" + ], + "description": "The room the photo was taken in, when one room clearly matched (it may not be the requested map's room)" + }, + "scan_id": { + "type": [ + "string", + "null" + ], + "description": "The scan whose world `pose` is in" + }, + "score": { + "type": "integer", + "minimum": 0, + "description": "RANSAC inliers of the best room" + }, + "runner_up_room": { + "type": [ + "string", + "null" + ] + }, + "runner_up_score": { + "type": "integer", + "minimum": 0 + }, + "pose": { + "type": [ + "array", + "null" + ], + "items": { + "type": "number" + }, + "minItems": 16, + "maxItems": 16, + "description": "The camera's pose (ARKit camera-to-world, column-major) in the requested map's world; null unless that room matched with at least 30 inliers and at most 4 px mean reprojection error. The phone applies it only when two fixes in a row agree.\n" + }, + "inliers": { + "type": "integer", + "minimum": 0, + "description": "RANSAC inliers in the requested map's world" + }, + "reproj_px": { + "type": [ + "number", + "null" + ], + "description": "Mean reprojection error of those inliers, pixels" + }, + "hint": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 3, + "description": "One-of-a-kind furniture in that room" + }, + "ms": { + "type": "integer", + "minimum": 0, + "description": "Time the Mac took" + } + } + }, "CompassRoute": { "type": "object", "description": "A walk to a thing from where its room's scan began, as compass legs (for a watch with no camera)", @@ -3498,6 +3700,48 @@ "type": "integer" } } + }, + "ReplyRequest": { + "type": "object", + "required": [ + "said" + ], + "properties": { + "said": { + "type": "string", + "minLength": 1, + "maxLength": 300, + "description": "What the person replied, as Muse Voice Transcribe heard it" + }, + "thing": { + "type": "string", + "maxLength": 80, + "description": "What they were just guided to" + } + } + }, + "ReplyIntent": { + "type": "object", + "required": [ + "intent", + "by" + ], + "properties": { + "intent": { + "type": "string", + "enum": [ + "go_back", + "done", + "question", + "unknown" + ], + "description": "unknown when the model couldn't be reached or broke the format; the phone then decides by itself" + }, + "by": { + "type": "string", + "description": "The model that read the reply" + } + } } }, "responses": { diff --git a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift index 24cb59b..0c2e2e0 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/FileStores.swift @@ -38,7 +38,18 @@ public struct FileEpisodeFiles: EpisodeFiles { } public func roomMap(scan: String) -> Data? { - try? Data(contentsOf: directory(scan: scan).appending(path: "worldmap.arworldmap")) + let dir = directory(scan: scan) + return (try? Data(contentsOf: dir.appending(path: Self.grownMap))) + ?? (try? Data(contentsOf: dir.appending(path: "worldmap.arworldmap"))) + } + + /// ARKit's map grown on later visits sits beside the scan's own (which stays exactly as uploaded and checksummed). + static let grownMap = "worldmap.grown.arworldmap" + + public func saveGrownRoomMap(scan: String, _ data: Data) { + let dir = directory(scan: scan) + guard FileManager.default.fileExists(atPath: dir.path) else { return } + try? data.write(to: dir.appending(path: Self.grownMap), options: [.atomic, .completeFileProtection]) } /// The folder, its zip and the upload's part files. diff --git a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift index 2424d99..ec2c86f 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift @@ -153,10 +153,41 @@ public final class FakeClock: Clock, @unchecked Sendable { public func showPath(_ path: [SIMD2], floorY: Double?) { paths.append(path) } /// The frame the arrival check gets (nil: the camera has none). public var frame: ArrivalFrame? = ArrivalFrame.sample - public func arrivalFrame() -> ArrivalFrame? { frame } + /// Each frame 1 cm further along x than the last (a live, tracking session), when set. + public var moves = false + private var step = 0.0 + public func arrivalFrame() -> ArrivalFrame? { + guard moves, let f = frame else { return frame } + step += 0.01 + var pose = f.pose + pose[12] += step + return ArrivalFrame(jpeg: f.jpeg, depthMM: f.depthMM, depthSize: f.depthSize, imageSize: f.imageSize, + intrinsics: f.intrinsics, pose: pose) + } + /// The photo correction in use (nil: none), and every one applied. + public private(set) var correction: Pose? + public private(set) var corrections: [Pose?] = [] + public func correct(mapFromSession: Pose?) { correction = mapFromSession; corrections.append(mapFromSession) } + /// Like ARLocator: with a correction, poses are in the map's world and tracking reads normal, until ARKit's own + /// match (a normal reading) drops it. public func reading() -> (pose: Pose, tracking: TrackingState)? { guard !readings.isEmpty else { return nil } - return readings.count > 1 ? readings.removeFirst() : readings[0] + let r = readings.count > 1 ? readings.removeFirst() : readings[0] + guard let c = correction else { return r } + if r.tracking == .normal { correction = nil; return r } + return (MapCorrection.apply(MapCorrection.matrix(c), to: r.pose), .normal) + } +} + +/// The Mac's photo matcher, answering from a script (the last answer repeats); records what it was asked. +public final class FakePlaces: PlaceLocating, @unchecked Sendable { + public var answers: [Result] + public private(set) var asked: [(scan: String, room: String?)] = [] + public init(_ answers: [Result]) { self.answers = answers } + public func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix { + asked.append((scan, room)) + guard !answers.isEmpty else { throw RecallError.server("no answer") } + return try (answers.count > 1 ? answers.removeFirst() : answers[0]).get() } } @@ -169,14 +200,30 @@ public extension Pose { } } +public struct FakeReplyReader: ReplyReading { + public var answer: FoundReply? + public var fails = false + public var hangs = false + public init(answer: FoundReply? = nil, fails: Bool = false, hangs: Bool = false) { + self.answer = answer; self.fails = fails; self.hangs = hangs + } + public func read(reply: String, about thing: String) async throws -> FoundReply? { + if hangs { try await Task.sleep(for: .seconds(30)) } + if fails { throw RecallError.status(503) } + return answer + } +} + @MainActor public final class FakeVoice: VoiceInput { public var allowed = true public var recording = Data("RIFF".utf8) public private(set) var isRecording = false + public private(set) var endedOnSilence = false private var finished: (@MainActor (Data?) -> Void)? public init() {} - public func start(maxSeconds: Double, finished: @escaping @MainActor (Data?) -> Void) async -> Bool { + public func start(maxSeconds: Double, endOnSilence: Bool, finished: @escaping @MainActor (Data?) -> Void) async -> Bool { guard allowed else { return false } + endedOnSilence = endOnSilence isRecording = true self.finished = finished return true @@ -200,6 +247,8 @@ public final class FakeEpisodeFiles: EpisodeFiles, @unchecked Sendable { public func directory(scan: String) -> URL { URL(fileURLWithPath: "/tmp/scan-\(scan)") } public func manifest(scan: String) -> Manifest? { manifests[scan] } public func roomMap(scan: String) -> Data? { maps[scan] } + public private(set) var grown: [String: Data] = [:] + public func saveGrownRoomMap(scan: String, _ data: Data) { grown[scan] = data; maps[scan] = data } public func delete(scan: String) { deleted.append(scan) } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift b/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift index 0b1367e..734aee6 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift @@ -8,12 +8,14 @@ public final class WavRecorder: NSObject, VoiceInput { private var recorder: AVAudioRecorder? private var finished: (@MainActor (Data?) -> Void)? private let file = URL.temporaryDirectory.appending(path: "question.wav") + private var watching: Task? + private var nothingSaid = false public override init() { super.init() } public var isRecording: Bool { recorder?.isRecording == true } - public func start(maxSeconds: Double, finished: @escaping @MainActor (Data?) -> Void) async -> Bool { + public func start(maxSeconds: Double, endOnSilence: Bool, finished: @escaping @MainActor (Data?) -> Void) async -> Bool { guard await AVAudioApplication.requestRecordPermission() else { return false } let session = AVAudioSession.sharedInstance() try? session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .duckOthers]) @@ -26,18 +28,43 @@ public final class WavRecorder: NSObject, VoiceInput { r.delegate = self recorder = r self.finished = finished - return r.record(forDuration: maxSeconds) + nothingSaid = false + r.isMeteringEnabled = endOnSilence + guard r.record(forDuration: maxSeconds) else { return false } + if endOnSilence { watch(r) } + return true + } + + /// Reads the microphone's level ten times a second and ends the turn when the talking has stopped. + private func watch(_ r: AVAudioRecorder) { + watching = Task { [weak self] in + var gate = SilenceGate() + while !Task.isCancelled, r.isRecording { + try? await Task.sleep(for: .milliseconds(100)) + r.updateMeters() + switch gate.hear(level: Double(r.averagePower(forChannel: 0)), at: r.currentTime) { + case .listen: continue + case .finished: r.stop() + case .nothingSaid: + self?.nothingSaid = true + r.stop() + } + return + } + } } /// Stops early; the delegate then hands over the recording (the same path as running out of time). public func stop() { recorder?.stop() } private func done(_ ok: Bool) { + watching?.cancel() + watching = nil recorder = nil try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio, options: [.duckOthers]) let handler = finished finished = nil - let audio = ok ? try? Data(contentsOf: file) : nil + let audio = ok && !nothingSaid ? try? Data(contentsOf: file) : nil try? FileManager.default.removeItem(at: file) // the question is in memory now; no copy is left on the phone handler?(audio) } diff --git a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift new file mode 100644 index 0000000..7800e40 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/PhotoAssistTests.swift @@ -0,0 +1,98 @@ +import Foundation +import RecallDomain +import RecallTestSupport +import Testing +@testable import RecallApplication + +private let mapPose = Pose(columnMajor: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 1.4, -1, 1]) + +/// A locator that hasn't recognised the room, whose camera moves a little between photos (a live session). +@MainActor private func limitedLocator() -> FakeLocator { + let l = FakeLocator(readings: [(.identity, .limited)]) + l.moves = true + return l +} + +@MainActor private func assist(_ locator: FakeLocator, _ places: FakePlaces, _ speech: FakeSpeech, _ clock: FakeClock, + room: String? = "Living room") -> PhotoAssist { + var s = PhotoAssist.Settings() + s.maxAsks = 6 + return PhotoAssist(home: "flat", scan: "s1", room: room, locator: locator, places: places, speech: speech, clock: clock, + settings: s) +} + +@MainActor @Test func whenARKitMatchesFirstNoPhotoIsSent() async { + let locator = FakeLocator(readings: [(.identity, .normal)]) + let places = FakePlaces([.success(PlaceFix(room: nil, pose: nil, inliers: 0, hint: []))]) + let a = assist(locator, places, FakeSpeech(), FakeClock()) + await a.run() + #expect(places.asked.isEmpty && !a.fixed) +} + +@MainActor @Test func twoAgreeingFixesLocateThePhoneAtOnce() async { + let locator = limitedLocator(), clock = FakeClock() + let places = FakePlaces([.success(PlaceFix(room: "Living room", pose: mapPose, inliers: 80, hint: ["tv"]))]) + let speech = FakeSpeech() + let a = assist(locator, places, speech, clock) + await a.run() + #expect(a.fixed && places.asked.count == 2 && places.asked[0].scan == "s1") + #expect(locator.correction != nil) + #expect(locator.reading()?.tracking == .normal) // the guide can plan now + #expect(speech.lines.isEmpty) // the opening line said how to move; it was found before another was due +} + +@MainActor @Test func itNeverSendsAnyoneToFindFurniture() async { + let locator = limitedLocator(), clock = FakeClock() + let places = FakePlaces([.success(PlaceFix(room: "Living room", pose: nil, inliers: 4, hint: ["tv", "shelf"]))]) + let speech = FakeSpeech() + var s = PhotoAssist.Settings() + s.maxAsks = 12 + let a = PhotoAssist(home: "flat", scan: "s1", room: "Living room", locator: locator, places: places, speech: speech, + clock: clock, settings: s) + await a.run() + #expect(!speech.lines.isEmpty && speech.lines.allSatisfy { $0.hasPrefix("Keep turning") }) +} + +@MainActor @Test func inAnotherRoomItSaysSoOnceAndGivesNoFix() async { + let locator = limitedLocator(), clock = FakeClock() + let places = FakePlaces([.success(PlaceFix(room: "Bedroom", pose: nil, inliers: 0, hint: ["bed"]))]) + let speech = FakeSpeech() + let a = assist(locator, places, speech, clock) + await a.run() + #expect(!a.fixed && locator.corrections.isEmpty && places.asked.count == 6) + #expect(speech.lines == ["You seem to be in Bedroom, not Living room. Walk to Living room and I'll keep looking."]) +} + +@MainActor @Test func withoutTheMacNothingChanges() async { + let locator = limitedLocator(), clock = FakeClock() + let speech = FakeSpeech() + let a = assist(locator, FakePlaces([.failure(RecallError.status(503))]), speech, clock) + await a.run() + #expect(!a.fixed && locator.corrections.isEmpty && speech.lines.isEmpty) +} + +@MainActor @Test func fixesThatDisagreeAreNotUsed() async { + let locator = limitedLocator(), clock = FakeClock() + let elsewhere = Pose(columnMajor: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, -2, 1.4, 3, 1]) + let fix = { (p: Pose) in Result.success(PlaceFix(room: nil, pose: p, inliers: 40, hint: [])) } + let places = FakePlaces([fix(mapPose), fix(elsewhere), fix(mapPose), fix(elsewhere), fix(mapPose), fix(elsewhere)]) + let a = assist(locator, places, FakeSpeech(), clock) + await a.run() + #expect(!a.fixed && locator.corrections.isEmpty) +} + +@MainActor @Test func theGuideIsLocatedByPhotosWhenARKitIsSlow() async { + let locator = limitedLocator(), clock = FakeClock() + let places = FakePlaces([.success(PlaceFix(room: "Office", pose: mapPose, inliers: 80, hint: []))]) + let service = FakeGuide(routes: [Route(sessionId: "gs", landmarksOnly: true, reason: "no clear path", + steps: [.init(clock: 12, metres: 2, say: "Your keys are on the desk.")], + waypoints: [], arriveSay: "")]) + var settings = GuideToThing.Settings() + settings.locateTimeout = 10 + let g = GuideToThing(home: "flat", thing: Thing.sample(), roomMap: Data("map".utf8), locator: locator, guide: service, + speech: FakeSpeech(), haptics: FakeHaptics(), clock: clock, photos: (places: places, scan: "s1"), + settings: settings) + await g.run() + #expect(locator.correction != nil || !locator.corrections.isEmpty) + #expect(service.routeRequests.first?.tracking == .normal) // planned as located, not landmarks-only for weak tracking +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift index 612a55e..a168855 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift @@ -45,7 +45,7 @@ struct GuideToThingTests { #expect(phases.contains { if case .guiding(let c, _) = $0 { c.clock == 3 } else { false } }) #expect(speech.lines.first == "Finding where you are. Hold the phone up at chest height and turn slowly.") #expect(speech.lines.contains("Straight ahead, then 2 metres.")) - #expect(speech.lines.contains("Your keys should be in front of you, on the desk. Hold the phone towards it. " + #expect(speech.lines.contains("Your keys should be in front of you, on the desk. Hold the phone out in front of you. " + "I'm checking with the camera.")) #expect(speech.lines.last == "Found it. Your keys are at 12 o'clock, 40 centimetres away.") // the camera check has the last word #expect(service.verified.count == 1) @@ -67,6 +67,32 @@ struct GuideToThingTests { #expect(g.phase == .arrived("Found it. Your keys are at 12 o'clock, 40 centimetres away.")) } + @Test func afterAFindItWalksBackAlongTheSameRoute() async { + let locator = FakeLocator(readings: [(.at(x: 0, z: 0), .normal), (.at(x: 0, z: 0), .normal), + (.at(x: 0, z: -3.7), .normal), (.at(x: 0, z: -3.7), .normal), + (.at(x: 0, z: -2), .normal), (.at(x: 0, z: -0.2), .normal)]) + locator.frame = .sample + let service = FakeGuide(routes: [route([[0, 0], [0, -4]])]) + service.check = ArrivalCheck(outcome: .confirmed, say: "Found it.") + let (g, speech, _, _) = guide(locator: locator, service: service) + await g.run() + #expect(g.phase == .arrived("Found it.") && g.canWalkBack) + await g.walkBack() + #expect(g.phase == .arrived("You're back.")) + #expect(speech.lines.contains("Going back.") && speech.lines.last == "You're back.") + #expect(service.routeRequests.count == 1) // no new route from the Mac: the same path, reversed + } + + @Test func thereIsNothingToGoBackAlongWithoutAWalkedRoute() async { + let locator = FakeLocator(readings: [(.at(x: 3, z: 3), .normal)]) + let (g, speech, _, _) = guide(locator: locator, service: FakeGuide(routes: [route([], landmarks: true)])) + await g.run() + #expect(!g.canWalkBack) + let said = speech.lines.count + await g.walkBack() + #expect(speech.lines.count == said) + } + @Test func arrivingChecksWithTheCamera() async { func arrive(check: ArrivalCheck?, frame: ArrivalFrame? = .sample) async -> (GuideToThing, FakeSpeech, FakeHaptics, FakeGuide) { let locator = FakeLocator(readings: [(.at(x: 0, z: 0), .normal), (.at(x: 0, z: 0), .normal), (.at(x: 0, z: -3.7), .normal)]) @@ -311,3 +337,82 @@ struct QuietOpenTests { #expect(speech.lines.count == 1) } } + +@Suite("Asking with the AR view open") @MainActor +struct AskInARTests { + private let at = Date(timeIntervalSince1970: 0) + private func thing(_ id: String, _ colour: String, _ noun: String) -> Thing { + Thing(id: id, label: noun, color: colour, heightM: 0.8, heightBand: "waist", confidence: 0.9, seenAt: at) + } + private func label(_ t: Thing) -> ArLabel { + ArLabel(id: t.id, title: t.title, place: t.place, confidence: 0.9, photos: 3, trusted: true, needsReview: false, + position: .zero) + } + + private func outcome(_ answer: Answer, question: String, known: [Thing], labels: [ArLabel]) async throws -> AskInAR.Outcome { + let use = AskInAR(ask: AskWhereIs(guide: FakeGuide(answer: answer))) + return try await use(question, home: "flat", known: known, labels: labels) + } + + @Test func aThingInThisRoomIsWalkedTo() async throws { + let keys = thing("k", "brown", "keys") + let got = try await outcome(Answer(text: "Near the desk.", thingId: "k", grounded: true), question: "my keys", + known: [keys], labels: [label(keys)]) + #expect(got == .walk(keys, say: "Near the desk.")) + } + + @Test func severalAlikeAskWhichOneInsteadOfWalking() async throws { + let (grey, black) = (thing("g", "grey", "laptop"), thing("b", "black", "laptop")) + let got = try await outcome(Answer(text: "I saw two laptops.", thingId: "g", grounded: true), question: "my laptop", + known: [grey, black], labels: [label(grey), label(black)]) + #expect(got == .which(say: "I saw two laptops. Double tap the one you mean, or say which.")) + } + + @Test func aThingInAnotherRoomIsOnlyAnswered() async throws { + let (keys, passport) = (thing("k", "brown", "keys"), thing("p", "grey", "passport")) + let got = try await outcome(Answer(text: "In the bedroom.", thingId: "p", grounded: true), question: "my passport", + known: [keys, passport], labels: [label(keys)]) + #expect(got == .answer("In the bedroom.")) + } +} + +@Suite("Guiding inside the AR view") @MainActor +struct GuideKeepsTheCameraTests { + @Test func theSessionStaysOnWhenTheWalkEnds() async { + let locator = FakeLocator(readings: [(.at(x: 0, z: 0), .normal), (.at(x: 0, z: 0), .normal), (.at(x: 0, z: 0), .normal)]) + var settings = GuideToThing.Settings() + settings.keepsSession = true + let thing = Thing(id: "k", label: "keys", heightM: 0.8, heightBand: "waist", confidence: 0.9, seenAt: Date(timeIntervalSince1970: 0)) + let g = GuideToThing(home: "flat", thing: thing, roomMap: Data([1]), locator: locator, + guide: FakeGuide(error: RecallError.status(500)), speech: FakeSpeech(), haptics: FakeHaptics(), + clock: FakeClock(), settings: settings) + await g.run() // the route fails: the walk ends + g.stop() + #expect(!locator.stopped) + #expect(locator.marks.last == .some(nil)) // its arrow is gone all the same + } +} + +@Suite("Replies after a find") struct ReadFoundReplyTests { + @Test func theModelsReadingWins() async { + let read = ReadFoundReply(reader: FakeReplyReader(answer: .goBack)) + #expect(await read("get me out of here", about: "keys") == .goBack) + // the rules would have called this a new question: the model understood it + let done = ReadFoundReply(reader: FakeReplyReader(answer: .done)) + #expect(await done("nah I'm good", about: "keys") == .done) + } + + @Test func thePhoneDecidesWhenTheMacCantTell() async { + #expect(await ReadFoundReply(reader: FakeReplyReader(answer: nil))("no thank you", about: "keys") == .done) + } + + @Test func aSlowModelDoesntKeepAnyoneWaiting() async { + let read = ReadFoundReply(reader: FakeReplyReader(answer: .goBack, hangs: true), patience: 0.05) + #expect(await read("no thank you", about: "keys") == .done) + } + + @Test func thePhoneDecidesWhenTheMacIsUnreachable() async { + #expect(await ReadFoundReply(reader: FakeReplyReader(fails: true))("take me back", about: "keys") == .goBack) + #expect(await ReadFoundReply(reader: nil)("where's my wallet", about: "keys") == .question) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift index 40183bc..f47b4a7 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift @@ -37,3 +37,42 @@ struct ArLabelTests { #expect(ScanSpeed.text(0.42) == "0.4 m/s") } } + +@Suite("What a spoken question settles on in AR") +struct ARPickTests { + private func label(_ id: String, _ title: String) -> ArLabel { + ArLabel(id: id, title: title, place: "near the desk", confidence: 0.9, photos: 3, trusted: true, needsReview: false, + position: .zero) + } + private var room: [ArLabel] { [label("a", "grey laptop"), label("b", "black laptop"), label("c", "brown keys")] } + + @Test func oneOfAKindWalks() { + #expect(ARPick.settle(thingId: "c", noun: "keys", question: "where are my keys?", labels: room) == .walk(room[2])) + } + + @Test func severalAlikeAskWhichEvenThoughTheAnswerNamedOne() { + let pick = ARPick.settle(thingId: "a", noun: "laptop", question: "where is my laptop", labels: room) + #expect(pick == .which([room[0], room[1]])) + } + + @Test func theWordsSaidChooseAmongThem() { + // the answer was about the grey one, but the person said black + #expect(ARPick.settle(thingId: "a", noun: "laptop", question: "the black laptop", labels: room) == .walk(room[1])) + #expect(ARPick.settle(thingId: "a", noun: "laptop", question: "take me to the black one", labels: room) == .walk(room[1])) + } + + @Test func twoTheSameStillAsk() { + let bottles = [label("a", "grey bottle"), label("b", "grey bottle")] + #expect(ARPick.settle(thingId: "a", noun: "bottle", question: "the grey bottle", labels: bottles) == .which(bottles)) + } + + @Test func aThingElsewhereIsNotWalkedTo() { + #expect(ARPick.settle(thingId: "z", noun: "passport", question: "my passport", labels: room) == .elsewhere) + #expect(ARPick.settle(thingId: nil, noun: nil, question: "my umbrella", labels: room) == .elsewhere) + } + + @Test func aThingWithItsOwnNameWalks() { + let named = [label("a", "Dad's mug"), label("b", "blue mug")] + #expect(ARPick.settle(thingId: "a", noun: nil, question: "dad's mug", labels: named) == .walk(named[0])) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/DomainTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/DomainTests.swift index e6cf491..661b848 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/DomainTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/DomainTests.swift @@ -114,3 +114,50 @@ private func pose(x: Float = 0, z: Float = 0, yawDegrees: Float = 0) -> simd_flo #expect(Array(m.columnMajor[12...14]) == [1, 2, 3]) } } + +@Suite("After a find") struct FoundReplyTests { + @Test func goingBack() { + for said in ["Yes", "yes please", "Go back.", "Take me back", "No, go back", "return to the start", "Yeah"] { + #expect(FoundReply.read(said) == .goBack, "\(said)") + } + } + + @Test func done() { + for said in ["No, thank you.", "no thanks", "Thank you", "I'm done", "Don't go back", "no need to go back", "Nope"] { + #expect(FoundReply.read(said) == .done, "\(said)") + } + } + + @Test func anythingElseIsANewQuestion() { + for said in ["Where's my wallet?", "No, where are my keys?", "Find my glasses", "wallet", "The black one"] { + #expect(FoundReply.read(said) == .question, "\(said)") + } + } + + @Test func aCurlyApostropheCounts() { + #expect(FoundReply.read("Don\u{2019}t go back") == .done) + #expect(FoundReply.read("Where\u{2019}s my watch") == .question) + } +} + +@Suite("Turn taking") struct TurnTests { + @Test func aTurnEndsAfterTheTalkingStops() { + var gate = SilenceGate() + #expect(gate.hear(level: -50, at: 0.5) == .listen) + #expect(gate.hear(level: -20, at: 1.0) == .listen) + #expect(gate.hear(level: -50, at: 2.0) == .listen) // a pause between words + #expect(gate.hear(level: -22, at: 2.1) == .listen) + #expect(gate.hear(level: -50, at: 3.2) == .listen) + #expect(gate.hear(level: -50, at: 3.4) == .finished) + } + + @Test func silenceGivesUpWithNothingToSend() { + var gate = SilenceGate() + #expect(gate.hear(level: -50, at: 7.9) == .listen) + #expect(gate.hear(level: -50, at: 8.0) == .nothingSaid) + } + + @Test func aLongerLineTakesLonger() { + #expect(SpokenLength.seconds("What are you looking for?") < SpokenLength.seconds("Your grey watch is here, but moved: at 12 o'clock.")) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift new file mode 100644 index 0000000..5a14ff6 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PhotoFixTests.swift @@ -0,0 +1,46 @@ +import Foundation +import simd +import Testing +@testable import RecallDomain + +/// A pose at (x, y, z), turned `yaw` degrees about the vertical (ARKit y). +private func pose(_ x: Double, _ y: Double, _ z: Double, yaw: Double = 0) -> Pose { + let a = yaw * .pi / 180 + return Pose(columnMajor: [cos(a), 0, -sin(a), 0, 0, 1, 0, 0, sin(a), 0, cos(a), 0, x, y, z, 1]) +} + +@Test func aCorrectionTurnsSessionPosesIntoTheMapsWorld() { + let T = MapCorrection.matrix(pose(2, 0, -1, yaw: 30)) // the session's world inside the map's + let session = pose(0.5, 1.4, 0.2, yaw: -10) + let map = MapCorrection.apply(T, to: session) + let back = MapCorrection.between(map: map, session: session) + let d = MapCorrection.distance(back, T) + #expect(d.metres < 1e-9 && d.degrees < 1e-6) + // a later session pose lands where the same correction puts it + let later = MapCorrection.apply(back, to: pose(1.5, 1.4, 0.2, yaw: -10)) + #expect(abs(simd_distance(SIMD2(later.columnMajor[12], later.columnMajor[14]), + SIMD2(map.columnMajor[12], map.columnMajor[14])) - 1) < 1e-9) +} + +@Test func twoFixesMustAgreeAndComeFromALiveSession() { + var a = FixAgreement() + let T1 = MapCorrection.matrix(pose(2, 0, -1, yaw: 30)), near = MapCorrection.matrix(pose(2.1, 0, -1, yaw: 33)) + #expect(a.add(T1, session: pose(0, 1.4, 0)) == nil) // one fix proves nothing + #expect(a.add(near, session: pose(0.05, 1.4, 0)) != nil) // 10 cm and 3° apart: agreed + + var far = FixAgreement() + _ = far.add(T1, session: pose(0, 1.4, 0)) + #expect(far.add(MapCorrection.matrix(pose(3, 0, -1, yaw: 30)), session: pose(0.05, 1.4, 0)) == nil) // 1 m apart + #expect(far.add(MapCorrection.matrix(pose(3, 0, -1, yaw: 31)), session: pose(0.1, 1.4, 0)) != nil) // then two agree + + var frozen = FixAgreement() + _ = frozen.add(T1, session: pose(0, 1.4, 0)) + #expect(frozen.add(T1, session: pose(0, 1.4, 0)) == nil) // the same session pose twice: ARKit isn't tracking +} + +@Test func coachingNamesThingsToPointAt() { + // what to do with your body, never where to look: nobody has to find a thing they can't see + for turn in 0..<3 { #expect(!LocateCoaching.keepTurning(turn: turn).contains("Point")) } + #expect(LocateCoaching.elsewhere(found: "Bedroom", wanted: "Living room") + == "You seem to be in Bedroom, not Living room. Walk to Living room and I'll keep looking.") +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift index ef76787..fa58b4c 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift @@ -57,7 +57,7 @@ struct ScanStepsTests { var m = metrics() m.zonesSeen = 12 m.zonesTotal = 40 - #expect(ScanTips.make(m) == ["Height zones seen 12 of 40. Point the phone at knee height and up at shelves too, not only straight ahead."]) + #expect(ScanTips.make(m) == ["Height zones seen 12 of 40. Sweep the phone from knee height to above your head, not only straight ahead."]) } @Test func tipsSayWhatToDoBetterAtMostThree() { diff --git a/apps/ios/Recall/App/AppContainer.swift b/apps/ios/Recall/App/AppContainer.swift index c26ec85..30d3d04 100644 --- a/apps/ios/Recall/App/AppContainer.swift +++ b/apps/ios/Recall/App/AppContainer.swift @@ -67,7 +67,26 @@ final class AppContainer { idle?.cancel() let scans = scans return ARLabelsModel(room: room, home: settings.connection.homeId, source: arLabelsSource(), locator: locator, - roomMap: { scans.roomMap(scan: $0) }, speech: speech) + roomMap: { scans.roomMap(scan: $0) }, saveGrownMap: { scans.saveGrownMap(scan: $0, $1) }, + places: UITestSamples.enabled ? nil : server, + asking: .init(ask: AskInAR(ask: AskWhereIs(guide: UITestSamples.enabled ? UITestSamples.Guide() : server)), + transcriber: server, replies: ReadFoundReply(reader: UITestSamples.enabled ? nil : server), + voice: WavRecorder(), things: things, + makeGuide: { thing, map in self.arGuideModel(for: thing, map: map) }), + speech: speech, haptics: haptics, clock: clock) + } + + /// A walk from the AR view: in the room map its labels are placed in, leaving its camera running at the end. + private func arGuideModel(for thing: Thing, map: (scan: String, data: Data)?) -> GuideModel { + guiding = true + var keepCamera = GuideToThing.Settings() + keepCamera.keepsSession = true + let use = GuideToThing(home: settings.connection.homeId, thing: thing, roomMap: map?.data, locator: locator, + guide: UITestSamples.enabled ? UITestSamples.Guide() : server, speech: speech, + haptics: haptics, clock: clock, compass: compass, + photos: map.map { (places: server, scan: $0.scan) }, settings: keepCamera) + return GuideModel(guide: use, session: locator.session, scene: locator.scene, supported: ARLocator.isSupported, + ended: { [weak self] in self?.guiding = false }) } func arLabelsSource() -> any ArLabelsSource { @@ -93,13 +112,14 @@ final class AppContainer { func findModel() -> FindModel { FindModel(ask: AskWhereIs(guide: UITestSamples.enabled ? UITestSamples.Guide() : server), transcriber: server, voice: WavRecorder(), things: things, settings: settings, speech: speech, haptics: haptics, - prepareGuide: { [weak self] in self?.prepareGuide() }) + prepareGuide: { [weak self] thing in self?.prepareGuide(room: thing.room) }) } - /// Starts recognising the saved room in the background; paused again if no guide starts within a minute. - func prepareGuide() { - guard ARLocator.isSupported, !guiding, let map = scans.roomMap() else { return } - locator.prepare(roomMap: map) + /// Starts recognising the thing's room in the background; paused again if no guide starts within a minute. + func prepareGuide(room: String?) { + guard ARLocator.isSupported, !guiding, let map = scans.guideMap(room: room) else { return } + growMap(scan: map.scan) + locator.prepare(roomMap: map.data) idle?.cancel() idle = Task { [weak self] in try? await Task.sleep(for: .seconds(60)) @@ -111,12 +131,21 @@ final class AppContainer { func guideModel(for thing: Thing) -> GuideModel { idle?.cancel() guiding = true - let use = GuideToThing(home: settings.connection.homeId, thing: thing, roomMap: scans.roomMap(), locator: locator, - guide: server, speech: speech, haptics: haptics, clock: clock, compass: compass) + let map = scans.guideMap(room: thing.room) + if let map { growMap(scan: map.scan) } + let use = GuideToThing(home: settings.connection.homeId, thing: thing, roomMap: map?.data, locator: locator, + guide: server, speech: speech, haptics: haptics, clock: clock, compass: compass, + photos: map.map { (places: server, scan: $0.scan) }) return GuideModel(guide: use, session: locator.session, scene: locator.scene, supported: ARLocator.isSupported, ended: { [weak self] in self?.guiding = false }) } + /// When ARKit recognises this scan's room, the map it grew is kept for next time. + private func growMap(scan: String) { + let scans = scans + locator.onMapGrown = { data in scans.saveGrownMap(scan: scan, data) } + } + /// `again`: a processed scan of this room to scan again in the same frame (its room map is recognised first). func scanModel(room: String, again: Scan? = nil) -> ScanModel { idle?.cancel() @@ -135,7 +164,7 @@ struct SystemClock: Clock { /// The Mac, as every Mac-facing port, reached with the current connection settings. Changing Settings swaps the /// client underneath; a call made before setup throws the setup issue, in words. -final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, ConnectionCheck, ScanServer, RobotSharing, +final class LiveServer: HomeMemory, GuideService, PlaceLocating, Transcriber, ReplyReading, ScanProcessing, ConnectionCheck, ScanServer, RobotSharing, RoomMaps, RobotPackages, CompassRoutes, ArLabelsSource, TrustSettings, PrivacySource, EvidenceSource, @unchecked Sendable { private let lock = NSLock() private var api: RecallAPI? @@ -179,6 +208,9 @@ final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, C func arLabels(home: String, room: String) async throws -> ArLabels { try await current().arLabels(home: home, room: room) } + func locate(home: String, scan: String, room: String?, frame: ArrivalFrame) async throws -> PlaceFix { + try await current().locate(home: home, scan: scan, room: room, frame: frame) + } func compassRoute(home: String, thing: String) async throws -> CompassRoute { try await current().compassRoute(home: home, thing: thing) } @@ -192,6 +224,9 @@ final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, C } func activity(limit: Int) async throws -> [AuditEntry] { try await current().activity(limit: limit) } func transcribe(wav: Data) async throws -> String { try await current().transcribe(wav: wav) } + func read(reply: String, about thing: String) async throws -> FoundReply? { + try await current().read(reply: reply, about: thing) + } func status(scan: String) async throws -> ScanReport { try await current().status(scan: scan) } func erase(scan: String) async throws { try await current().erase(scan: scan) } func privacy() async throws -> PrivacyStatus { try await current().privacy() } diff --git a/apps/ios/Recall/App/UITestSamples.swift b/apps/ios/Recall/App/UITestSamples.swift index 0450c79..d1634bd 100644 --- a/apps/ios/Recall/App/UITestSamples.swift +++ b/apps/ios/Recall/App/UITestSamples.swift @@ -98,7 +98,7 @@ enum UITestSamples { PrivacyStatus(storedOn: "This Mac", sent: [ DataFlow(what: "Scan photos, about 50 per room (no depth, no sound)", to: "Meta Model API: SAM 3.1", when: "While this Mac processes a scan, to find furniture and things"), - DataFlow(what: "Your question as text", to: "Baseten: Muse Spark Open", when: "Each time you ask"), + DataFlow(what: "Your question, or your reply after a find, as text", to: "Baseten: Muse Spark Open", when: "Each time you ask"), ], robotAccess: false, auditDays: 90) } } diff --git a/apps/ios/Recall/Features/AR/ARLabelsModel.swift b/apps/ios/Recall/Features/AR/ARLabelsModel.swift index da326b2..33d0f79 100644 --- a/apps/ios/Recall/Features/AR/ARLabelsModel.swift +++ b/apps/ios/Recall/Features/AR/ARLabelsModel.swift @@ -1,18 +1,42 @@ import ARKit import Foundation import Observation +import RecallApplication import RecallCapture import RecallDomain +import SceneKit import SwiftUI /// "See it in AR": recognise the room in its saved map, then float a label over everything Recall found there. The /// labels are SAM 3.1's results from the Mac, placed by MapAnything's geometry; nothing is recognised on the phone. +/// The view stays open: double tap a label, or say what you're looking for, and the walk there runs in this same camera +/// (the same ARKit session, so nothing has to be recognised again), then the labels come back. It runs by voice: once +/// the room is recognised it asks what you're looking for and listens, and after a find it asks whether to go back. @MainActor @Observable final class ARLabelsModel { enum Phase: Equatable { case loading, locating, showing, listOnly(String), failed(String) } + enum Hearing: Equatable { case idle, listening, thinking } + + /// What a question or a double tap needs: `things` to name what a label is, `makeGuide` to walk there in the map + /// the labels are placed in. Nil in a preview of the list alone. + struct Asking { + let ask: AskInAR + let transcriber: any Transcriber + let replies: ReadFoundReply + let voice: any VoiceInput + let things: ThingsModel + let makeGuide: @MainActor (Thing, (scan: String, data: Data)?) -> GuideModel + } private(set) var phase = Phase.loading private(set) var labels: [ArLabel] = [] + private(set) var hearing = Hearing.idle + /// What the person said, as Muse Voice Transcribe heard it: shown lightly over the camera so a wrong word is seen. + private(set) var heard: String? + /// The walk under way: its directions replace the list; the camera stays. + private(set) var guide: GuideModel? + /// Walking back the way the person came, after a find. + private(set) var isReturning = false var look = CameraLook.camera let room: String let locator: ARLocator @@ -20,13 +44,38 @@ final class ARLabelsModel { private let source: any ArLabelsSource private let home: String private let roomMap: (String) -> Data? + private let saveGrownMap: (String, Data) -> Void + private let places: (any PlaceLocating)? private let speech: any SpeechOutput + private let haptics: (any HapticsOutput)? + private let clock: any Clock private var task: Task? + private var assist: PhotoAssist? + private let asking: Asking? + /// The room map the labels are placed in: a walk from this view must use the same one. + private var mapInUse: (scan: String, data: Data)? + /// The thing the last answer was about, so "the black one" means something. + private var lastAbout: String? + private var starting = false + /// The next thing heard answers "Do you want to go back, or are you done?". + private var afterFound = false + /// The next recording is thrown away: it was cut off by Done, a double tap or leaving the view. + private var dropping = false + private var followUp: Task? + /// Seconds before the list is shown instead, and how long the room is still looked for after that. + static let listAfter = 25.0 + static let keepLooking = 120.0 + + /// `places`: the Mac's photo matcher, to recognise the room alongside ARKit; `saveGrownMap`: keeps the map ARKit + /// grew on this visit, so the next one recognises the room from here too. init(room: String, home: String, source: any ArLabelsSource, locator: ARLocator, roomMap: @escaping (String) -> Data?, - speech: any SpeechOutput) { + saveGrownMap: @escaping (String, Data) -> Void = { _, _ in }, places: (any PlaceLocating)? = nil, + asking: Asking? = nil, speech: any SpeechOutput, haptics: (any HapticsOutput)? = nil, + clock: any Clock = SystemClock()) { self.room = room; self.home = home; self.source = source; self.locator = locator; self.roomMap = roomMap - self.speech = speech + self.saveGrownMap = saveGrownMap; self.places = places; self.speech = speech; self.haptics = haptics + self.clock = clock; self.asking = asking } func start() { @@ -35,17 +84,216 @@ final class ARLabelsModel { func stop() { task?.cancel() + assist?.cancel() + guide?.stop() + guide = nil + stopListening() + locator.onMapGrown = nil locator.clearLabels() locator.stop() } - /// What the camera points at, said aloud (Magic Tap). - func sayInView() { + /// Whether a label can be walked to right now (not while the room data loads, nor during a walk). + var canGo: Bool { asking != nil && guide == nil && !labels.isEmpty } + + /// VoiceOver's two-finger double tap: directions to what the camera points at. + func goToLabelInView() { + guard canGo else { return } guard let label = locator.labelInView() else { - speech.say("Point the phone at a label.", interrupt: true) + speech.say("There's no label in front of the phone. Turn a little, or say what you're looking for.", interrupt: true) + return + } + go(to: label) + } + + /// A double tap on the camera: directions to the label under the finger. + func doubleTapped(at point: CGPoint, in view: SCNSceneRenderer) { + guard canGo, phase == .showing else { return } + guard let label = locator.label(at: point, in: view) else { + speech.say("Double tap on a label.", interrupt: true) + return + } + go(to: label) + } + + /// Directions to a label, from the camera or the list. + func go(to label: ArLabel) { + guard canGo else { return } + // the Mac plans no route to something seen in one photo: it waits for the owner to confirm it on Found + guard label.mark != .seenOnce else { + speech.say("Recall saw your \(label.title) only once, so it can't guide you there yet. It's \(label.place). " + + "Confirm it on Found to get directions.", interrupt: true) + return + } + haptics?.tap() + Task { + await asking?.things.refreshIfNeeded(for: label.id) + guard let thing = asking?.things.things.first(where: { $0.id == label.id }) else { + speech.say("I can't reach the Mac for \(label.title) right now. It's \(label.place).", interrupt: true) + return + } + await walk(to: thing, saying: "Directions to your \(label.title).") + } + } + + /// Tap to talk, tap to stop; a turn also ends when you stop talking, and after 15 s at most. + func toggleListening() async { + guard let asking else { return } + if asking.voice.isRecording { + asking.voice.stop() + return + } + guard guide == nil else { return } + await listen() + } + + /// Opens the microphone; the tap on the phone tells someone who can't see the screen that it's their turn. + private func listen() async { + guard let asking, hearing == .idle, !asking.voice.isRecording else { return } + haptics?.tap() + heard = nil + let started = await asking.voice.start(maxSeconds: 15, endOnSilence: true) { [weak self] wav in + Task { await self?.heard(wav) } + } + hearing = started ? .listening : .idle + if !started { speech.say(UserMessage.explain(RecallError.noMicrophone), interrupt: true) } + } + + /// Ends a turn nobody wants the words of any more. + private func stopListening() { + followUp?.cancel() + afterFound = false + guard asking?.voice.isRecording == true else { return } + dropping = true + asking?.voice.stop() + } + + /// Waits for a line to end: the speech engine says when; under VoiceOver only an estimate is possible. + private func spoken(_ line: String) async { + try? await clock.sleep(seconds: 0.2) + let started = clock.now() + while speech.isSpeaking, clock.now().timeIntervalSince(started) < 12 { + try? await clock.sleep(seconds: 0.1) + } + if UIAccessibility.isVoiceOverRunning { try? await clock.sleep(seconds: SpokenLength.seconds(line)) } + } + + /// Asks out loud, then listens without a tap. + private func ask(_ line: String, interrupt: Bool = true) async { + speech.say(line, interrupt: interrupt) + await spoken(line) + guard !Task.isCancelled else { return } + await listen() + } + + private func heard(_ wav: Data?) async { + guard let asking, let wav, !wav.isEmpty, !dropping else { + dropping = false + hearing = .idle + return + } + hearing = .thinking + do { + let question = try await asking.transcriber.transcribe(wav: wav).trimmingCharacters(in: .whitespacesAndNewlines) + guard !question.isEmpty else { + hearing = .idle + return + } + heard = question + if afterFound { + afterFound = false + switch await asking.replies(question, about: guide?.thing.title ?? "") { + case .goBack: + hearing = .idle + return goBack() + case .done: + hearing = .idle + endWalk() + return speech.say("You're welcome.", interrupt: true) + case .question: + endWalk() // the next thing: answered below like any question + heard = question + } + } + speech.say(Acknowledgement.line(for: question, known: asking.things.things), interrupt: true) + if !asking.things.loaded { await asking.things.refresh() } + let outcome = try await asking.ask(question, home: home, known: asking.things.things, labels: labels, + about: lastAbout) + hearing = .idle + switch outcome { + case .walk(let thing, let say): + lastAbout = thing.id + await walk(to: thing, saying: say) + case .which(let say), .answer(let say): + speech.say(say, interrupt: true) + } + } catch { + hearing = .idle + speech.say(UserMessage.explain(error), interrupt: true) + } + } + + /// Says the line to the end, then walks: the first direction would cut it off otherwise. + private func walk(to thing: Thing, saying line: String) async { + guard let asking, guide == nil, !starting else { return } + starting = true + defer { starting = false } + task?.cancel() // still looking for the room: the guide does that itself now + assist?.cancel() + stopListening() + speech.say(line, interrupt: true) + await spoken(line) + locator.focus(label: thing.id) + let walk = asking.makeGuide(thing, mapInUse) + walk.onFinished = { [weak self] in self?.walkFinished($0) } + guide = walk + walk.start() + } + + /// The walk reached its end. At the thing: ask what next. Back at the start: ask for the next thing. + private func walkFinished(_ end: GuideToThing.Phase) { + guard case .arrived(let said) = end else { return } + if isReturning { + endWalk() + followUp = Task { [weak self] in await self?.ask("You're back. What else are you looking for?") } return } - speech.say(label.spoken, interrupt: true) + followUp = Task { [weak self] in + await self?.spoken(said) + guard !Task.isCancelled else { return } + self?.afterFound = true + await self?.ask("Do you want to go back, or are you done?") + } + } + + /// Back the way the person came, by voice or the Go back button. + func goBack() { + guard let guide, guide.canWalkBack else { + return speech.say("I can't retrace this walk.", interrupt: true) + } + stopListening() + isReturning = true + guide.walkBack() + } + + /// Closes the walk and brings the labels back. + private func endWalk() { + guide?.stop() + guide = nil + isReturning = false + heard = nil + locator.focus(label: nil) + } + + /// The walk ended (Stop or Done; GuidePanel has stopped it): the labels are back for the next thing. + func walkEnded() { + stopListening() + guide = nil + isReturning = false + heard = nil + locator.focus(label: nil) + guard phase == .showing else { return start() } // it was still looking for the room: carry on + followUp = Task { [weak self] in await self?.ask("What else are you looking for?") } } private func run() async { @@ -56,21 +304,41 @@ final class ARLabelsModel { return phase = .listOnly("The room map of this scan isn't on this iPhone, so here is the list.") } phase = .locating - speech.say("Point the phone around \(room) so I can recognise it.", interrupt: true) + speech.say("Scanning \(room). Hold the phone at chest height and turn slowly.", interrupt: true) + let scan = found.scanId + mapInUse = (scan, map) + locator.onMapGrown = { [saveGrownMap] data in saveGrownMap(scan, data) } locator.start(roomMap: map) - // ARKit says the room is recognised when tracking turns normal in the saved map - for _ in 0..<50 { - try await Task.sleep(for: .milliseconds(500)) - if locator.reading()?.tracking == .normal { break } + // the Mac's photo matcher helps alongside ARKit (what the phone is pointing at, a fix from a photo) + if let places { + let a = PhotoAssist(home: home, scan: scan, room: room, locator: locator, places: places, speech: speech, + clock: clock) + assist = a + Task { await a.run() } } - guard locator.reading()?.tracking == .normal else { - return phase = .listOnly("I couldn't recognise \(room) from here, so here is the list.") + // recognised when tracking turns normal: ARKit matched the saved map, or two photo fixes agreed + let started = clock.now() + var late = false + while locator.reading()?.tracking != .normal { + let waited = clock.now().timeIntervalSince(started) + if !late, waited >= Self.listAfter { + late = true // the list now, but keep looking: the labels come on if the room is recognised + phase = .listOnly("I couldn't recognise \(room) from here yet, so here is the list. Still looking.") + } + if waited >= Self.listAfter + Self.keepLooking { + assist?.cancel() + return phase = .listOnly("I couldn't recognise \(room) from here, so here is the list.") + } + try await Task.sleep(for: .milliseconds(250)) } + assist?.cancel() + haptics?.tap() let tags = labels.map { (label: $0, tag: Self.tagImage($0)) } locator.showLabels(tags, floorY: found.floorY) phase = .showing let n = labels.count == 1 ? "1 thing" : "\(labels.count) things" - speech.say("\(n) in \(room). Point at a label and double tap with two fingers to hear it.", interrupt: false) + let lead = late ? "Recognised \(room). " : "" + await ask("\(lead)\(n) in \(room). What are you looking for?", interrupt: late) } catch is CancellationError { } catch { phase = .failed(UserMessage.explain(error)) diff --git a/apps/ios/Recall/Features/AR/ARLabelsView.swift b/apps/ios/Recall/Features/AR/ARLabelsView.swift index 1f20511..57f1f85 100644 --- a/apps/ios/Recall/Features/AR/ARLabelsView.swift +++ b/apps/ios/Recall/Features/AR/ARLabelsView.swift @@ -6,7 +6,8 @@ import SceneKit import SwiftUI /// "See it in AR": the camera with a label over everything Recall found in the room, and the same things as a list -/// underneath, for VoiceOver and for anyone who can't hold the phone up. +/// underneath, for VoiceOver and for anyone who can't hold the phone up. Double tap a label (on the camera or in the +/// list) or tap Speak for directions: the walk runs here, with the camera open, and its panel takes the list's place. struct ARLabelsView: View { @State var model: ARLabelsModel let close: () -> Void @@ -15,7 +16,9 @@ struct ARLabelsView: View { VStack(spacing: 0) { ZStack(alignment: .top) { if ARLocator.isSupported { - LabelsCamera(session: model.locator.session, scene: model.locator.scene) + LabelsCamera(session: model.locator.session, scene: model.locator.scene) { point, view in + model.doubleTapped(at: point, in: view) + } .ignoresSafeArea(edges: .top) .accessibilityHidden(true) LiveLook(session: model.locator.session, look: model.look) @@ -38,53 +41,122 @@ struct ARLabelsView: View { .padding(16) } .frame(maxHeight: .infinity) - List { - Section { - ForEach(model.labels) { label in - HStack(spacing: 12) { - Circle().fill(Color(ARLocator.colour(label.mark))).frame(width: 12, height: 12) - VStack(alignment: .leading, spacing: 2) { - Text(label.title.prefix(1).uppercased() + label.title.dropFirst()).font(.headline) - Text("\(label.place) · \(label.detail)").font(.subheadline).foregroundStyle(Tokens.muted) - } + .overlay(alignment: .bottom) { + VStack(spacing: 10) { + if let heard = model.heard { + Text("“\(heard)”") + .font(.callout.italic()).foregroundStyle(.white.opacity(0.9)) + .multilineTextAlignment(.center).lineLimit(3) + .padding(.horizontal, 14).padding(.vertical, 8) + .background(.black.opacity(0.45), in: .rect(cornerRadius: 16)) + .accessibilityLabel("You said: \(heard)") + .accessibilityIdentifier("ar-heard") + } + if model.canGo { + BigButton(speakTitle, systemImage: model.hearing == .listening ? "stop.fill" : "mic.fill", + hint: "Say what you're looking for and Recall walks you there.", style: .secondary) { + Task { await model.toggleListening() } } - .accessibilityElement(children: .ignore) - .accessibilityLabel(label.spoken) - .accessibilityIdentifier("ar-label-\(label.id)") + .disabled(model.hearing == .thinking) + .accessibilityIdentifier("ar-speak") } - } header: { - Text("Found in \(model.room) · green: Recall tells you · amber: check · red: seen once") - .font(.footnote).textCase(nil) } + .padding(16) + } + if let guide = model.guide { + GuidePanel(model: guide, goBack: model.isReturning ? nil : { model.goBack() }) { model.walkEnded() } + .padding(16) + .accessibilityIdentifier("ar-guide") + } else { + labelList } - .frame(maxHeight: 260) } .background(Tokens.canvas) .onAppear { model.start() } .onDisappear { model.stop() } - .accessibilityAction(.magicTap) { model.sayInView() } + .accessibilityAction(.magicTap) { + if let guide = model.guide { guide.repeatLast() } else { model.goToLabelInView() } + } + } + + private var labelList: some View { + List { + Section { + ForEach(model.labels) { label in + HStack(spacing: 12) { + Circle().fill(Color(ARLocator.colour(label.mark))).frame(width: 12, height: 12) + VStack(alignment: .leading, spacing: 2) { + Text(label.title.prefix(1).uppercased() + label.title.dropFirst()).font(.headline) + Text("\(label.place) · \(label.detail)").font(.subheadline).foregroundStyle(Tokens.muted) + } + Spacer(minLength: 0) + } + .contentShape(Rectangle()) + .onTapGesture(count: 2) { model.go(to: label) } + .accessibilityElement(children: .ignore) + .accessibilityLabel(label.spoken) + .accessibilityHint("Double tap for directions.") + .accessibilityAddTraits(.isButton) + .accessibilityAction { model.go(to: label) } + .accessibilityIdentifier("ar-label-\(label.id)") + } + } header: { + Text("Found in \(model.room) · double tap for directions · green: Recall tells you · amber: check · red: seen once") + .font(.footnote).textCase(nil) + } + } + .frame(maxHeight: 260) + } + + private var speakTitle: String { + switch model.hearing { + case .idle: "Speak" + case .listening: "Stop" + case .thinking: "Thinking…" + } } private var status: String { - switch model.phase { + if model.hearing == .listening { return "Listening…" } + if model.isReturning { return "Going back" } + if let guide = model.guide { return "Walking to \(guide.thing.title)" } + return switch model.phase { case .loading: "Loading what Recall found…" - case .locating: "Point at the room…" + case .locating: "Scanning \(model.room)…" case .showing: "\(model.labels.count) things in \(model.room)" case .listOnly(let why), .failed(let why): why } } } -/// The locator's camera feed with the labels drawn into its scene. +/// The locator's camera feed with the labels drawn into its scene; a double tap reports where, on the view. private struct LabelsCamera: UIViewRepresentable { let session: ARSession let scene: SCNScene + let doubleTap: (CGPoint, SCNSceneRenderer) -> Void + + func makeCoordinator() -> Coordinator { Coordinator(doubleTap: doubleTap) } + func makeUIView(context: Context) -> ARSCNView { let v = ARSCNView(frame: .zero) v.session = session v.scene = scene v.automaticallyUpdatesLighting = false + let tap = UITapGestureRecognizer(target: context.coordinator, action: #selector(Coordinator.tapped)) + tap.numberOfTapsRequired = 2 + v.addGestureRecognizer(tap) return v } - func updateUIView(_ uiView: ARSCNView, context: Context) {} + + func updateUIView(_ uiView: ARSCNView, context: Context) { context.coordinator.doubleTap = doubleTap } + + @MainActor final class Coordinator: NSObject { + var doubleTap: (CGPoint, SCNSceneRenderer) -> Void + init(doubleTap: @escaping (CGPoint, SCNSceneRenderer) -> Void) { self.doubleTap = doubleTap } + + @objc func tapped(_ gesture: UITapGestureRecognizer) { + guard let view = gesture.view as? ARSCNView else { return } + doubleTap(gesture.location(in: view), view) + } + } } diff --git a/apps/ios/Recall/Features/Find/FindModel.swift b/apps/ios/Recall/Features/Find/FindModel.swift index 521a891..7d9eb82 100644 --- a/apps/ios/Recall/Features/Find/FindModel.swift +++ b/apps/ios/Recall/Features/Find/FindModel.swift @@ -24,10 +24,10 @@ final class FindModel { private let speech: any SpeechOutput private let haptics: any HapticsOutput /// Starts recognising the room while the answer is read out, so "Take me there" begins at once. - private let prepareGuide: @MainActor () -> Void + private let prepareGuide: @MainActor (Thing) -> Void init(ask: AskWhereIs, transcriber: any Transcriber, voice: any VoiceInput, things: ThingsModel, settings: SettingsModel, - speech: any SpeechOutput, haptics: any HapticsOutput, prepareGuide: @escaping @MainActor () -> Void = {}) { + speech: any SpeechOutput, haptics: any HapticsOutput, prepareGuide: @escaping @MainActor (Thing) -> Void = { _ in }) { self.ask = ask; self.transcriber = transcriber; self.voice = voice; self.things = things self.settings = settings; self.speech = speech; self.haptics = haptics; self.prepareGuide = prepareGuide } @@ -72,7 +72,7 @@ final class FindModel { about: lastAnswer?.answer.thingId) lastAnswer = result state = .answered(result) - if result.thing != nil { prepareGuide() } + if let thing = result.thing { prepareGuide(thing) } // its room's map starts recognising now speech.say(result.answer.text + (result.thing == nil ? "" : " Tap Take me there for directions."), interrupt: true) } catch { state = .failed(UserMessage.explain(error)) diff --git a/apps/ios/Recall/Features/Guide/GuideModel.swift b/apps/ios/Recall/Features/Guide/GuideModel.swift index fadd699..859d69c 100644 --- a/apps/ios/Recall/Features/Guide/GuideModel.swift +++ b/apps/ios/Recall/Features/Guide/GuideModel.swift @@ -17,11 +17,17 @@ final class GuideModel { private var task: Task? private let ended: () -> Void + /// Told when the walk reaches an end of its own (arrived, landmarks, failed), not when it is stopped. + var onFinished: ((GuideToThing.Phase) -> Void)? + var canWalkBack: Bool { guide.canWalkBack } init(guide: GuideToThing, session: ARSession, scene: SCNScene, supported: Bool, ended: @escaping () -> Void = {}) { self.guide = guide; self.session = session; self.scene = scene; self.supported = supported; thing = guide.thing self.ended = ended - guide.onChange = { [weak self] in self?.phase = $0 } + guide.onChange = { [weak self] in + self?.phase = $0 + if $0.isFinished { self?.onFinished?($0) } + } } func start() { @@ -39,4 +45,9 @@ final class GuideModel { } func repeatLast() { guide.repeatLast() } + + func walkBack() { + phase = .planning + task = Task { await guide.walkBack() } + } } diff --git a/apps/ios/Recall/Features/Guide/GuideView.swift b/apps/ios/Recall/Features/Guide/GuideView.swift index 5776cf9..573296a 100644 --- a/apps/ios/Recall/Features/Guide/GuideView.swift +++ b/apps/ios/Recall/Features/Guide/GuideView.swift @@ -21,20 +21,7 @@ struct GuideView: View { VStack(spacing: 12) { if case .guiding(_, weakTracking: true) = model.phase { CueBanner("Slow down, tracking is weak") } Spacer() - // the panel keeps its own solid fill (glass in the container was drawn over its words); only the - // buttons are glass - panel - GlassEffectContainer(spacing: 12) { - HStack(spacing: 12) { - BigButton("Repeat", systemImage: "speaker.wave.2.fill", hint: "Says the direction again.", - style: .secondary) { model.repeatLast() } - BigButton(finished ? "Done" : "Stop", systemImage: finished ? "checkmark" : "stop.fill", - hint: "Ends the directions.", style: finished ? .primary : .stop) { - model.stop() - close() - } - } - } + GuidePanel(model: model, close: close) } .padding(16) } @@ -42,6 +29,38 @@ struct GuideView: View { .onDisappear { model.stop() } .accessibilityAction(.magicTap) { model.repeatLast() } } +} + +/// The directions: the clock dial, the words and Repeat / Stop. On the guide screen, and under the camera in AR. +struct GuidePanel: View { + let model: GuideModel + /// In AR: back the way you came, once you've arrived (nil on the guide screen). + var goBack: (() -> Void)? + let close: () -> Void + + var body: some View { + VStack(spacing: 12) { + // the panel keeps its own solid fill (glass in the container was drawn over its words); only the buttons + // are glass + panel + if let goBack, case .arrived = model.phase, model.canWalkBack { + BigButton("Go back", systemImage: "arrow.uturn.backward", hint: "Walks you back the way you came.", + style: .secondary, action: goBack) + .accessibilityIdentifier("ar-go-back") + } + GlassEffectContainer(spacing: 12) { + HStack(spacing: 12) { + BigButton("Repeat", systemImage: "speaker.wave.2.fill", hint: "Says the direction again.", + style: .secondary) { model.repeatLast() } + BigButton(finished ? "Done" : "Stop", systemImage: finished ? "checkmark" : "stop.fill", + hint: "Ends the directions.", style: finished ? .primary : .stop) { + model.stop() + close() + } + } + } + } + } private var finished: Bool { switch model.phase { @@ -95,7 +114,7 @@ struct GuideView: View { private var headline: String { switch model.phase { case .locating: "Finding where you are. Turn slowly." - case .checking: "Checking with the camera. Hold the phone towards it." + case .checking: "Checking with the camera. Hold the phone out in front of you." case .planning: "Planning the way…" case .guiding(let cue, _): Directions.say(cue) case .landmarks(let text), .arrived(let text), .failed(let text): text diff --git a/apps/ios/Recall/Models/ScansModel.swift b/apps/ios/Recall/Models/ScansModel.swift index 65d5954..2eeb20d 100644 --- a/apps/ios/Recall/Models/ScansModel.swift +++ b/apps/ios/Recall/Models/ScansModel.swift @@ -106,14 +106,21 @@ final class ScansModel { return await Task.detached { archiver.archive(episode: dir) }.value } - /// The saved room map to relocalise in: the home's first processed scan made on this phone (routes are planned - /// in that scan's world). - /// The room map guidance starts from: the newest processed scan's (a recognised rescan's map holds the first - /// scan's world, extended). - func roomMap() -> Data? { - scans.filter(\.isProcessed).sorted { $0.createdAt > $1.createdAt }.lazy.compactMap { self.files.roomMap(scan: $0.id) }.first + /// The room map to guide to a thing in `room`: that room's newest processed scan with a map on this phone (the + /// Mac plans in that scan's world: a scan that wasn't recognised started the room's memory over). Without a room, + /// or no map of it, the newest processed scan's. + func guideMap(room: String?) -> (scan: String, data: Data)? { + let ready = scans.filter(\.isProcessed).sorted { $0.createdAt > $1.createdAt } + let inRoom = room.map { r in ready.filter { $0.room == r } } ?? [] + for s in inRoom + ready { + if let data = files.roomMap(scan: s.id) { return (s.id, data) } + } + return nil } + /// Keeps the room map ARKit grew on a later visit (the next visit recognises the room from there too). + func saveGrownMap(scan: String, _ data: Data) { files.saveGrownRoomMap(scan: scan, data) } + /// One scan's saved room map (the AR labels are placed in it). func roomMap(scan id: String) -> Data? { files.roomMap(scan: id) } diff --git a/apps/ios/Recall/Models/ThingsModel.swift b/apps/ios/Recall/Models/ThingsModel.swift index 737ecb8..8110e1f 100644 --- a/apps/ios/Recall/Models/ThingsModel.swift +++ b/apps/ios/Recall/Models/ThingsModel.swift @@ -31,6 +31,11 @@ final class ThingsModel { var unsureCount: Int { things.filter(\.needsReview).count } /// Find and Found both ask on appearing; they share one request, and leaving a screen doesn't cancel it. + /// Fetches the list if it doesn't have this thing yet (a label can be newer than the list). + func refreshIfNeeded(for id: String) async { + if !things.contains(where: { $0.id == id }) { await refresh() } + } + func refresh() async { guard !fixed else { return } loading = true diff --git a/apps/ios/RecallUITests/AccessibilityTests.swift b/apps/ios/RecallUITests/AccessibilityTests.swift index 4579898..444bb42 100644 --- a/apps/ios/RecallUITests/AccessibilityTests.swift +++ b/apps/ios/RecallUITests/AccessibilityTests.swift @@ -402,6 +402,29 @@ final class AccessibilityTests: XCTestCase { XCTAssertTrue(button.waitForExistence(timeout: 3)) } + /// In AR, double tapping a thing starts directions in the same view; Stop brings the labels back. + /// (The simulator has no world tracking, so the panel says so; the flow around it is the same.) + func testDoubleTapALabelGivesDirectionsWithoutLeavingAR() { + let app = XCUIApplication() + app.launchArguments += ["-UITestSampleThings"] + app.launch() + let any = app.descendants(matching: .any) + let button = any["see-in-ar-Living room"] + XCTAssertTrue(button.waitForExistence(timeout: 5)) + button.tap() + let keys = any["ar-label-s1"] + XCTAssertTrue(keys.waitForExistence(timeout: 5)) + XCTAssertTrue(any["ar-speak"].exists, "Speak is there with the labels") + keys.doubleTap() + let panelDone = app.buttons.matching(NSPredicate(format: "label == 'Done' AND identifier != 'ar-done'")).firstMatch + XCTAssertTrue(panelDone.waitForExistence(timeout: 15)) + XCTAssertFalse(keys.exists, "The directions take the list's place") + XCTAssertTrue(any["ar-done"].exists, "The AR view stays open") + panelDone.tap() + XCTAssertTrue(keys.waitForExistence(timeout: 15), "The labels come back for the next thing") + any["ar-done"].tap() + } + /// Before every scan: the plan, each step with what to do, why, when it's done and how high to hold the phone. func testEveryScanStartsWithThePlan() throws { let app = XCUIApplication() diff --git a/apps/web/app/(app)/demo/page.tsx b/apps/web/app/(app)/demo/page.tsx new file mode 100644 index 0000000..949a92f --- /dev/null +++ b/apps/web/app/(app)/demo/page.tsx @@ -0,0 +1,47 @@ +import type { Metadata } from "next"; +import Link from "next/link"; +import { ArOnPhone } from "@/components/viewer/ArOnPhone"; +import { WalkDemo, type Walks } from "@/components/viewer/WalkDemo"; +import { getDictionary } from "@/i18n"; +// made by Recall's own code from the recorded run (make demo-walks): the replayed answers and the planner's routes +import walks from "@/public/real-scan/flat/walks.json"; + +const t = getDictionary().app.demo; + +export const metadata: Metadata = { + title: `${t.metaTitle} · Recall`, + description: t.metaDescription, +}; + +/** The no-keys demo: the published real scan, the recorded answers and the walks to each thing. Signed out, no API. */ +export default function DemoPage() { + return ( +
+

+ + {t.badge} +

+

{t.title}

+

{t.lead}

+ + + + + +
+

+ {t.judges.title} +

+

{t.judges.body}

+ 
+          {t.judges.code}
+
+

+ + {t.viewer} + +

+
+
+ ); +} diff --git a/apps/web/app/(app)/layout.tsx b/apps/web/app/(app)/layout.tsx index 4927d68..ba91ea8 100644 --- a/apps/web/app/(app)/layout.tsx +++ b/apps/web/app/(app)/layout.tsx @@ -1,4 +1,5 @@ import Link from "next/link"; +import { BackToTop } from "@/components/ui/BackToTop"; import { Logo } from "@/components/ui/Logo"; import { getDictionary } from "@/i18n"; @@ -24,6 +25,7 @@ export default function AppLayout({ children }: LayoutProps<"/">) { {children} + ); } diff --git a/apps/web/app/(marketing)/layout.tsx b/apps/web/app/(marketing)/layout.tsx index 6891515..fe3221d 100644 --- a/apps/web/app/(marketing)/layout.tsx +++ b/apps/web/app/(marketing)/layout.tsx @@ -1,3 +1,4 @@ +import { BackToTop } from "@/components/ui/BackToTop"; import { CookieNotice } from "@/components/ui/CookieNotice"; import { Footer } from "@/components/ui/Footer"; import { Nav } from "@/components/ui/Nav"; @@ -9,6 +10,7 @@ export default function MarketingLayout({ children }: LayoutProps<"/">) {
- + {/* short here, so the bar stays one line; the hero carries the badges and screen readers hear them */} + {t.nav.tryDemo} - {t.nav.actualScan} + + + {t.nav.runDemo}
diff --git a/apps/web/components/viewer/ArOnPhone.tsx b/apps/web/components/viewer/ArOnPhone.tsx new file mode 100644 index 0000000..a0bd8dc --- /dev/null +++ b/apps/web/components/viewer/ArOnPhone.tsx @@ -0,0 +1,29 @@ +import { getDictionary } from "@/i18n"; + +/** What the iPhone does live, in AR and by voice, and exactly when it listens: the pages around it replay or show a scan. */ +export function ArOnPhone() { + const t = getDictionary().app.arMode; + return ( +
+

+ {t.title} +

+

{t.lead}

+
    + {t.steps.map((step, i) => ( +
  1. +

    + {i + 1} +

    +

    {step.title}

    +

    {step.body}

    +
    2. + ))} +
+
+

{t.listenTitle}

+

{t.listen}

+
+
+ ); +} diff --git a/apps/web/components/viewer/FilmFlythrough.tsx b/apps/web/components/viewer/FilmFlythrough.tsx new file mode 100644 index 0000000..8a214f1 --- /dev/null +++ b/apps/web/components/viewer/FilmFlythrough.tsx @@ -0,0 +1,231 @@ +"use client"; + +import { Canvas, useThree } from "@react-three/fiber"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { CatmullRomCurve3, Color, type Group, Vector3 } from "three"; +import { Furniture, Room, Route, Things, type FurnitureBox, type RoomShape, type Thing, type Tokens, type Walk } from "./Layers"; +import { makeUniforms, RoomCloud, type CloudData } from "./RoomCloud"; +import path from "./film-path.json"; + +/** + * The film's fly-through of the real scan (/film, not linked from the site). Time is set from outside, never from the + * clock: `window.__film.seek(t)` places the camera on its path and sets every reveal for that moment, then renders one + * frame, so a capture script can step through it frame by frame. `?t=` shows one moment. + */ + +export interface FilmWalk { + label: string; + color?: string | null; + place?: string; + target: number[]; + steps: { clock: number; metres: number; say: string }[]; + path: [number, number][]; +} + +interface Meta { + room: RoomShape; + furniture: FurnitureBox[]; + objects: Thing[]; +} + +const TOKENS: Tokens = { line: "#bdc7d8", blue: "#3b5998", blueDark: "#29487d", bg: "#f0f2f5", muted: "#4b4f56", cta: "#42b72a" }; +const BG = "#f0f2f5"; + +/** The reveal timeline, in seconds of the film's 13. */ +export const FILM = { + gather: [0, 3.2], + room: 2.4, + furniture: 4.0, + pin: 6.4, + route: [8.0, 11.4], + proof: 11.8, +} as const; + +const ease = (x: number) => (x < 0.5 ? 4 * x * x * x : 1 - Math.pow(-2 * x + 2, 3) / 2); +const clamp01 = (x: number) => Math.min(1, Math.max(0, x)); + +async function loadCloud(src: string): Promise { + const get = (f: string) => fetch(`${src}/${f}`).then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(f)))); + const [positions, conf] = await Promise.all([get("points.bin"), get("conf.bin")]); + return { positions: new Float32Array(positions), colors: null, conf: new Uint8Array(conf) }; +} + +interface Labels { + furniture: { text: string; at: Vector3 }[]; + pin: { text: string; at: Vector3 }; + steps: { text: string; clock: number; at: Vector3; from: number }[]; +} + +/** Places the camera, sets the reveals, renders a frame and moves the HTML labels, for time t. */ +function Rig({ data, meta, walk, labels, overlay }: { data: CloudData; meta: Meta; walk: FilmWalk; labels: Labels; overlay: React.RefObject }) { + const { camera, gl, scene, size } = useThree(); + const uniforms = useMemo(() => { + const u = makeUniforms(0, 2.2); + u.uLow.value = new Color(TOKENS.line); + u.uHigh.value = new Color(TOKENS.blue); + u.uBg.value = new Color(BG); + return u; + }, []); + const room = useRef(null); + const furniture = useRef(null); + const pin = useRef(null); + const walkLayer = useRef(null); + const curve = useMemo(() => new CatmullRomCurve3(path.points.map((p) => new Vector3(...p))), []); + const aim = useMemo(() => new CatmullRomCurve3(path.targets.map((p) => new Vector3(...p))), []); + const route: Walk = useMemo(() => ({ path: walk.path, progress: { value: 0 } }), [walk]); + const keys = useMemo(() => [{ label: walk.label, color: walk.color ?? null, at: walk.target, confidence: 1 } as Thing], [walk]); + + useEffect(() => { + camera.up.set(0, 0, 1); + const seek = (t: number) => { + const p = ease(clamp01(t / path.duration)); + camera.position.copy(curve.getPointAt(p)); + camera.lookAt(aim.getPointAt(p)); + camera.updateMatrixWorld(); + uniforms.uProgress.value = ease(clamp01((t - FILM.gather[0]) / (FILM.gather[1] - FILM.gather[0]))); + if (room.current) room.current.visible = t >= FILM.room; + if (furniture.current) furniture.current.visible = t >= FILM.furniture; + if (pin.current) pin.current.visible = t >= FILM.pin; + if (walkLayer.current) walkLayer.current.visible = t >= FILM.route[0]; + route.progress.value = clamp01((t - FILM.route[0]) / (FILM.route[1] - FILM.route[0])); + gl.render(scene, camera); // one frame now (the Route layer reads its progress in its own frame hook too) + // the HTML labels follow their 3D points + const el = overlay.current; + if (!el) return; + const place = (node: HTMLElement | null, at: Vector3, show: boolean) => { + if (!node) return; + const v = at.clone().project(camera); + const behind = v.z > 1; + node.style.opacity = show && !behind ? "1" : "0"; + // kept inside the frame (a label near the edge slides in rather than leaving) + const w = node.offsetWidth, h = node.offsetHeight, m = 48; + const x = Math.min(size.width - m - w / 2, Math.max(m + w / 2, ((v.x + 1) / 2) * size.width)); + const y = Math.min(size.height - m, Math.max(m + h, ((1 - v.y) / 2) * size.height)); + node.style.transform = `translate(${x}px, ${y}px) translate(-50%, -100%)`; + }; + labels.furniture.forEach((f, i) => place(el.querySelector(`[data-furniture="${i}"]`), f.at, t >= FILM.furniture + 0.25 * i && t < FILM.pin)); + place(el.querySelector("[data-pin]"), labels.pin.at, t >= FILM.pin + 0.3); + // only the step being walked, from the moment the walk reaches it until the next one starts + const walking = t >= FILM.route[0] && t < FILM.proof; + labels.steps.forEach((s, i) => { + const next = labels.steps[i + 1]?.from ?? 1.01; + place(el.querySelector(`[data-step="${i}"]`), s.at, walking && route.progress.value >= s.from && route.progress.value < next); + }); + const proof = el.querySelector("[data-proof]"); + if (proof) proof.style.opacity = t >= FILM.proof ? "1" : "0"; + }; + const film = { + duration: path.duration, + seek: (t: number) => + new Promise((resolve) => { + seek(t); + // two frames: the walk's own frame hook draws its progress, then the page is painted + requestAnimationFrame(() => { + seek(t); + requestAnimationFrame(() => resolve()); + }); + }), + }; + (window as unknown as { __film?: typeof film }).__film = film; + const start = Number(new URLSearchParams(location.search).get("t") ?? "0"); + film.seek(Number.isFinite(start) ? start : 0).then(() => document.documentElement.setAttribute("data-film-ready", "true")); + }, [camera, gl, scene, size, curve, aim, uniforms, route, labels, overlay]); + + return ( + <> + + + + + + + + + + + + + + + ); +} + +const shape = "absolute left-0 top-0 whitespace-nowrap rounded-full px-5 py-2 font-bold shadow-[0_0_0_2px_#bdc7d8,0_10px_24px_-8px_rgba(59,89,152,0.45)]"; +const pill = `${shape} bg-white text-[30px] text-ink`; + +export function FilmFlythrough({ walk, src }: { walk: FilmWalk; src: string }) { + const overlay = useRef(null); + const [data, setData] = useState(null); + const [meta, setMeta] = useState(null); + useEffect(() => { + let cancelled = false; + Promise.all([loadCloud(src), fetch(`${src}/meta.json`).then((r) => r.json())]).then(([d, m]) => { + if (!cancelled) { + setData(d); + setMeta(m); + } + }); + return () => { + cancelled = true; + }; + }, [src]); + + const labels: Labels | null = useMemo(() => { + if (!meta) return null; + // one label per kind of furniture, the largest of each, at its top + const seen = new Map(); + for (const f of meta.furniture) { + const vol = (b: FurnitureBox) => (b.max[0] - b.min[0]) * (b.max[1] - b.min[1]); + const prev = seen.get(f.label); + if (!prev || vol(f) > vol(prev)) seen.set(f.label, f); + } + const furniture = [...seen.values()] + .filter((f) => f.label !== "door") + .slice(0, 6) + .map((f) => ({ text: f.label, at: new Vector3((f.min[0] + f.max[0]) / 2, (f.min[1] + f.max[1]) / 2, f.max[2] + 0.1) })); + const [tx, ty, tz] = walk.target; + const steps = walk.steps.map((s, i) => { + const [x, y] = walk.path[i]; + const total = walk.steps.reduce((a, b) => a + b.metres, 0); + const before = walk.steps.slice(0, i).reduce((a, b) => a + b.metres, 0); + return { text: s.say.replace(/\.$/, ""), clock: s.clock, at: new Vector3(x, y, 0.15), from: before / total }; + }); + return { furniture, pin: { text: `${walk.label} · ${walk.place ?? ""}`.replace(/ · $/, ""), at: new Vector3(tx, ty, tz + 0.55) }, steps }; + }, [meta, walk]); + + return ( +
+ {data && meta && labels && ( + + + + )} +
+ {labels?.furniture.map((f, i) => ( + + {f.text} + + ))} + {labels && ( + + {labels.pin.text} + + )} + {labels?.steps.map((s, i) => ( + + + + + + {s.text} + + ))} +
+

3 cm

+

arrival check on the real keys

+

SAM 3.1 + LiDAR depth vs where Recall remembered them

+
+
+
+ ); +} diff --git a/apps/web/components/viewer/HearRoom.tsx b/apps/web/components/viewer/HearRoom.tsx new file mode 100644 index 0000000..b2dab45 --- /dev/null +++ b/apps/web/components/viewer/HearRoom.tsx @@ -0,0 +1,31 @@ +"use client"; + +import { useEffect, useState, useSyncExternalStore } from "react"; +import { format, getDictionary } from "@/i18n"; +import { canSpeak, sayLine, stopSpeaking } from "@/lib/speech"; + +const t = getDictionary().app.hearRoom; +const noSubscribe = () => () => {}; + +/** The room read aloud by the browser's own voice (Web Speech, no model): what a blind visitor would want first. */ +export function HearRoom({ area, things }: { area: string; things: string[] }) { + const speech = useSyncExternalStore(noSubscribe, canSpeak, () => false); + const [speaking, setSpeaking] = useState(false); + useEffect(() => () => stopSpeaking(), []); + if (!speech) return null; + const toggle = async () => { + if (speaking) { + stopSpeaking(); + return setSpeaking(false); + } + setSpeaking(true); + const list = things.length > 1 ? `${things.slice(0, -1).join(", ")} and ${things.at(-1)}` : things.join(""); + await sayLine(format(t.line, { area, things: list }), "assistant", { long: true }); + setSpeaking(false); + }; + return ( + + ); +} diff --git a/apps/web/components/viewer/Layers.tsx b/apps/web/components/viewer/Layers.tsx index 2503ef8..4859ee8 100644 --- a/apps/web/components/viewer/Layers.tsx +++ b/apps/web/components/viewer/Layers.tsx @@ -1,8 +1,8 @@ "use client"; -import { useFrame } from "@react-three/fiber"; -import { useMemo, useRef } from "react"; -import { BoxGeometry, BufferGeometry, CanvasTexture, Color, EdgesGeometry, Float32BufferAttribute, type Group, PlaneGeometry, SRGBColorSpace } from "three"; +import { useFrame, useThree } from "@react-three/fiber"; +import { useEffect, useMemo, useRef } from "react"; +import { BoxGeometry, BufferGeometry, CanvasTexture, Color, CurvePath, EdgesGeometry, Float32BufferAttribute, type Group, LineCurve3, PlaneGeometry, SRGBColorSpace, TubeGeometry, Vector3 } from "three"; import { BAND_EDGES } from "./bands"; export interface Surface { @@ -100,8 +100,30 @@ export interface RoomShape { export interface FurnitureBox { label: string; + /** world bounds (of the turned box, when it is turned) */ min: number[]; max: number[]; + /** newer exports: the box fitted in the room's own frame, so it sits square to the walls and inside them */ + centre?: number[]; + size?: number[]; + /** radians about z */ + yaw?: number; +} + +export interface Placement { + centre: number[]; + size: number[]; + yaw: number; +} + +/** Where a furniture box goes: turned with the walls when the export says so, else axis-aligned from min/max. */ +export function placement(f: FurnitureBox): Placement { + if (f.centre && f.size) return { centre: f.centre, size: f.size, yaw: f.yaw ?? 0 }; + return { + centre: [0, 1, 2].map((k) => (f.min[k] + f.max[k]) / 2), + size: [0, 1, 2].map((k) => f.max[k] - f.min[k]), + yaw: 0, + }; } export interface Thing { @@ -164,17 +186,17 @@ export function Furniture({ items, tokens }: { items: FurnitureBox[]; tokens: To return ( <> {items.map((f, i) => ( - + ))} ); } -function BoxOutline({ min, max, colour }: { min: number[]; max: number[]; colour: string }) { - const size = [max[0] - min[0], max[1] - min[1], max[2] - min[2]]; - const edges = useMemo(() => new EdgesGeometry(new BoxGeometry(size[0], size[1], size[2])), [size[0], size[1], size[2]]); // eslint-disable-line react-hooks/exhaustive-deps +function BoxOutline({ at, colour }: { at: Placement; colour: string }) { + const [sx, sy, sz] = at.size; + const edges = useMemo(() => new EdgesGeometry(new BoxGeometry(sx, sy, sz)), [sx, sy, sz]); return ( - + ); @@ -350,3 +372,109 @@ export function NorthArrow({ floor, northDeg, tokens, label, theta = 0 }: { floo ); } + +/** A walk to show: floor points (x, y) from start to thing, and how much of it is drawn (0 to 1; GSAP tweens it). */ +export interface Walk { + path: [number, number][]; + progress: { value: number }; +} + +const ROUTE_Z = 0.03; // just above the floor, so the points don't cover it + +/** + * The route on the floor as a green band, drawn up to `walk.progress`, with a ring where the walk starts. Like the + * arrows it is drawn over the points (no depth test), so furniture never hides it. + */ +export function Route({ walk, tokens }: { walk: Walk; tokens: Tokens }) { + const { invalidate } = useThree(); + const tube = useMemo(() => { + const curve = new CurvePath(); + walk.path.slice(1).forEach(([x, y], i) => { + const [x0, y0] = walk.path[i]; + curve.add(new LineCurve3(new Vector3(x0, y0, ROUTE_Z), new Vector3(x, y, ROUTE_Z))); + }); + // the tube's triangles run from start to end, so drawing the first n of them draws the first part of the walk + return new TubeGeometry(curve, Math.max(24, Math.round(curve.getLength() * 40)), 0.035, 8, false); + }, [walk.path]); + // the whole walk, faint, so where it goes reads before and while it is drawn + const ahead = useMemo(() => tube.clone(), [tube]); + useEffect(() => () => ahead.dispose(), [ahead]); + const drawn = useRef(-1); + const draw = () => { + const total = tube.index?.count ?? 0; + const n = Math.floor((total * Math.min(1, Math.max(0, walk.progress.value))) / 3) * 3; + if (n === drawn.current) return false; + tube.setDrawRange(0, n); + drawn.current = n; + return true; + }; + // the first frame shows the current progress (all of it when motion is reduced); later frames follow the tween + useEffect(() => { + drawn.current = -1; + draw(); + invalidate(); + return () => tube.dispose(); + // eslint-disable-next-line react-hooks/exhaustive-deps -- draw reads the same tube and progress box + }, [tube, walk.progress, invalidate]); + // the walker: where the person is along the route now, facing the way they walk + const legs = useMemo(() => { + const out: { x0: number; y0: number; dx: number; dy: number; length: number; end: number }[] = []; + let total = 0; + for (let i = 1; i < walk.path.length; i++) { + const [x0, y0] = walk.path[i - 1]; + const [x, y] = walk.path[i]; + const length = Math.hypot(x - x0, y - y0); + total += length; + out.push({ x0, y0, dx: x - x0, dy: y - y0, length, end: total }); + } + return { out, total }; + }, [walk.path]); + const walker = useRef(null); + const place = () => { + const g = walker.current; + if (!g || legs.total === 0) return; + const d = Math.min(1, Math.max(0, walk.progress.value)) * legs.total; + const leg = legs.out.find((l) => d <= l.end) ?? legs.out[legs.out.length - 1]; + const k = leg.length ? 1 - (leg.end - d) / leg.length : 1; + g.position.set(leg.x0 + leg.dx * k, leg.y0 + leg.dy * k, ROUTE_Z + 0.01); + g.rotation.set(0, 0, Math.atan2(leg.dy, leg.dx)); + }; + useFrame(() => { + if (draw()) place(); + }); + useEffect(() => { + place(); + invalidate(); + // eslint-disable-next-line react-hooks/exhaustive-deps -- place reads the same legs and progress box + }, [legs, walk.progress, invalidate]); + const [sx, sy] = walk.path[0]; + return ( + <> + + + + + + + + + + + {/* you: a dark disc with a white edge and a wedge pointing the way you walk (+x in the group, turned to the leg) */} + + + + + + + + + + + + + + + + ); +} diff --git a/apps/web/components/viewer/LocalScan.tsx b/apps/web/components/viewer/LocalScan.tsx index 567f93b..794f4c6 100644 --- a/apps/web/components/viewer/LocalScan.tsx +++ b/apps/web/components/viewer/LocalScan.tsx @@ -10,7 +10,10 @@ import { extent, floorArea, perimeter } from "./scanStats"; import { Sheets } from "./Sheets"; import { RealScanStory } from "./RealScanStory"; import type { FurnitureBox, RoomShape, Surface, Thing } from "./Layers"; +import { ArOnPhone } from "./ArOnPhone"; +import { HearRoom } from "./HearRoom"; import { ViewerStats } from "./ViewerStats"; +import type { ViewerStrings } from "./ViewerCanvas"; const s = getDictionary().app.viewer.scan; @@ -43,6 +46,36 @@ export interface ScanMeta { drawings?: { id: string; title: string; file: string; summary: string }[]; } +/** The viewer's labels, from the dictionary (shared with the demo page). */ +export function viewerStrings(): ViewerStrings { + return { + label: s.alt, + loading: s.loading, + view: s.view, + views: s.views, + viewNotes: s.viewNotes, + colour: s.colour, + modes: s.modes, + layers: s.layers, + grid: s.grid, + bands: s.bands, + surfaces: s.surfaces, + room: s.room, + things: s.things, + axes: s.axes, + axisLabels: s.axisLabels, + best: s.best, + more: s.more, + camera: s.camera, + presets: s.presets, + wallOpacity: s.wallOpacity, + showFurniture: s.showFurniture, + wireframe: s.wireframe, + flat: s.flat, + cut: s.cut, + }; +} + function bandRange(i: number) { if (i === 0) return format(s.bandBelow, { to: BAND_EDGES[0] }); if (i === BANDS.length - 1) return format(s.bandAbove, { from: BAND_EDGES[BAND_EDGES.length - 1] }); @@ -74,32 +107,7 @@ export function LocalScan({ steps }: { steps: ReactNode }) { { value: String(meta.stats.sam_calls), label: s.stats.sam }, { value: `${Math.round(meta.count / 1000)}k`, label: s.stats.points }, ]; - const viewer = { - label: s.alt, - loading: s.loading, - view: s.view, - views: s.views, - viewNotes: s.viewNotes, - colour: s.colour, - modes: s.modes, - layers: s.layers, - grid: s.grid, - bands: s.bands, - surfaces: s.surfaces, - room: s.room, - things: s.things, - axes: s.axes, - axisLabels: s.axisLabels, - best: s.best, - more: s.more, - camera: s.camera, - presets: s.presets, - wallOpacity: s.wallOpacity, - showFurniture: s.showFurniture, - wireframe: s.wireframe, - flat: s.flat, - cut: s.cut, - }; + const viewer = viewerStrings(); return ( <> {PUBLIC_SCANS.has(homeId) && ( @@ -135,8 +143,19 @@ export function LocalScan({ steps }: { steps: ReactNode }) {
+ o.label))].sort()} />
+ {PUBLIC_SCANS.has(homeId) && ( + <> +
+ +

{getDictionary().app.fieldStats.note}

+
+ + + )} + {PUBLIC_SCANS.has(homeId) && } {meta.drawings && meta.drawings.length > 0 && ( diff --git a/apps/web/components/viewer/PhonePanel.tsx b/apps/web/components/viewer/PhonePanel.tsx new file mode 100644 index 0000000..13afd8f --- /dev/null +++ b/apps/web/components/viewer/PhonePanel.tsx @@ -0,0 +1,247 @@ +"use client"; + +import { useEffect, useRef, useState, type ReactNode } from "react"; +import { format, getDictionary } from "@/i18n"; +import { gsap } from "@/lib/gsap"; +import type { Walk } from "./Layers"; + +const t = getDictionary().app.demo.phone; + +/** One cut of a real iPhone screen recording (make demo-clips → public/local-video/clips.json; gitignored). */ +export interface Clip { + id: string; + kind: "find" | "scan" | "image"; + /** the thing a find clip walks to, by label */ + thing?: string; + title: string; + /** when it was recorded, YYYY-MM-DD */ + recorded: string; + /** the seconds of the clip in which the person walks: the route follows them */ + walk?: [number, number]; + captions: [number, string][]; + file: string; + seconds: number; +} + +export interface Clips { + /** "local": cut on this computer, never on the hosted site; "published": reviewed and published by the owner */ + source: "local" | "published"; + base: string; + clips: Clip[]; +} + +/** This computer's clips if there are any (make demo-clips), else published ones; none on the hosted site today. */ +export function useClips(): Clips | null { + const [clips, setClips] = useState(null); + useEffect(() => { + let cancelled = false; + const get = (base: string) => + fetch(`${base}/clips.json`, { cache: "no-store" }) + .then((r) => (r.ok ? r.json() : null)) + .then((j) => (j && Array.isArray(j.clips) && j.clips.length ? ({ ...j, base } as Clips) : null)) + .catch(() => null); + get("/local-video") + .then((local) => local ?? get("/demo-video")) + .then((c) => !cancelled && setClips(c)); + return () => { + cancelled = true; + }; + }, []); + return clips; +} + +/** A phone outline around a screen; decorative, so its text is repeated in the page for screen readers. */ +function Frame({ children, label }: { children: ReactNode; label: string }) { + return ( +
+
+
{children}
+
+
{label}
+
+ ); +} + +/** + * The AR camera view, drawn: no photo of the home goes on the public page, so the room is a floor grid. The chevrons + * lean the way the clock-face step says; the thing's tag grows as the walk nears it, as the labels do in the camera. + */ +function ArScene({ thing, hour, progress, before, there }: { thing: string; hour: number; progress: number; before: boolean; there: boolean }) { + const lean = ((hour % 12 > 6 ? hour % 12 - 12 : hour % 12) * 18).toFixed(0); + const name = thing.toUpperCase(); + const scale = there ? 1.2 : 0.65 + progress * 0.4; + const width = name.length * 8.6 + 26; + return ( + + + + + + + + + + + + + + + {[-200, -110, -40, 40, 110, 200, 320, -320].map((x) => ( + + ))} + {[166, 190, 226, 276].map((y) => ( + + ))} + + {!before && !there && ( + + {[ + { y: 286, s: 1, o: 1 }, + { y: 244, s: 0.74, o: 0.8 }, + { y: 212, s: 0.54, o: 0.6 }, + { y: 190, s: 0.38, o: 0.4 }, + ].map((c) => ( + + ))} + + )} + + + {there && } + + + + + {name} + + + + + ); +} + +function ClockFace({ hour, done }: { hour: number; done?: boolean }) { + return ( + + + {done ? ( + + ) : ( + <> + + + + )} + + ); +} + +/** + * The guidance screen as the app lays it out (the arrow, then a card with the clock face, the thing and the step), + * drawn in HTML for the walks no one recorded. `step` is the step being spoken: -1 before the walk, steps.length + * once there. + */ +export function SimulatedPhone({ thing, heard, steps, arrive, step }: { thing: string; heard?: string; steps: { clock: number; say: string }[]; arrive: string; step: number }) { + const before = step < 0; + const there = step >= steps.length; + const now = steps[Math.min(Math.max(step, 0), steps.length - 1)]; + const progress = there ? 1 : Math.max(step + 1, 0) / Math.max(steps.length, 1); + return ( + +
+
+ +
+ {format(t.walkingTo, { thing })} + {t.done} +
+
+ {t.camera} + {t.edges} +
+ {before &&

{t.startHere}

} + {heard && ( +

“{heard}”

+ )} +
+
+
+ +
+

{thing.charAt(0).toUpperCase() + thing.slice(1)}

+

{before ? t.takeMeThere : there ? arrive : now.say}

+
+
+
+
+ {t.repeat} + {there ? t.done : t.stop} +
+
+ + ); +} + +function captionAt(clip: Clip, time: number) { + let line = clip.captions[0]?.[1] ?? ""; + for (const [at, text] of clip.captions) if (at <= time) line = text; + return line; +} + +/** + * A real screen recording. For a find clip the route on the 3D views follows the video: while it plays, the walker + * is where the person is in the clip's walking seconds; paused or before playing, the whole route shows. + */ +export function RecordedPhone({ clip, base, walk, still, autoPlay }: { clip: Clip; base: string; walk?: Walk | null; still: boolean; autoPlay: boolean }) { + const video = useRef(null); + const [caption, setCaption] = useState(clip.captions[0]?.[1] ?? ""); + + useEffect(() => { + const v = video.current; + if (!v) return; + let frame = 0; + const follow = () => { + if (walk && clip.walk && !still) { + const [a, b] = clip.walk; + gsap.set(walk.progress, { value: Math.min(1, Math.max(0, (v.currentTime - a) / (b - a))) }); + } + if (!v.paused) frame = requestAnimationFrame(follow); + }; + const play = () => { + cancelAnimationFrame(frame); + follow(); + }; + const rest = () => { + cancelAnimationFrame(frame); + if (walk && v.ended) gsap.set(walk.progress, { value: 1 }); + }; + const time = () => setCaption(captionAt(clip, v.currentTime)); + v.addEventListener("play", play); + v.addEventListener("pause", rest); + v.addEventListener("ended", rest); + v.addEventListener("seeked", follow); + v.addEventListener("timeupdate", time); + if (autoPlay && !still) v.play().catch(() => {}); + return () => { + cancelAnimationFrame(frame); + v.removeEventListener("play", play); + v.removeEventListener("pause", rest); + v.removeEventListener("ended", rest); + v.removeEventListener("seeked", follow); + v.removeEventListener("timeupdate", time); + if (walk) gsap.set(walk.progress, { value: 1 }); + }; + }, [clip, walk, still, autoPlay]); + + return ( +
+ + {/* muted until the viewer turns the sound on with the player's own control */} +
+ ); +} diff --git a/apps/web/components/viewer/RoomViews.tsx b/apps/web/components/viewer/RoomViews.tsx index 6dbc3ab..eb7fcfa 100644 --- a/apps/web/components/viewer/RoomViews.tsx +++ b/apps/web/components/viewer/RoomViews.tsx @@ -2,7 +2,7 @@ import { useMemo } from "react"; import { BoxGeometry, BufferAttribute, BufferGeometry, Color, EdgesGeometry, PlaneGeometry, Shape, ShapeGeometry, SRGBColorSpace, TextureLoader, type Texture } from "three"; -import { NorthArrow, type FurnitureBox, type RoomShape, type Thing, type Tokens } from "./Layers"; +import { NorthArrow, placement, type FurnitureBox, type Placement, type RoomShape, type Thing, type Tokens } from "./Layers"; /** One colour per kind of furniture, like a semantic map (a legend in the controls names them). */ export const KIND_COLOURS: Record = { @@ -73,29 +73,18 @@ export function RoomModel({ {showFurniture && furniture.map((f, i) => ( ))} {things.map((t, i) => { + // a thing's size is measured along the world axes (min/max), so its box stays square to them const s = (t.size ?? [0.12, 0.12, 0.12]).map((v) => Math.max(v, 0.05)); - const min = [ - t.at[0] - s[0] / 2, - t.at[1] - s[1] / 2, - t.at[2] - s[2] / 2, - ]; - const max = [ - t.at[0] + s[0] / 2, - t.at[1] + s[1] / 2, - t.at[2] + s[2] / 2, - ]; return ( new BoxGeometry(sx, sy, sz), [sx, sy, sz]); const edges = useMemo(() => new EdgesGeometry(box), [box]); return ( - + diff --git a/apps/web/components/viewer/ViewerCanvas.tsx b/apps/web/components/viewer/ViewerCanvas.tsx index 3ce2f05..48afef5 100644 --- a/apps/web/components/viewer/ViewerCanvas.tsx +++ b/apps/web/components/viewer/ViewerCanvas.tsx @@ -8,7 +8,7 @@ import { Color, Vector3 } from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js"; import { gsap, MOTION_OK } from "@/lib/gsap"; import { useMotionPrefs } from "@/lib/motion-prefs"; -import { Axes, FloorGrid, Furniture, HeightBands, Room, Surfaces, Things, type Bounds, type FurnitureBox, type RoomShape, type Surface, type Thing, type Tokens } from "./Layers"; +import { Axes, FloorGrid, Furniture, HeightBands, Room, Route, Surfaces, Things, type Bounds, type FurnitureBox, type RoomShape, type Surface, type Thing, type Tokens, type Walk } from "./Layers"; import { makeUniforms, MODES, RoomCloud, type CloudData, type CloudUniforms, type ColourMode } from "./RoomCloud"; import { ArchitectRoom, CUT, kindColour, loadArchitect, loadMesh, loadPhoto, PhotoRoom, RoomModel, SurfaceMesh, type ArchitectData, type MeshData, type PhotoData } from "./RoomViews"; @@ -25,6 +25,9 @@ const PRESETS: Record = { front: [0, -9.5, 2.4], side: [9.5, 0, 2.4], }; +/** A preset camera position moved `k` times as far from the room's centre. */ +const away = ([x, y, z]: [number, number, number], k: number): [number, number, number] => [x * k, y * k, z * k]; + /** The views that read best: every measured point, and the regenerated room. */ export const BEST_VIEWS: View[] = ["dots", "architect"]; @@ -81,6 +84,13 @@ export interface ViewerProps { hasPhoto?: boolean; /** the export has the regenerated room (architect.json) */ hasArchitect?: boolean; + /** a walk to draw on the floor (the demo page), or none */ + walk?: Walk | null; + /** how the viewer opens (default: every point, from the side, turning slowly); `distance` scales the camera's + * distance so a whole room fits a smaller frame */ + start?: { view: View; preset: Preset; distance?: number; spin?: boolean }; + /** false hides the view switch and the other controls (two viewers side by side on the demo page) */ + controls?: boolean; t: ViewerStrings; } @@ -93,10 +103,10 @@ interface Layers { things: boolean; } -type Model = Pick; +type Model = Pick; /** Drag to orbit (sideways on phones, so vertical swipes still scroll the page); a slow turn until the first drag. */ -function Controls({ spin, preset }: { spin: boolean; preset: Preset }) { +function Controls({ spin, preset, distance }: { spin: boolean; preset: Preset; distance: number }) { const { camera, gl, invalidate } = useThree(); const controls = useMemo(() => { camera.up.set(0, 0, 1); @@ -135,11 +145,11 @@ function Controls({ spin, preset }: { spin: boolean; preset: Preset }) { first.current = false; return; } - camera.position.set(...PRESETS[preset]); + camera.position.set(...away(PRESETS[preset], distance)); controls.autoRotate = false; controls.update(); invalidate(); - }, [preset, camera, controls, invalidate]); + }, [preset, distance, camera, controls, invalidate]); useEffect(() => () => controls.dispose(), [controls]); useFrame(() => controls.update()); @@ -169,11 +179,12 @@ function useTokens(): Tokens { return tokens; } -function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bounds, surfaces, model, tokens, options, axisLabels }: { view: View; mesh: MeshData | null; photo: PhotoData | null; architect: ArchitectData | null; data: CloudData; uniforms: CloudUniforms; spin: boolean; layers: Layers; bounds: Bounds; surfaces: Surface[]; model: Model; tokens: Tokens; options: Options; axisLabels: ViewerStrings["axisLabels"] }) { +function Scene({ view, mesh, photo, architect, data, uniforms, spin, distance, layers, bounds, surfaces, model, tokens, options, axisLabels }: { view: View; distance: number; mesh: MeshData | null; photo: PhotoData | null; architect: ArchitectData | null; data: CloudData; uniforms: CloudUniforms; spin: boolean; layers: Layers; bounds: Bounds; surfaces: Surface[]; model: Model; tokens: Tokens; options: Options; axisLabels: ViewerStrings["axisLabels"] }) { const { invalidate } = useThree(); - useEffect(() => invalidate(), [model.selected, layers, view, mesh, photo, architect, options, invalidate]); + useEffect(() => invalidate(), [model.selected, model.walk, layers, view, mesh, photo, architect, options, invalidate]); const axes = options.axes && ; - const controls = ; + const controls = ; + const walk = model.walk && ; useEffect(() => { uniforms.uLow.value = new Color(tokens.line); uniforms.uHigh.value = new Color(tokens.blue); @@ -189,6 +200,7 @@ function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bou {axes} + {walk} {controls} ); @@ -199,6 +211,7 @@ function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bou {axes} + {walk} {controls} ); @@ -207,8 +220,9 @@ function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bou return ( <> {/* an architect's model shows north and scale, not x/y/z axes */} - + + {walk} {controls} ); @@ -219,6 +233,7 @@ function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bou {axes} + {walk} {controls} ); @@ -233,6 +248,7 @@ function Scene({ view, mesh, photo, architect, data, uniforms, spin, layers, bou {layers.room && } {layers.things && } {axes} + {walk} {controls} ); @@ -250,14 +266,15 @@ async function load(src: string): Promise { return { positions: new Float32Array(positions!), colors: colors ? new Uint8Array(colors) : null, conf: new Uint8Array(conf!) }; } -export default function ViewerCanvas({ src, bounds, surfaces, room, furniture, things, selected, hasMesh = false, hasPhoto = false, hasArchitect = false, t }: ViewerProps) { +export default function ViewerCanvas({ src, bounds, surfaces, room, furniture, things, selected, hasMesh = false, hasPhoto = false, hasArchitect = false, walk = null, start, controls = true, t }: ViewerProps) { const root = useRef(null); const id = useId(); const { reduced } = useMotionPrefs(); const [data, setData] = useState(null); const [layers, setLayers] = useState({ mode: "height", grid: true, bands: false, surfaces: false, room: true, things: true }); - const [view, setView] = useState("dots"); - const [options, setOptions] = useState({ preset: "iso", axes: true, wallOpacity: 0.45, showFurniture: true, wireframe: false, flat: false, cut: CUT }); + const [view, setView] = useState(start?.view ?? "dots"); + const [options, setOptions] = useState({ preset: start?.preset ?? "iso", axes: true, wallOpacity: 0.45, showFurniture: true, wireframe: false, flat: false, cut: start?.view === "architect" ? 1.5 : CUT }); + const distance = start?.distance ?? 1; const [moreOpen, setMoreOpen] = useState(false); const set = (key: K, value: Options[K]) => setOptions((o) => ({ ...o, [key]: value })); // each view opens with the settings it reads best in (each can be changed after): @@ -368,15 +385,16 @@ export default function ViewerCanvas({ src, bounds, surfaces, room, furniture, t - + ) : (

{t.loading}

)} + {controls && (
{t.view} @@ -507,6 +525,7 @@ export default function ViewerCanvas({ src, bounds, surfaces, room, furniture, t
+ )} ); } diff --git a/apps/web/components/viewer/ViewerStats.tsx b/apps/web/components/viewer/ViewerStats.tsx index 3e93ba3..f11633e 100644 --- a/apps/web/components/viewer/ViewerStats.tsx +++ b/apps/web/components/viewer/ViewerStats.tsx @@ -3,9 +3,9 @@ export function ViewerStats({ label, items }: { label: string; items: { value: s return (

{label}

-
+
{items.map((item) => ( -
+
{item.label}
{item.value}
diff --git a/apps/web/components/viewer/WalkDemo.tsx b/apps/web/components/viewer/WalkDemo.tsx new file mode 100644 index 0000000..707153b --- /dev/null +++ b/apps/web/components/viewer/WalkDemo.tsx @@ -0,0 +1,440 @@ +"use client"; + +import { useEffect, useId, useMemo, useRef, useState, useSyncExternalStore } from "react"; +import { format, getDictionary } from "@/i18n"; +import { gsap, MOTION_OK, SplitText, useGSAP } from "@/lib/gsap"; +import { useMotionPrefs } from "@/lib/motion-prefs"; +import { canSpeak, sayLine, stopSpeaking } from "@/lib/speech"; +import type { Walk } from "./Layers"; +import { type ScanMeta, viewerStrings } from "./LocalScan"; +import { RecordedPhone, SimulatedPhone, useClips } from "./PhonePanel"; +import { RoomViewer } from "./RoomViewer"; + +/** A walk to one thing: the planner's clock-face steps from where the scan began (walks.json, `make demo-walks`). */ +export interface WalkOption { + label: string; + /** the colour and the place, from the Guide's own tool output: "grey", "near the table · waist height" */ + color?: string | null; + place?: string; + /** the thing, in the viewer's frame */ + target: number[]; + /** "landmarks" when no clear path exists: one step that names the place instead */ + mode: "route" | "landmarks"; + steps: { clock: number; metres: number; say: string }[]; + arrive: string; + /** floor points from where the scan began to the thing (empty without a clear path) */ + path: [number, number][]; +} + +/** One recorded question: the Guide's answer, then one walk, or the choices when it asks "which one?". */ +export interface WalkItem extends Partial { + question: string; + answer: string; + grounded: boolean; + model: string; + choices?: WalkOption[]; +} + +export interface Walks { + label: string; + recorded_at: string; + captured_at: string; + room: string; + walks: WalkItem[]; +} + +const t = getDictionary().app.demo; + +/** The page opens on the watch: the walk with the real recordings behind it. */ +const firstQuestion = (walks: Walks) => Math.max(walks.walks.findIndex((w) => /watch/i.test(w.question)), 0); + +function optionOf(item: WalkItem, pick: number): WalkOption | null { + if (item.choices) return item.choices[Math.min(pick, item.choices.length - 1)]; + return item.steps && item.label ? (item as WalkOption) : null; +} + +/** The thing's arrow in the viewer: the one with this label nearest the walk's target. */ +function thingIndex(meta: ScanMeta | null, option: WalkOption | null): number | null { + if (!meta || !option) return null; + const [x, y] = option.target; + let best: number | null = null; + let bestD = Infinity; + for (const [i, o] of meta.objects.entries()) { + const d = Math.hypot(o.at[0] - x, o.at[1] - y); + if (o.label === option.label && d < bestD) { + best = i; + bestD = d; + } + } + return best; +} + +const metresOf = (o: WalkOption) => o.steps.reduce((sum, s) => sum + s.metres, 0); + +/** A small clock face with its hand on the step's hour (12 is straight ahead). */ +function Clock({ hour }: { hour: number }) { + return ( + + + + + + + ); +} + +const noSubscribe = () => () => {}; +const watchMotion = (change: () => void) => { + const mq = window.matchMedia(MOTION_OK); + mq.addEventListener("change", change); + return () => mq.removeEventListener("change", change); +}; + +/** + * Ask, then walk: pick a recorded question, read (or hear) Recall's answer, and watch the walk. GSAP writes the answer + * in word by word, draws the route on the floor of the 3D room (two views side by side on a wide screen), moves "you" + * along it and brings in each spoken step as the route reaches it, while the phone beside it shows that step. When + * this computer has a real screen recording of that walk, the phone plays it and the route follows the video instead. + * Everything is in the page's HTML first, so it reads the same without JavaScript or with motion reduced. + */ +export function WalkDemo({ walks, src }: { walks: Walks; src: string }) { + const id = useId(); + const root = useRef(null); + const { reduced } = useMotionPrefs(); + const motionOk = useSyncExternalStore(watchMotion, () => window.matchMedia(MOTION_OK).matches, () => true); + const still = reduced || !motionOk; + const [chosen, setChosen] = useState(() => firstQuestion(walks)); + const [pick, setPick] = useState(0); + const [replays, setReplays] = useState(0); + const [meta, setMeta] = useState(null); + const [speaking, setSpeaking] = useState(false); + const [phone, setPhone] = useState<"recording" | "simulated">("recording"); + // the step the phone is speaking: -1 before the walk, steps.length once there (and from the start without motion) + const [step, setStep] = useState(() => optionOf(walks.walks[firstQuestion(walks)], 0)?.steps.length ?? 0); + const [asked, setAsked] = useState(false); + // speech synthesis exists only in the browser: the server (and the first render) draws no Hear button + const speech = useSyncExternalStore(noSubscribe, canSpeak, () => false); + const clips = useClips(); + const item = walks.walks[chosen]; + const option = optionOf(item, pick); + const clip = clips?.clips.find((c) => c.kind === "find" && option && c.thing === option.label) ?? null; + const scanClip = clips?.clips.find((c) => c.kind === "scan") ?? null; + const arPhoto = clips?.clips.find((c) => c.kind === "image") ?? null; + // the questions that walk to a thing this computer has a real recording of (by label, as the phone panel matches) + const recorded = (w: WalkItem) => Boolean(clips?.clips.some((c) => c.kind === "find" && (w.choices ?? [w]).some((o) => o.label === c.thing))); + const firstRecorded = walks.walks.findIndex(recorded); + const playing = Boolean(clip) && phone === "recording"; + + useEffect(() => { + let cancelled = false; + fetch(`${src}/meta.json`) + .then((r) => (r.ok ? r.json() : null)) + .catch(() => null) + .then((m) => !cancelled && setMeta(m)); + return () => { + cancelled = true; + }; + }, [src]); + + // a new box per walk (and per replay): the route redraws from its start; full when motion is reduced + const walk: Walk | null = useMemo( + () => (option && option.path.length > 1 ? { path: option.path, progress: { value: 1 } } : null), + // eslint-disable-next-line react-hooks/exhaustive-deps -- a replay asks for a fresh box on purpose + [option, replays], + ); + const selected = thingIndex(meta, option); + + useGSAP( + () => { + const el = root.current; + if (still) { + if (el) el.dataset.played = "true"; + return; + } + if (el) el.dataset.played = "false"; + const steps = gsap.utils.toArray("[data-walk-step]"); + const words = SplitText.create("[data-answer-text]", { type: "words", aria: "none" }); + const tl = gsap.timeline({ + defaults: { ease: "power2.out" }, + onComplete: () => { + if (el) el.dataset.played = "true"; + }, + }); + tl.from(words.words, { opacity: 0, y: 6, duration: 0.35, stagger: 0.035 }); + if (walk && option && !playing) { + // the route draws at walking pace on screen; each step arrives when the route reaches its start + const metres = option.steps.map((s) => Math.max(s.metres, 0.2)); + const total = metres.reduce((a, b) => a + b, 0); + const span = Math.min(5, 1.2 + total * 0.5); + const at = tl.duration(); + tl.call(() => setStep(-1), [], 0); + tl.fromTo(walk.progress, { value: 0 }, { value: 1, duration: span, ease: "none", immediateRender: true }, at); + let done = 0; + steps.forEach((s, i) => { + const start = at + (span * done) / total; + tl.from(s, { opacity: 0, x: -14, duration: 0.4 }, start); + tl.from(s.querySelector("[data-hand]"), { rotation: 0, svgOrigin: "20 20", duration: 0.5, ease: "back.out(2)" }, start); + tl.call(() => setStep(i), [], start); + done += metres[i] ?? 0; + }); + tl.call(() => setStep(option.steps.length), [], at + span); + tl.from("[data-walk-arrive]", { opacity: 0, y: 8, duration: 0.4 }, at + span); + } else { + tl.from("[data-walk-note], [data-walk-step]", { opacity: 0, y: 8, duration: 0.4, stagger: 0.05 }); + } + return () => { + words.revert(); + if (walk) gsap.set(walk.progress, { value: 1 }); + }; + }, + { scope: root, dependencies: [walk, still, playing], revertOnUpdate: true }, + ); + + useEffect(() => () => stopSpeaking(), []); + const quiet = () => { + stopSpeaking(); + setSpeaking(false); + }; + const choose = (i: number) => { + quiet(); + setChosen(i); + setPick(0); + setAsked(true); + setStep(optionOf(walks.walks[i], 0)?.steps.length ?? 0); + }; + const choosePick = (i: number) => { + quiet(); + setPick(i); + setStep(item.choices?.[i]?.steps.length ?? 0); + }; + const hear = async () => { + if (speaking) return quiet(); + setSpeaking(true); + const lines = [item.answer, ...(option ? [t.start, ...option.steps.map((s) => s.say), option.arrive] : [])]; + await sayLine(lines.join(" "), "assistant", { long: true }); + setSpeaking(false); + }; + + const viewer = (props: { start: { view: "dots" | "architect"; preset: "iso" | "top"; distance: number; spin: boolean }; controls: boolean; label: string }) => + meta ? ( + + ) : ( +

{viewerStrings().loading}

+ ); + const architect = { view: "architect", preset: "top", distance: 1.45, spin: false } as const; + + return ( +
+
+ {t.ask} +

+ {t.ask} +

+
+ {walks.walks.map((w, i) => ( + + ))} +
+
+ + {/* one view with its switch on smaller screens; every point and the architect's room side by side on wide ones */} +
+ {viewer({ start: architect, controls: true, label: t.architect })} +

{t.legend}

+
+
+ {[ + { start: { view: "dots", preset: "iso", distance: 1.05, spin: false } as const, label: t.dots }, + { start: architect, label: t.architect }, + ].map((v) => ( +
+ {viewer({ start: v.start, controls: false, label: v.label })} +
+ {v.label}. {t.legend} +
+
+ ))} +
+ +
+
+
+

+ {t.youAsked} “{item.question}” +

+

+ {t.answer} +

+ {/* screen readers get the sentence whole; the copy on screen is split into words for the animation */} +

{item.answer}

+

+ {item.answer} +

+ {item.choices && ( +
+ {t.choices} +
+ {item.choices.map((c, i) => ( + + ))} +
+
+ )} +
+ {speech && ( + + )} + {walk && !playing && ( + + )} +
+
+ +
+

+ {t.walkTitle} +

+ {option ? ( +
    +
  1. {t.start}
    2. + {option.mode === "landmarks" && ( +
  2. + {t.noPath} +
    3. + )} + {option.steps.map((s, i) => ( +
  3. + + + + {s.say} +
    4. + ))} + {option.mode === "route" && ( +
  4. + {t.arrive}: + {option.arrive} +
    5. + )} +
+ ) : ( +

+ {t.notSeen} +

+ )} +
+ +

+ {format(t.recorded, { date: walks.recorded_at.slice(0, 10) })} +

+
+ + +
+ + {arPhoto && clips && ( +
+
+

+ {t.arPhoto.title} +

+

{format(t.arPhoto.body, { date: arPhoto.recorded })}

+

{t.arPhoto.key}

+ {clips.source === "local" &&

{t.phone.localOnly}

} +
+ {/* eslint-disable-next-line @next/next/no-img-element -- a frame on this computer only, not in the build */} + {arPhoto.title} +
+ )} + + {scanClip && clips && ( +
+
+

+ {t.scanClip.title} +

+

{format(t.scanClip.body, { date: scanClip.recorded })}

+
    + {scanClip.captions.map(([at, line]) => ( +
  1. + {`${Math.floor(at / 60)}:${String(at % 60).padStart(2, "0")}`} {line} +
    2. + ))} +
+ {clips.source === "local" &&

{t.phone.localOnly}

} +
+ +
+ )} +
+ ); +} diff --git a/apps/web/components/viewer/film-path.json b/apps/web/components/viewer/film-path.json new file mode 100644 index 0000000..d5af1b0 --- /dev/null +++ b/apps/web/components/viewer/film-path.json @@ -0,0 +1,58 @@ +{ + "note": "The fly-through's camera for the film (/film): a slow high orbit of the real scan, then a dolly down to the keys. Positions and targets in the viewer's frame (metres, z up). Tweak freely; `duration` is seconds.", + "duration": 13, + "points": [ + [ + 7.0, + -9.0, + 8.6 + ], + [ + 9.2, + -2.4, + 7.2 + ], + [ + 6.2, + 4.6, + 6.6 + ], + [ + 3.9, + -0.9, + 5.4 + ], + [ + 2.2, + -0.9, + 4.5 + ] + ], + "targets": [ + [ + 0, + 0, + 0.6 + ], + [ + 0, + 0, + 0.7 + ], + [ + 0.3, + 0.6, + 0.7 + ], + [ + 0.9, + 1.3, + 0.7 + ], + [ + 1.28, + 1.6, + 0.66 + ] + ] +} \ No newline at end of file diff --git a/apps/web/e2e/demo.spec.ts b/apps/web/e2e/demo.spec.ts new file mode 100644 index 0000000..381234b --- /dev/null +++ b/apps/web/e2e/demo.spec.ts @@ -0,0 +1,206 @@ +import fs from "node:fs"; +import path from "node:path"; +import AxeBuilder from "@axe-core/playwright"; +import { expect, test, type Page } from "@playwright/test"; +import { en } from "../i18n/dictionaries/en"; +import { open, prepare } from "./helpers"; + +/** + * /demo: the no-keys demo. Recorded answers from a real run (samples/replay/flat) and the planner's walks, read from + * public/real-scan/flat/walks.json (`make demo-walks`). Nothing here calls a model or the API. The real screen + * recordings live only on the author's computer (public/local-video, gitignored), so these tests never use them: a + * made-up test pattern (fixtures/test-clip.mp4) stands in when the recording path is tested. + */ + +const PAGE = "/demo"; +const t = en.app.demo; +type Option = { label: string; steps: { say: string }[]; arrive: string; mode: string; path: number[][] }; +type Item = Partial