diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd970bf..d180c45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 + # Open3D's Linux wheel loads the system's EGL and OpenGL libraries even when nothing is drawn + - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libegl1 libgl1 libgomp1 - run: uvx ruff check . && uvx ruff format --check . - run: >- uv run --no-project --python 3.12 --with numpy --with scipy --with shapely --with pytest --with httpx @@ -56,6 +58,6 @@ jobs: node-version: 22 cache: pnpm - run: pnpm install --frozen-lockfile - - run: pnpm --filter @recall/web exec tsc --noEmit + - run: pnpm --filter @recall/web typecheck # next typegen first: PageProps and LayoutProps are generated - run: pnpm --filter @recall/web lint - run: pnpm --filter @recall/web build diff --git a/CLAUDE.md b/CLAUDE.md index 9cc63eb..75f96ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -154,7 +154,7 @@ Grounding rules (system prompt **and** enforced in code): - **Guide** (above) is the lead agent and the only one on the live question path. - **Verifier** confirms arrival from one camera frame: SAM 3.1 + geometry decide; **Muse Spark 1.1** (hosted, multimodal) is asked only as a tie-break and may only downgrade to `uncertain`, never confirm. It is an API use case, so its memory updates (self-healing) keep the one-writer rule. Reached through the `ArrivalVerifier` - port: **in-process by default and in the live demo**; over **A2A** optional (`VERIFIER_TRANSPORT`). + port: **in-process** (`VERIFIER_TRANSPORT=inprocess`, the only transport built; A2A for the Verifier is optional and not built). - **Home Memory agent** exposes read-only skills to external agents over **A2A**, off by default, never the user's live position. Inbound input is structured only (one object name, validated against `resources/agents/find-object.input.schema.json`; free text never reaches a model or a tool), then the `PromptScreener` port (**Muse Spark Open** or Glimmer, strict JSON, fails closed). @@ -172,7 +172,9 @@ models are banned): into 3D per frame, then **`cluster_dbscan` per label** merges the sightings, so one mug seen in 30 frames becomes one object with one centroid and one confidence. 4. **2D geometry (Shapely):** floor outline polygon, wall segments and furniture footprints, used for `room.json` and the occupancy grid. -5. **Later scans:** global registration (FPFH + RANSAC) then ICP into the first scan's frame, with Open3D, never hand-rolled. +5. **Later scans:** the phone relocalises against the room's saved ARWorldMap (ARKit, not a model), so a rescan's poses are + already in the first scan's frame (`aligned_to`); if it doesn't relocalise, memory starts fresh and says so. Point-cloud + registration (Open3D FPFH + ICP) is **not built** (ADR-037); never hand-roll it. Umeyama stays for ARKit ↔ MapAnything poses. Result: `room.json` = floor polygon, wall segments, furniture boxes, validated against `schemas/RoomModel.yaml`; written by @@ -207,7 +209,7 @@ feeds it the ARKit intrinsics, poses and LiDAR depth. Its output is then already - If any view has a pose, the first view must too. - **Umeyama** similarity transform (ARKit vs MapAnything camera centres) is the **check and fallback**: it measures drift, and aligns an images-only run if ARKit inputs are missing. -- Later scans align to the first with Open3D (global registration, then ICP). +- Later scans share the first scan's frame through ARWorldMap relocalisation (FPFH + ICP not built, ADR-037). ## 5. Scan episodes and upload @@ -286,9 +288,9 @@ recall/ │ │ │ │ └── tools.py tool registry, validated against resources/tools.json │ │ │ ├── adapters/ │ │ │ │ ├── inbound/http/ FastAPI routers: mapping only, no logic -│ │ │ │ ├── inbound/a2a/ verifier_server.py, home_memory_server.py +│ │ │ │ ├── inbound/a2a/ home_memory_server.py │ │ │ │ └── outbound/ -│ │ │ │ ├── agents/ verifier_a2a_client.py, verifier_inprocess.py +│ │ │ │ ├── agents/ verifier_inprocess.py (ArrivalVerifier) │ │ │ │ ├── vision/ sam31_meta_api.py, spark_vision_meta_api.py (HTTP only) │ │ │ │ ├── safety/ meta_model_screen.py (Muse Spark Open or Glimmer, strict JSON verdict) │ │ │ │ ├── llm/ openai_compatible.py (+ baseten_spark_open, glimmer_hosted, lmstudio, meta configs) @@ -308,7 +310,7 @@ recall/ │ └── src/recall_pipeline/ │ ├── domain/ Episode, Frame, Pose, arkit_to_opencv(), align.py (Umeyama check/fallback), room_model.py (planes, Shapely 2D geometry), merge.py (per-label clustering), blur.py │ ├── application/ process_episode.py, ports.py (Reconstructor, Segmenter, EpisodeSource, ResultSink) -│ └── adapters/ mapanything_local.py, mapanything_remote.py, sam31_meta_api.py, sam31_modal.py (optional), open3d_geometry.py (clean-up, segment_plane, DBSCAN, FPFH+ICP), rerun_log.py, storage, api_client.py +│ └── adapters/ mapanything_local.py, mapanything_remote.py, sam31_meta_api.py, sam31_modal.py (optional), open3d_geometry.py (clean-up, segment_plane, DBSCAN), rerun_log.py, storage, api_client.py ├── deploy/ │ ├── baseten/README.md Spark Open: hosted endpoint, key only │ ├── modal/glimmer/ Glimmer serving (TODO(verify) Modal API); fireworks/ alternative @@ -321,7 +323,7 @@ recall/ │ ├── run.py │ ├── verifier.yaml 20 arrival frames: 10 right spot, 10 wrong spot │ └── results.md dated accuracy table -├── examples/a2a-client/home_robot.py external agent asking Recall over A2A (Muse Spark) +├── examples/ros2-robot/ a ROS 2 robot loading Recall's export (GET /homes/{id}/robot-package) ├── samples/replay/ recorded scan + recorded model responses for `make demo` (labelled as recorded) └── docs/ (docs/private/ is gitignored: planning docs, ADRs, progress notes, the key list) ├── architecture.md hexagon diagram + both paths (mermaid) @@ -346,9 +348,11 @@ recall/ ### Make targets -`setup dev demo live probes rerun test lint lint-imports migrate bundle-spec gen-models gen-swift-client eval` +`setup dev demo live probes rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval` - `demo`: replay from `samples/replay/`, no keys, output labelled as recorded. `live`: real Meta model calls with `.env`. +- `test-all`: every suite (API, pipeline, lint, imports, web, iOS unit + UI) under `caffeinate`; each run's logs and + summary go to `data/test-runs/` (gitignored) with a line in `runs.jsonl` and `history.md`. `SUITES="api pipeline"` picks some. - `probes`: one small real call per external model or service (Meta Model API, Baseten Spark Open, Muse Voice Transcribe, SAM 3.1, R2) that prints what worked and appends the result to `docs/verification.md`; needs `.env`; never run in CI. - `rerun EPISODE=`: replay a stored episode into the Rerun viewer (`RERUN=1`); dev only, never needed for tests or `make demo`. @@ -368,7 +372,7 @@ recall/ | 1 Scan spike + iPhone Scan | Oct 30 to Nov 1 | Full iPhone Scan feature (plain ARKit episode capture, SpeedCoach, metrics, scan report, upload, professional UI); one room episode with ARKit inputs → plane fitting → `room.json` + 3D objects in metres | Objects within ~20 cm of reality on an ARKit-input run (**go/no-go**). Runs alongside phase 2; about 8 to 11 h of iOS work | | 2 Memory + agent | Oct 31 to Nov 1 | Memory, spatial relations, Guide agent (Spark Open) with tools | 30-question eval runs; answers only from tool output | | 3 Guidance | Nov 1 to 2 | Find, ask, live ARKit guidance, voice; Watch; **Verifier (in-process)** | Walk to an object with eyes closed; arrival confirmed on camera | -| 4 Proof | Nov 2 to 3 | Viewer, real output on landing page, eval results, sample scan, `samples/replay/`; **A2A servers + schema validation and Meta-model screening + `home_robot.py` demo** | `make demo` works without GPU or keys; verifier precision ≥ 0.9 on 20 frames | +| 4 Proof | Nov 2 to 3 | Viewer, real output on landing page, eval results, sample scan, `samples/replay/`; **A2A Home Memory server + schema validation and Meta-model screening + robot export demo** | `make demo` works without GPU or keys; verifier precision ≥ 0.9 on 20 frames | | 5 Ship | Nov 3 to 4 | README ("What runs where"), 3-minute video, X post, submission, rehearsal | Submitted before Nov 4 evening Warsaw; tag `submission` | **Schedule:** phases keep their dates; we work overtime (evenings) to finish each one on time rather than cut scope. diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..287d640 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +# Installing Recall + +The installation guide lives in [docs/install/README.md](docs/install/README.md): what Recall is, then step by step +for a person or an AI agent (the website, the no-keys demo, and the live setup with the iPhone and the Watch). diff --git a/Makefile b/Makefile index 5b7282b..8d09d48 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,12 @@ SHELL := /bin/bash API := services/api PIPE := services/pipeline +# where the API listens: the LAN by default so the phone reaches it; BIND=127.0.0.1 when a Cloudflare Tunnel is the only way in +BIND ?= 0.0.0.0 WEB := apps/web REDOCLY := npx -y @redocly/cli@2.72.1 -.PHONY: help setup dev worker robot-export regression viewer live demo record probes diagrams pair rerun test lint lint-imports migrate bundle-spec gen-models gen-swift-client eval +.PHONY: help setup dev worker robot-export regression viewer public-scan live demo record probes diagrams pair rerun test test-all lint lint-imports migrate bundle-spec gen-models gen-swift-client eval help: @grep -E '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | awk -F':.*?## ' '{printf " %-18s %s\n", $$1, $$2}' @@ -17,7 +19,7 @@ setup: ## Install Python 3.12 services (uv), the web app (pnpm) and generate the $(MAKE) bundle-spec gen-models migrate dev: ## Run the API on :8000 (reachable on the LAN), the scan worker and the web app on :3000 - ( cd $(API) && uv run recall-api ) & $(MAKE) worker & ( cd $(WEB) && pnpm dev ) ; wait + ( cd $(API) && RECALL_BIND_HOST=$(BIND) uv run recall-api ) & $(MAKE) worker & ( cd $(WEB) && pnpm dev ) ; wait worker: ## The scan job worker (Huey on SQLite) and the warm pipeline (MapAnything kept loaded, freed after 10 min idle) ( cd $(PIPE) && uv run recall-pipeline serve ) & ( cd $(API) && uv run huey_consumer recall_api.infra.jobs.huey -w 1 -k thread ) ; wait @@ -31,8 +33,18 @@ robot-export: ## One room's memory for a robot (Nav2 map, things, furniture): ma regression: ## Re-run every processed scan now (no posting; cached SAM answers) and compare with the last run cd $(API) && uv run python -c "from recall_api.infra import jobs, regression; print('\n'.join(regression.run(jobs.settings, jobs.run_pipeline)))" +public-scan: ## The real scan for the public site, geometry only (no photo data): make public-scan [HOME_ID=flat] -> apps/web/public/real-scan// (review before committing) + @H="$(or $(HOME_ID),flat)"; \ + case "$$H" in *[!a-z0-9-]*|"") echo "HOME_ID: lower-case letters, digits and hyphens only."; exit 2;; esac; \ + ID=$$(sqlite3 data/recall.db "SELECT id FROM scans WHERE home_id = '$$H' AND status = 'processed' ORDER BY created_at DESC LIMIT 1" 2>/dev/null); \ + E=$$(ls -d data/blobs/scans/*_$$ID/episode 2>/dev/null | head -1); \ + if [ -z "$$ID" ] || [ ! -f "$$E/home_colors.npy" ]; then echo "No processed scan of '$$H' yet."; exit 1; fi; \ + cd $(PIPE)/scripts && uv run --project .. python export_viewer_scan.py --public "$(CURDIR)/$$E" "$(CURDIR)/$(WEB)/public/real-scan/$$H" && \ + echo "Review apps/web/public/real-scan/$$H/ (no photos, nothing personal) before committing it." + viewer: ## Export a home's newest processed scan for /homes//viewer: make viewer [HOME_ID=flat] (gitignored output) @H="$(or $(HOME_ID),flat)"; \ + case "$$H" in *[!a-z0-9-]*|"") echo "HOME_ID: lower-case letters, digits and hyphens only."; exit 2;; esac; \ ID=$$(sqlite3 data/recall.db "SELECT id FROM scans WHERE home_id = '$$H' AND status = 'processed' ORDER BY created_at DESC LIMIT 1" 2>/dev/null); \ E=$$(ls -d data/blobs/scans/*_$$ID/episode 2>/dev/null | head -1); \ if [ -z "$$ID" ] || [ ! -f "$$E/home_colors.npy" ]; then \ @@ -41,7 +53,7 @@ viewer: ## Export a home's newest processed scan for /homes//viewer: make echo "Open http://localhost:3000/homes/$$H/viewer" live: ## The API with real Meta model calls (keys from .env), and the scan worker - ( cd $(API) && uv run recall-api ) & $(MAKE) worker ; wait + ( cd $(API) && RECALL_BIND_HOST=$(BIND) uv run recall-api ) & $(MAKE) worker ; wait demo: ## No keys: the API on a memory built from a recorded real scan; the Guide's replies are replayed (labelled recorded) cd $(API) && uv run python -m recall_api.infra.demo serve $(if $(PORT),--port $(PORT)) @@ -68,6 +80,9 @@ test: ## Unit, integration and contract tests for both services cd $(API) && uv run pytest cd $(PIPE) && uv run pytest +test-all: ## Every suite (API, pipeline, lint, imports, web, iOS unit + UI), logs kept in data/test-runs/ (gitignored): make test-all [SUITES="api pipeline"] + python3 scripts/test_all.py + lint: ## ruff for both services, ESLint for the web app cd $(API) && uv run ruff check . && uv run ruff format --check . cd $(PIPE) && uv run ruff check . && uv run ruff format --check . diff --git a/README.md b/README.md index cf278c3..8b51e9a 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,19 @@

RECALL

-

Recall remembers where you put things, and walks you back to them.

+

Recall remembers where you put things, and walks you back to them.
+Built first for blind and low-vision people, useful to anyone who misplaces things.

-

“Where did I put my keys?� · “Have you seen my reading glasses?�
-“Where's my grey bag?� · “Why do you think so?�

+

LiDAR + Meta models: the iPhone's LiDAR measures, Meta's MapAnything builds the 3D map, +SAM 3.1 finds your things in it, and Muse Voice Transcribe + Muse Spark Open hear the question and answer it.
+Private · Never guesses · Voice-first

+ +

+ “Where did I put my keys?�
+ “Have you seen my reading glasses?�
+ “Where's my grey bag?�
+ “Why do you think so?� +

…then tap Take me there, and the camera checks it's really there when you arrive.

@@ -31,6 +40,7 @@ Indie track · Theme: Build for Your World

Muse Spark Open Muse Glimmer Muse Voice Transcribe + Baseten: serves Muse Spark Open

@@ -45,41 +55,85 @@ Indie track · Theme: Build for Your World

> What isn't finished yet is listed under [Not in this submission](#not-in-this-submission). +
+ +
+ +| **2.3 s** | **3 cm** | **60 s** | **2–7 cm** | **0** | +|:---:|:---:|:---:|:---:|:---:| +| from question to answer,
median of 10 real questions | arrival check on the real keys,
SAM 3.1 + LiDAR depth | to rebuild a room in 3D
on a Mac (M5 Pro) | where a no-model rebuild put
all 10 things vs MapAnything | non-Meta models
at runtime | + +Every number is measured and dated in docs/verification.md; see Results so far. + +
+ +
+ +### For judges, in one minute + +| Criterion | Where to look | +|---|---| +| **Impact** (30 %) | [Why I built it](#why-i-built-it): a true story · [What it is](#what-it-is): built first for blind and low-vision people, for anyone who misplaces things, voice first · [Privacy](docs/privacy.md): the home stays on your own computer · the [tester session plan](evals/usability/) | +| **Use of Meta AI tooling** (25 %) | [Only Meta models, each with one job](#how-it-works-the-meta-models): MapAnything, SAM 3.1, Muse Spark Open (Muse Glimmer on your own Mac), Muse Voice Transcribe, Muse Spark 1.1 · [Recall's three agents](#recalls-three-agents) · [Sharing with robots](#sharing-with-robots) over A2A | +| **Demo** (25 %) | The screenshots below are real output · `make demo` replays a real run with no keys · [How to run it](#how-to-run-it) | +| **Technical depth** (20 %) | [Architecture](docs/architecture.md) · [28 decisions](docs/decisions.md) · [Guardrails](#guardrails): geometry decides the facts, the model only phrases them · [Algorithms](#algorithms) · tests on every part ([Results so far](#results-so-far)) | + --- ## Contents 1. [What it is](#what-it-is) 2. [What's in this repo](#whats-in-this-repo) -3. [Why I built it](#why-i-built-it) -4. [Recall's three agents](#recalls-three-agents) -5. [Sharing with robots](#sharing-with-robots) -6. [How it works: the Meta models](#how-it-works-the-meta-models) -7. [Architecture in brief](#architecture-in-brief) -8. [API](#api) -9. [Guardrails](#guardrails) -10. [Algorithms](#algorithms) -11. [Results so far](#results-so-far) -12. [How to run it](#how-to-run-it) -13. [Not in this submission](#not-in-this-submission) -14. [Contributing](#contributing) -15. [Built during the hackathon](#built-during-the-hackathon) -16. [Licence](#licence) +3. [How the code is laid out](#how-the-code-is-laid-out) +4. [Why I built it](#why-i-built-it) +5. [Meta Muse in the code](#meta-muse-in-the-code) +6. [Recall's three agents](#recalls-three-agents) +7. [Built with Muse Code: three tiers of agents](#built-with-muse-code-three-tiers-of-agents) +8. [Sharing with robots](#sharing-with-robots) +9. [How it works: the Meta models](#how-it-works-the-meta-models) +10. [Architecture in brief](#architecture-in-brief) +11. [API](#api) +12. [Guardrails](#guardrails) +13. [Security and privacy](#security-and-privacy) +14. [Data model](#data-model) +15. [Algorithms](#algorithms) +16. [Results so far](#results-so-far) +17. [Installation](#installation) +18. [How to run it](#how-to-run-it) +19. [Not in this submission](#not-in-this-submission) +20. [Contributing](#contributing) +21. [Built during the hackathon](#built-during-the-hackathon) +22. [Licence](#licence) ## Screenshots -No image of anyone's home is in this repo: the 3D viewer shows only a room you scan yourself. - -![The landing page hero](docs/media/hero.png) - -The iPhone app, shown with sample data, not a real home: - -

- Find: the question 'Where are my keys?' and the answer 'Your keys are near the desk in the Living room, at waist height', with Take me there and Why? buttons - Why Recall thinks so: the answer, 'Phrased by Muse Spark Open from 1 sighting, in 2.3 seconds', and the sighting: seen in the Living room scan, 92% sure, found by SAM 3.1, placed by MapAnything - Robot Access: the switch to share the home map with robots, off; a preview of the room map for robots; and what robots receive - Privacy: your home's memory lives on your own computer; right now scan photos go to SAM 3.1 and your question as text to Muse Spark Open on Baseten; robot access is off -

+**Real output, from the author's own living room:** one walk with an iPhone, rebuilt on a Mac. The room map shows +floor, furniture and walls as cells, and the things as dots: no photos of the home are in this repo. + + + + + + + + + + + + + + + + + + + + + + +
Find: the question 'Where are my keys?' and the answer 'Your keys are near the desk in the living room at waist height, seen on Friday at 04:21', with Take me there and Why? buttons, and the room map belowFind: Your rooms, the Living room card with its map, 10 things as blue dots, and the list of things belowWhy Recall thinks so: the answer, 'Phrased by Muse Spark Open from 1 sighting, in 1.8 seconds', and the sighting: seen in the Living room scan on 30 Oct 2026, 70% sure
Ask. Muse Spark Open picks the tools; the place comes from the 3D memory. Take me there walks you to it.Your rooms. The map MapAnything and SAM 3.1 built from one walk. Swipe between rooms; tap one to see only its things.Why? The sighting behind the answer, and which model worded it, in how long.
Find: Your things, the Living room, on or near the desk: grey bag, red book, beige keys, bottles, glasses and a laptop, each with a Go buttonFound: search, filters, group by place, height or kind; the Living room, 10 things, 4 to check, with Check marks on the unsure onesRobot Access: the switch to share the home map with robots, off; the Living room map for robots; what robots receive and what is never shared
Your things. By room, then the furniture they're on or near. Go on every row.Found. Check what the scan found; fix a name, colour or height; the unsure ones are marked.Robot Access. Off by default. A robot gets the map and where things are, never photos or anyone's position.
+ +![The landing page: "Find your keys", RECALL in Braille, built first for blind and low-vision people and useful to anyone who forgets where they put things, the Ask Recall card, and Explore a real room · Actual scan](docs/media/hero.png) ## What it is @@ -121,13 +175,106 @@ The iPhone app, shown with sample data, not a real home: | A usability session plan for a blind or low-vision tester | `evals/usability/` | Ready | | The 3-minute demo video | | Coming | +## How the code is laid out + +``` +apps/ + ios/ RecallKit package (Domain, Application, Networking, Capture, Voice, Persistence, Design) + + thin Recall (iPhone), RecallWatch and RecallUITests targets + web/ Next.js: landing page, 3D viewer, voice demo route, Playwright tests (e2e/) +services/ + api/ FastAPI: domain/ → application/ → adapters/ + infra/ ; resources/ = the OpenAPI spec (source of truth) + pipeline/ the scan pipeline, same layers: MapAnything, SAM 3.1, planes, merging, the room model +docs/ architecture, privacy, verification, Meta Muse (docs/muse.md) +evals/ questions, usability plan +``` + +- **Hexagonal (ports and adapters) in every service**: domain � application � adapters and infra. + - *Why:* a Meta model, a store or a transport is swapped in one place (`infra/container.py`), and the domain is + tested with no network. + - *Kept honest by:* import-linter (4 contracts in CI) and `services/api/tests/unit/test_architecture.py`, which + checks that routers only map and only use cases write the memory. +- **One writer**: only `services/api` writes the home's memory; the pipeline posts its result + (`POST /scans/{id}/results`). + - *Why:* no two processes can race on what Recall remembers. +- **Spec-first API**: `services/api/resources/openapi.yaml` is the contract. The API's Pydantic models and the + iPhone's Swift client are generated from it. + - *Why:* the phone and the Mac can't drift. Schemathesis checks the running API against the spec. +- **The iPhone app is a package plus thin targets**: all logic lives in RecallKit (pure Swift domain, use cases, + adapters). The app, Watch and test targets only compose it. + - *Why:* the logic is unit-tested without a device or simulator. +- **The web is two motion systems, never on one element**: GSAP owns scroll storytelling and Motion owns UI state. + Reduced motion is respected by both. + +**Tests, and where they live** + +| Suite | Where | What it checks | +|---|---|---| +| API unit | `services/api/tests/unit` | Domain, use cases with fake adapters, architecture rules, request limits | +| API contract | `services/api/tests/contract` | Routes vs the spec, Schemathesis on every GET, `tools.json`, privacy notes in sync | +| API integration | `services/api/tests/integration` | SQLite + disk, scan to answer end to end | +| Pipeline | `services/pipeline/tests` | Geometry (planes, Umeyama, merging), the warm server's security | +| Web | `apps/web/e2e` | Every section, accessibility (axe, WCAG 2.2 AA), usability, security headers | +| iPhone unit | `apps/ios/Packages/RecallKit/Tests` | Domain, use cases, networking, capture | +| iPhone UI | `apps/ios/RecallUITests` | Accessibility audit of every screen, usability flows, pairing | + +`make test-all` runs every suite under `caffeinate` and logs each run locally; CI runs on GitHub Actions. + ## Why I built it -- The question "where did I put my keys?" is asked in every home, and for someone who is blind or has low vision it can end a morning. +- I lose things too: keys, glasses, the charger. For me it's a few minutes of searching. For someone who is blind or has low vision it can mean searching by touch, or asking for help, every day. +- So Recall is designed for the hardest case first: no screen needed, voice in and out, directions you can follow with your eyes closed. Built that way, it's simply quicker for everyone (the curb-cut effect). - A home barely changes; what moves are the small things. That calls for a memory of the space, not a smarter camera. - The pieces now exist as open Meta models: metric 3D from photos (MapAnything), find anything by name (SAM 3.1), a language model that can call tools and phrase a short spoken answer (Muse Spark Open / Glimmer). - I wanted the result measured on real objects in a real flat, with the numbers published, rather than demonstrated on slides. +## Meta Muse in the code + +Muse models are the core of Recall's question path: they hear the question, choose the tools and speak the answer. +Geometry decides every fact; Muse turns facts into speech. Full detail: [docs/muse.md](docs/muse.md). + +- **Muse Voice Transcribe hears the question** + - *What:* speech to text, with turn detection (`ENDPOINTING`), so Recall knows when you've finished. + - *Why:* Recall is voice-first, and this is the only speech model it uses (no Apple speech recognition). + - *How:* the phone records WAV → the Mac sends it to the Meta Model API → only the words are kept, never the audio. + - *Code:* `services/api/.../adapters/outbound/voice/muse_transcribe.py`, `application/transcribe_speech.py` +- **Muse Spark Open is the Guide** + - *What:* understands "where are my keys?", calls Recall's tools and says one or two short sentences. + - *Why:* it calls tools and phrases things naturally, with open weights. It never decides a place. + - *How:* + 1. The question arrives with five tools (`resources/tools.json`). + 2. Spark Open picks `find_object`, and Recall answers it from the 3D map. + 3. Spark Open words it: landmark, then shelf and height, then when. + 4. The code checks that every place it says came from a tool (`domain/assistant`), within 6 steps at most. + - *Code:* `application/ask_where_is.py`, `adapters/outbound/llm/openai_compatible.py` +- **Baseten serves Muse Spark Open** + - *What:* Baseten hosts Meta's open-weights Muse Spark Open (`meta-models/muse-spark-open-1026`) behind an + OpenAI-compatible endpoint (`https://inference.baseten.co/v1`) with tool calling. + - *Used for:* the Guide (the default, `LLM_PROVIDER=baseten_spark_open`, at `reasoning_effort: low`) and the + screen that checks what outside agents send (`SCREEN_PROVIDER=baseten_spark_open`). + - *Why:* open weights served fast (a median 2.3 s from question to answer), on the hackathon's Baseten credit, and + the Mac's memory stays free for MapAnything. + - *How:* one `BASETEN_API_KEY` in `.env`; the same OpenAI-compatible adapter as every other Muse provider. + - *Code:* `adapters/outbound/llm/openai_compatible.py`, `adapters/outbound/safety/meta_model_screen.py` +- **Muse Glimmer is private mode** + - *What:* the same Guide, on Muse Glimmer in LM Studio on your Mac. + - *Why:* your questions never leave home. + - *How:* `LLM_PROVIDER=lmstudio`; same adapter, same tools. +- **Muse Spark 1.1 is the Verifier's second opinion** + - *What:* looks at the arrival photo when SAM 3.1 is unsure. + - *Why:* a wrong "it's here" is worse than "I'm not sure". + - *How:* it can only lower a result to "uncertain", never confirm one. + - *Code:* `application/verify_arrival.py`, `adapters/outbound/vision/spark_vision_meta_api.py` +- **Muse Spark Open (or Glimmer) screens outside agents** + - *What:* checks the object name a robot or another app sends the Home Memory agent. + - *Why:* outside text must never steer a model or a tool. + - *How:* schema first (one name, at most 40 characters) → a strict JSON verdict → anything unclear is refused + (fails closed). + - *Code:* `adapters/outbound/safety/meta_model_screen.py`, `application/answer_external_query.py` +- **Recorded Muse replies make the demo work without keys** + - *What:* `make demo` replays real Muse Spark Open replies to a real scan, labelled as recorded. + - *Code:* `adapters/outbound/replay/recorded_llm.py` + ## Recall's three agents These are Recall's own parts. Each one runs only Meta models. @@ -137,6 +284,42 @@ These are Recall's own parts. Each one runs only Meta models. - **Home Memory** answers a robot or another app you connect (a pair of glasses, another assistant) over the open A2A 1.0 protocol: read-only, off by default, never the user's live position. It accepts only structured input (one object name, checked against a schema), and a Meta model (Muse Spark Open, or Glimmer) checks that name before any tool runs. Every request is in the owner's audit log. - One rule: every agent has a job no other agent does. Complexity is not the goal; the three jobs are different. +## Built with Muse Code: three tiers of agents + +Recall's three agents above run in the product. Separately, while building it, **Meta's Muse Code** (v1.4.14) tested +and audited the repository with agents of its own. Details and every finding: [docs/muse.md](docs/muse.md#part-b-how-muse-code-tested-recall). + +``` + Tester ─� + ├─▶ Tie-breaker ─▶ Synthesis (the only writer) +Reviewer ┘ +``` + +- **A workflow program spawns the agents** + - Muse Code writes a small program: `host.parallel` runs agents side by side, and `host.agent` runs the next stage. + - Results pass between stages as references, not files. +- **Every agent answers with evidence or "unresolved"** + - Each claim carries a `path:line` and the test behind it. A search result alone doesn't count. +- **Tier 1: Tester and Reviewer, side by side** + - The tester ran every suite three times, with an anti-loop rule: one attempt, then report "blocked" with the exact + error. + - The reviewer only read code: the hexagonal layers, the ports, the composition roots and the tests behind each. +- **Tier 2: Tie-breaker** + - Took every disagreement and re-read the code to settle it, without running the tests again. +- **Tier 3: Synthesis** + - The only agent that writes, so nothing is overwritten: the test report and a code, pipeline, UI and algorithm + checklist. +- **The security audit used the same shape** + - Backend, web and iPhone/Watch auditors ran side by side with no live attacks, then a synthesis wrote it up. + - A camera-privacy review was done by Muse Code directly. +- **What it found, and what happened next** + - A pipeline crash risk without an Apple GPU, a Watch that could wait forever, logic in routers, no Schemathesis, + a silent pairing-link takeover, camera prompts that didn't name Meta's models, and missing rate limits. + - Each was checked against the code and fixed. Commits are listed in [docs/muse.md](docs/muse.md#what-muse-code-found). +- **Where the tests ran** + - Muse Code ran the API, pipeline, web and lint suites in its sandbox. + - The iOS, Watch and network-server suites run on the Mac with `make test-all`, and every suite passes. + ## Sharing with robots A home robot can use the same memory to go and find something. Recall finds it on the map; picking it up is the @@ -238,7 +421,7 @@ routes match it. While `make demo` or `make live` runs, **Swagger UI** is at `ht | `POST /transcribe` | Speech to text through Muse Voice Transcribe | | `POST /scans`, `PUT /scans/{scan_id}/parts/{part_number}`, `POST /scans/{scan_id}/complete` | Resumable scan upload | | `DELETE /scans/{scan_id}` | Erase a scan from your computer, with what only it found | -| `GET /homes/{home_id}/objects`, `GET /homes/{home_id}/room` | What Recall found, and the room model | +| `GET /homes/{home_id}/objects`, `GET /homes/{home_id}/room`, `GET /homes/{home_id}/room-map` | What Recall found, the room model, and each room's map for Find | | `POST /guidance/sessions`, `POST /guidance/sessions/{session_id}/verify` | Directions, and the arrival check | | `GET /privacy`, `GET /audit` | What is sent where right now; who did what | | `GET /homes/{home_id}/robot-export`, `/a2a/home-memory` | For robots: the Nav2 map, and the Home Memory agent ([Agent Card](services/api/resources/agents/home-memory.card.json)) | @@ -255,13 +438,119 @@ Every route except `/health` needs a key: the owner's device key, or a robot's o 4. If tracking is limited, give landmark directions only, and say so. 5. One or two short sentences: landmark first, then shelf and height, then time. 6. At most 6 agent steps; on failure, the best grounded partial answer. +- **Never guesses, in four checks:** + 1. *The tools decide.* Places, relations, heights and times are computed from the 3D map (`domain/spatial`, `application/tools.py`); the model only chooses tools and words the answer. + 2. *The rule is in the prompt:* never state a place, distance or time that no tool returned. + 3. *The rule is in code:* `check_grounding` (`domain/assistant/grounding.py`) checks every number and every known place name in the model's sentence against the tool results. One miss gets one correction; a second miss drops the model's words, and a sentence built only from the tool facts is spoken (`application/ask_where_is.py`). + 4. *Not found is a fixed sentence,* never the model's: "I haven't seen that yet." + 5. *Tested:* `tests/unit/test_grounding.py` (an invented place or number is caught) and `tests/unit/test_ask_where_is.py` (an invented place is corrected, then replaced by the tool facts; a place with no sighting behind it is never said). - **Every answer is traceable.** Sightings are append-only. Every answer is logged with its model and tool steps and the sightings it rests on, and **Why?** on the phone (or `GET /answers/{answer_id}/evidence`, or asking the Guide "why?") leads back to the camera's view, the scan and the models that made it. An answer that names a place with no sighting behind it is never said. - **The Verifier can only downgrade.** The vision model may turn a result into "uncertain"; only SAM plus geometry can confirm. - **Two checks on the door.** A robot or outside app can send only one object name (letters, digits, spaces and hyphens, at most 40 characters, `services/api/resources/agents/find-object.input.schema.json`); free text is refused and never reaches a model or a tool. Then Muse Spark Open (or Glimmer) must answer `{"verdict": "safe", ...}` in strict JSON; anything else, a timeout or "injection" refuses. Measured: 21 ordinary names allowed, 18 instruction-like names refused, none wrong. -- **Privacy.** The home's memory lives on the owner's own computer (a Mac today); there is no Recall server. Scan photos go to SAM 3.1 on the Meta Model API, voice clips to Muse Voice Transcribe, and questions (no photos) to Muse Spark Open, or stay on the Mac with Glimmer in private mode. Deleting a scan on the phone erases it from the Mac too. No scan of anyone's home is in the repo or on the hosted site. Full note: [docs/privacy.md](docs/privacy.md), the same as the app's Settings → Privacy and the site's `/privacy`. +- **Privacy and security.** See [Security and privacy](#security-and-privacy): what stays on the Mac, what is sent where, and how the home's memory is protected. - **Honesty.** Sample and recorded data are labelled as such in the UI and the docs; the demo shows only what was built. - **Rules.** Only Meta models at runtime; no non-Meta model anywhere, and no TTS model of any kind. +## Security and privacy + +**Privacy: what stays home, and what goes where** + +- **On your Mac:** every scan you upload, the 3D map, the things and furniture found, your own names and notes, the + answer log and the audit log. There is no Recall server and no account; the developer never sees your home. +- **What leaves, and to which Meta model** (the same table as [docs/privacy.md](docs/privacy.md), the app's + Settings → Privacy and the site's `/privacy`): + + | What | Sent to | When | + |---|---|---| + | Scan photos, about 50 per room (no depth, no sound) | Meta Model API: SAM 3.1 | While the Mac processes a scan | + | A spoken question (a few seconds of audio; only the words are kept) | Meta Model API: Muse Voice Transcribe | When you ask by voice | + | The question as text and what memory says (no photos) | Baseten: Muse Spark Open, or Muse Glimmer on your Mac in private mode | Each time you ask | + | One camera photo | Meta Model API: SAM 3.1, and Muse Spark 1.1 only when SAM is unsure | On arrival | + | The room map, things and furniture (no photos, names, notes or anyone's position) | Robots and apps you gave the robot key | Only while Robot Access is on (off by default) | + +- **Your controls:** delete a scan and it is erased from the iPhone and the Mac; forget a thing; Robot Access off by + default; private mode keeps questions on the Mac. +- **People in view:** scan when nobody else is in the room. Anyone the camera sees is in the scan photos sent to SAM + 3.1; Recall never looks for people or faces. +- **The published demo scan:** the author's own living room is published for `/homes/flat/viewer`, reviewed and + approved by the author: + - points and surfaces tinted by height, the room model, the drawings, labels and robot files; + - no photos, video, depth images, keys or paths; guard tests fail if a photo file or photo colours ever appear. + +**Security: how the home's memory is protected** + +- **One key, on every call that touches the home** + - *What:* a device key the phone and the Mac share (pairing, not a login), compared in constant time; a robot has + its own read-only key. + - *Where:* `adapters/inbound/http/deps.py`; the warm pipeline server answers nothing without its key. +- **Pairing asks first** + - *What:* a `recall://pair` link shows "Pair with this Mac?" with the address and the key's length, never the key, + and changes nothing until you confirm. + - *Why:* any app or web page can open such a link. +- **The key stays on this phone** + - The Keychain item is this-device-only; copying the key is this-phone-only and expires after 2 minutes; the key is + sent only to the paired Mac; a bare address is https unless it is on your network or Tailscale. +- **Limits in front of the API** + - Per-key rate limits on the calls that reach a model or start the pipeline (429, recorded in the audit log). + - Body-size caps per route (413); episode zips refused past 1 GB unpacked or 20,000 files. + - Pipeline results with NaN or infinity, or labels with control characters, are refused. +- **Where the API listens:** loopback unless asked; `make live` opens it to your network (`BIND=0.0.0.0`) so the + phone can reach it; behind a tunnel, `make live BIND=127.0.0.1`. +- **Outside agents:** a schema allows one object name, then Muse Spark Open must return a strict JSON verdict; anything + unclear is refused (fails closed). +- **The website:** no framing, no type sniffing, a quiet referrer, the microphone only; `robots.txt` keeps crawlers off + local scans and the API; the contact form uses schema checks, a honeypot, a minimum fill time, BotID and rate limits. +- **On record:** an append-only audit log of every change, refusal and model call (who, what, outcome), kept 90 days. +- **In the process:** no secrets in git (`.env` is ignored); CodeQL on Python and TypeScript; Schemathesis checks the + running API against its spec; import-linter and an architecture test keep the one-writer rule. +- **How it was checked:** Meta's Muse Code audited the backend, the website and the iPhone and Watch apps side by + side; every finding was checked in the code and 28 were fixed ([docs/muse.md](docs/muse.md#what-muse-code-found)). +- **Not done yet, honestly:** with `make live` the API is reachable on your home network (still behind the key and the + limits); a scan shared from the app as a zip contains its photos (the share button's VoiceOver hint says so; there is no confirm step yet). + +## Data model + +All of Recall's memory is **SQLite on your Mac** (`data/recall.db`), in metres in the home frame (floor at z = 0, +z up). Only `services/api` writes it. The schema comes from the Alembic migrations in +[`services/api/db/migrations/versions/`](services/api/db/migrations/versions) (0001 to 0008). + +```mermaid +erDiagram + homes ||--o{ scans : "scanned in" + homes ||--o{ landmarks : "furniture" + homes ||--o{ objects : "things" + homes ||--o{ rejections : "not there" + scans ||--o{ keyframes : "frames" + scans ||--o{ observations : "saw" + objects ||--o{ observations : "sightings" + objects ||--o{ set_asides : "set aside" + homes ||--o{ answer_log : "answers" +``` + +| Table | What it holds | Notes | +|---|---|---| +| `homes` | A home: its name, room model (`room.json`), and first scan | | +| `scans` | One walkthrough: room, status, archive size and SHA-256, part size, frame and object counts, transform to home, the scan it was aligned to, whether memory started fresh, what changed, provenance | | +| `keyframes` | Per kept frame: index, time, ARKit pose (4×4), tracking state, image and depth file URIs | Files stay in `data/blobs` | +| `landmarks` | Furniture: label, bounding box, centroid, room, which way its front faces | | +| `objects` | A thing: label, colour, size, and your corrections (name, colour, height, note), confirmed or not | | +| `observations` | Each sighting: object, scan, keyframe, time, x/y/z, SAM 3.1 confidence, box in the photo, how many photos showed it | **Append-only** | +| `set_asides` | Sightings set aside by a rescan or a correction, and why | **Append-only** | +| `rejections` | "Not there": a label at a place, so later scans don't bring it back | | +| `answer_log` | Every Guide answer: question, text, grounded or not, tool steps, the sightings behind it, provider, model, tokens, latency | **Append-only**; the evidence behind **Why?** | +| `audit_events` | Who (owner, robot, pipeline, unknown), what action, outcome, target, a cleaned detail, request id | **Append-only**; kept 90 days | +| `owner_settings` | Key and value: robot access, trust level | | + +- **View `last_seen`:** the newest sighting of each thing with confidence ≥ 0.5 that has not been set aside. +- **Append-only, in the database itself:** triggers refuse any UPDATE on `observations`, `set_asides`, `answer_log` + and `audit_events`. +- **Indexes:** scans, landmarks, objects and rejections by home; observations and set-asides by object; audit events + by time and by action; the answer log by home and time. +- **The job queue** is a separate SQLite file (`data/jobs.db`, Huey) that holds only queued scan jobs. +- **Not in SQL:** the scan files (`data/blobs` on the Mac, or Cloudflare R2). On the iPhone there is no database: + scans are folders in Application Support, the device key is in the Keychain, and the Mac's address is in + UserDefaults. + ## Algorithms - **Keyframe selection:** a frame is kept when the phone moved about 15 cm or turned about 10°; blurry frames are dropped by a sharpness check. @@ -290,13 +579,42 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima - **Cost, measured:** a room scan is **456** SAM 3.1 calls and **48 MB** on disk; a question is **2** Muse Spark Open calls, about **2,460 + 261 tokens**. - **Flat surfaces:** the geometry-only detector found **12 level patches** in the room; the two largest sit at the heights of the two measured desks. - **Viewer:** about 150k points, the gather animation, photo and confidence colouring, and the floor grid, height bands and surfaces as layers, on a phone and a laptop. -- **Tests:** API 140, pipeline 45 (including a synthetic room run through the whole pipeline), website 143 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, and keyboard-only and 200 % text usability); on the iPhone, accessibility audits and usability tasks done with VoiceOver labels at default and largest text. +- **Tests (2 November):** API 202 (including Schemathesis against the spec), pipeline 59 (including a synthetic room run through the whole pipeline), website 190 (accessibility with axe, WCAG 2.2 AA, reduced motion, no-JavaScript, keyboard-only and 200 % text usability, security headers); on the iPhone, 182 unit tests and 31 UI tests: accessibility audits and usability tasks done with VoiceOver labels at default and largest text; the Watch app builds. +- **Usability, honestly:** the checks above are automated. In person, the author's screen-off run (VoiceOver Screen Curtain: the screen stays black) is planned, and a session with blind and low-vision testers is planned; its plan is ready in `evals/usability/`. **Planned evaluation:** median position error against hand-measured positions, questions answered correctly, Verifier precision, and time to find an object by touch versus with Recall, scored without AI; the dated table will live in `evals/results.md`. The session plan for a blind or low-vision tester is in `evals/usability/`. +## Installation + +**The full guide: [docs/install/README.md](docs/install/README.md).** It covers what Recall answers, then every step +for a person or an AI agent, each with its command, what you should see, and what to do if you don't. The website +offers the same guide as a download (`/INSTALL.md`). + +| Path | You get | Needs | Time | +|---|---|---|---| +| **A. Website** | The landing page and the real 3D scan at `/homes/flat/viewer` | Node 22 | 5 min | +| **B. Demo, no keys** | `make demo`: Recall answering "where are my keys?" from a recorded real scan, labelled as recorded | uv (Python 3.12 comes with it) | 10 min | +| **C. Live** | Your own room: `make live`, the iPhone app (Xcode + XcodeGen), pairing by QR, the Watch | A Mac with Apple silicon, your own Meta Model API and Baseten keys, an iPhone with LiDAR on iOS 26 | 45 min | + +The short version: + +```bash +git clone https://github.com/coder-RT/recall.git && cd recall +make setup # Python services (uv), the web app (pnpm), generated models, the database +make demo # B: no keys; then: curl -s -X POST localhost:8000/ask -H "Authorization: Bearer demo" ... +cd apps/web && pnpm dev # A: http://localhost:3000 and /homes/flat/viewer +``` + +- **Keys** go only in `.env` at the repo root, which git ignores. The guide shows the variable names, never values. +- **The phone reaches the Mac** on the same Wi-Fi or over Tailscale. `make pair URL=…` shows a QR code, and the app + asks "Pair with this Mac?" before it uses it. +- **Checked from a fresh clone:** `make setup`, the website, `make demo`, `make test lint lint-imports` (API 202, + pipeline 57) and the iPhone build. + ## How to run it -**Status today:** the website, the 3D viewer, the backend, the pipeline and the iPhone app all run. The Apple Watch app is planned. +**Status today:** the website, the 3D viewer, the backend, the pipeline, the iPhone app and a preview of the Apple +Watch app all run. ### 1. The landing page and the 3D viewer (runs now) @@ -309,7 +627,7 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima ``` - Open: - `http://localhost:3000/` for the landing page; - - `http://localhost:3000/homes/flat/viewer` for your own scanned home (`flat` is the app's default home name); until it has a scan, the page lists the steps (section 2). + - `http://localhost:3000/homes/flat/viewer` for the **real scan** of the author's living room. It's published as geometry only (no photo data; `make public-scan`), with which Meta model made each part, the measured numbers, what SAM 3.1 found, and what a robot gets. Any other home name shows your own scan, or the steps to make one (section 2). - Optional: "Ask by voice" on the landing page calls Muse Voice Transcribe through a server route. Create `.env` in the repo root with `META_MUSE_API_KEY=`; without it the card still works with the preset questions. Every spoken line on the site is the browser's own voice. - Checks: ```bash @@ -338,6 +656,7 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima - `make setup`: Python 3.12 with uv, both services, the web app. - `make demo`: no keys, no GPU. Builds a memory from a recorded real scan (`samples/replay/`, geometry and labels only, no photos) through the real ingest code, then serves the API on `localhost:8000` with Muse Spark Open's recorded replies; every answer is labelled `replay:muse-spark-open (recorded)`, and the command prints questions to try. `make record` re-records it with real calls. - `make live`: the API and the scan worker (`make worker` alone starts the worker: the warm pipeline server plus the job queue), with real Meta model calls using the keys from `.env`. The minimum is one Meta Model API key from [dev.meta.ai](https://dev.meta.ai) with `LLM_PROVIDER=meta`; the demo video uses Muse Spark Open on Baseten (`LLM_PROVIDER=baseten_spark_open`); the local privacy mode uses Muse Glimmer in LM Studio (`LLM_PROVIDER=lmstudio`). +- **Where the API listens:** `make live` and `make dev` listen on your network (`BIND=0.0.0.0`) so the phone can reach the Mac; behind a Cloudflare Tunnel use `make live BIND=127.0.0.1`. Run on its own (`uv run recall-api`), it listens on 127.0.0.1 only. Every route except `/health` and the API docs needs the device key, the calls that reach a model are rate-limited, and bodies are size-checked. - **Home Memory agent (A2A) with `make live`:** its check runs on Muse Spark Open with the Baseten key (`SCREEN_PROVIDER=baseten_spark_open`, the default) or on Glimmer in LM Studio (`SCREEN_PROVIDER=lmstudio`); no other key is needed. - `make test`, `make lint`, `make lint-imports`: tests, ruff and ESLint, and the architecture rule (domain � application � adapters). @@ -347,10 +666,21 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima - In the app's Settings, enter the Mac's address (the API on your network, a Tailscale HTTPS address or a Cloudflare Tunnel URL) and the device key from `.env`. No other keys live on the phone. - Tabs: - **Devices** (first; the app still opens on Find): where Recall runs. The iPhone is available now; Apple Watch Ultra 4 support is coming next and Meta AI glasses are planned, each labelled as such. **Home robots → Robot Access**: the switch, what robots receive and never receive, a map preview, the export, and recent robot activity. **Your privacy** opens the privacy note. - - **Find**: ask "where are my keys?" by typing or speaking (Muse Voice Transcribe on the Mac), hear the answer, then **Why?** (the sightings behind it, read aloud) or **Take me there**. The phone relocalises in the saved room map, the Mac plans a route, and the phone speaks clock-face turns with the compass point and distance ("At 2 o'clock, north-east, 3 metres"), with a tick when you face the right way and a blue arrow over the thing in the camera view for a sighted helper. Without a map match it says the landmark and which way it lies, and ticks when you face it. - - **Scans**: record a room, led by voice, with a step card, a top-down mini-map and the room-map state on screen; then upload it. **Scan again** compares the room with its last scan and says what moved, what's new and what's gone. Deleting a scan asks first, then erases it from the iPhone and the Mac. + - **Find**: ask "where are my keys?" by typing or speaking (Muse Voice Transcribe on the Mac), hear the answer, then **Why?** (the sightings behind it, read aloud) or **Take me there**. Below, **Your rooms** shows a map card per room (swipe sideways; tap one to show only its things), and **Your things** is grouped like Found: each room, then the furniture things are on or near, with **Go** on every row. The phone relocalises in the saved room map, the Mac plans a route, and the phone speaks clock-face turns with the compass point and distance ("At 2 o'clock, north-east, 3 metres"), with a tick when you face the right way and a blue arrow over the thing in the camera view for a sighted helper. Without a map match it says the landmark and which way it lies, and ticks when you face it. + - **Scans**: record a room, led step by step: + - **Before:** "How to scan well" lists every step, with what to do, why it matters, when it's done, and how high to hold the phone. + - **During:** + - the step card shows a progress meter ("Wall sides seen 5 of 9"); + - the phone's measured height above the floor (chest height 1.1 to 1.5 m, waist height 0.8 to 1.1 m to sweep tables), with a gentle spoken reminder; + - the distance to the wall on the walls step; + - Camera, LiDAR (depth as heat) and Edges views; + - a neon grid of 1 m columns by Recall's five height zones that glows where the LiDAR has looked; + - live readouts of distance, zones, photos, blurry photos and speed. + - **After:** a report shows how each step went, the zones, walls, floor, pace, light and compass, and up to three tips for next time. + + Then upload it. **Scan again** compares the room with its last scan and says what moved, what's new and what's gone. Deleting a scan asks first, then erases it from the iPhone and the Mac. - **Found**: what Recall found, grouped by room and furniture with an icon for each kind of thing; correct the name, colour or height, or forget a thing. - - **Settings**: **Privacy** (what stays home and what is sent where, read live from the Mac), the Mac's address and key, and Appearance (automatic, light or dark, plus high contrast in black, white and yellow). + - **Settings**: **Privacy** (what stays home and what is sent where, read live from the Mac), the Mac's address and key, and Appearance (automatic, light or dark, plus high contrast in black, white and yellow, and Tap sounds: a soft tap and a light buzz when you switch tabs, muted by the silent switch). ### 5. Hosted version (optional) @@ -359,6 +689,9 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima rate-limited per visitor by a Vercel firewall rule (`voice-transcribe`, via `@vercel/firewall`), has a daily cap (`VOICE_DAILY_CAP`, default 300), and can be switched off at once with `VOICE_DEMO=off`. The preset questions always work. +- The contact form sends messages through the email service Resend, so no address appears on the site. It takes only + the site's own pages, checks with Vercel BotID that a browser sent it, has a hidden trap field, a minimum fill time, + a per-visitor limit and a daily cap (`CONTACT_DAILY_CAP`, default 20), and turns off with `CONTACT_FORM=off`. - Everything also runs locally from the steps above. ### 6. Environment variables @@ -371,12 +704,12 @@ Measured, dated, and recorded in `docs/verification.md`. Nothing below is estima Planned next, not built or not working yet (updated before submission; removed if everything works): -- The Apple Watch app (taps and Digital Crown repeat) and Meta AI glasses. -- The ROS 2 robot example skill (in progress). +- The Apple Watch app on a real Ultra 4 (built and unit-tested, in preview) and Meta AI glasses (planned). +- A physical robot running the ROS 2 example (the example and the robot package are in the repo). - The tape-measured 20 cm accuracy check with the phone's depth and poses. - Quantitative evaluations and the Verifier precision figure. -- The usability session with a blind or low-vision tester (plan ready in `evals/usability/`). -- Daily spend caps on model calls (today only the website's voice route is rate-limited). +- The usability session with blind and low-vision testers (plan ready in `evals/usability/`), and the author's screen-off run. +- Daily spend caps on model calls (the API and the website's voice route are rate-limited per key or visitor; there is no daily cap yet). - The 3-minute demo video. ## Contributing diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index 1c4255a..a95adfa 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -19,6 +19,7 @@ Library versions are pinned in the lockfiles (`pnpm-lock.yaml`, `uv.lock`). | What | Where | Terms | |---|---|---| | Thing icons (keys, wallet, bag, backpack, book, bottle, glasses, phone, remote, mug, laptop, headphones, charger, watch, passport, and the box for unsure things) | `apps/ios/Recall/Resources/Assets.xcassets/Things`, drawn by `apps/ios/scripts/make_thing_icons.py` | Drawn for Recall; Apache-2.0 with the rest of the repo. No third-party art | +| Tab tap sound | `apps/ios/Recall/Resources/tap.wav`, made by `apps/ios/scripts/make_tap_sound.py` (two damped sine waves) | Made for Recall; Apache-2.0 with the rest of the repo. No third-party audio | ## Hosting and infrastructure @@ -41,6 +42,8 @@ Direct dependencies only. Licences below are as published by each project. |---|---|---| | Next.js | Web framework | MIT | | `@vercel/firewall` 1.2.6 | The voice demo route asks Vercel's firewall rule `voice-transcribe` whether a visitor is over the limit | Apache-2.0 (npm `license` field, checked 2026-11-01). Used as an npm dependency. | +| `botid` 1.5.11 | Vercel BotID: an invisible check that the landing page's contact form is sent by a browser (`apps/web/instrumentation-client.ts`, `app/api/contact/route.ts`) | MIT, per the package README (the npm `license` field is empty and no LICENSE file ships; checked 2026-11-01). Used as an npm dependency. | +| Resend (email service) | Delivers the contact form's messages to the developer's inbox; called over HTTPS from `apps/web/app/api/contact/route.ts`, no SDK | Service, used under Resend's terms; not a model. | | React | UI | MIT | | three.js (`three` 0.186.1) | Point-cloud rendering in the 3D viewer | MIT | | react-three-fiber (`@react-three/fiber` 9.8.2) | React renderer for three.js (`apps/web/components/viewer/`) | MIT | diff --git a/apps/ios/Packages/RecallKit/Package.swift b/apps/ios/Packages/RecallKit/Package.swift index 499b5b6..a3118ec 100644 --- a/apps/ios/Packages/RecallKit/Package.swift +++ b/apps/ios/Packages/RecallKit/Package.swift @@ -5,7 +5,7 @@ import PackageDescription let package = Package( name: "RecallKit", - platforms: [.iOS("26.0"), .macOS("15.0")], // macOS only so `swift test` can check the pure targets on the Mac + platforms: [.iOS("26.0"), .watchOS("26.0"), .macOS("15.0")], // watchOS: the Watch app uses RecallDomain; macOS for `swift test` products: [ .library(name: "RecallDomain", targets: ["RecallDomain"]), .library(name: "RecallCapture", targets: ["RecallCapture"]), diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/AnswerWatch.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/AnswerWatch.swift new file mode 100644 index 0000000..0543dfa --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/AnswerWatch.swift @@ -0,0 +1,51 @@ +import Foundation +import RecallDomain + +/// The iPhone answering its Watch: the things to pick from, the compass route to a picked thing, or a spoken question +/// (Muse Voice Transcribe, then the Guide) and the route to what it found. The Watch never holds a key; this is the +/// only way its questions reach the Mac. Failures come back as words for the Watch to say. +public struct AnswerWatch: Sendable { + private let memory: any HomeMemory + private let guide: any GuideService + private let transcriber: any Transcriber + private let routes: any CompassRoutes + + public init(memory: any HomeMemory, guide: any GuideService, transcriber: any Transcriber, routes: any CompassRoutes) { + self.memory = memory; self.guide = guide; self.transcriber = transcriber; self.routes = routes + } + + public func callAsFunction(_ request: WatchRequest, home: String, audio: Data? = nil) async -> WatchReply { + do { + switch request { + case .things: + // only what the owner's trust level counts: the same things Find offers + let things = try await memory.things(home: home).filter(\.trusted).sorted { $0.title < $1.title } + guard !things.isEmpty else { return WatchReply(things: [], say: "I don't remember anything yet. Scan a room first.") } + return WatchReply(things: things.map { WatchThing(id: $0.id, title: $0.title.capitalizedFirstLetter, place: $0.place) }) + case .route(let id): + return try await walk(to: id, home: home, answer: nil) + case .ask: + guard let audio, !audio.isEmpty else { return WatchReply(say: "I didn't hear anything. Try again.") } + let question = try await transcriber.transcribe(wav: audio) + guard !question.trimmingCharacters(in: .whitespaces).isEmpty else { + return WatchReply(say: "I didn't catch that. Turn the crown to pick a thing instead.") + } + let answer = try await guide.ask(home: home, question: question, about: nil) + guard let id = answer.thingId, answer.grounded else { return WatchReply(say: answer.text) } + return try await walk(to: id, home: home, answer: answer.text) + } + } catch { + return WatchReply(say: UserMessage.explain(error)) + } + } + + private func walk(to id: String, home: String, answer: String?) async throws -> WatchReply { + let route = try await routes.compassRoute(home: home, thing: id) + let first = answer ?? "Your \(route.title): \(route.place)." + return WatchReply(say: "\(first) To walk there, stand \(route.start), then double tap.", route: route) + } +} + +private extension String { + var capitalizedFirstLetter: String { prefix(1).uppercased() + dropFirst() } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/Setup.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/Setup.swift index b9e4c98..c8f9df4 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallApplication/Setup.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/Setup.swift @@ -8,7 +8,17 @@ public struct PairDevice: Sendable { /// The new connection, nil when the link isn't a pairing link; throws when it's an incomplete one. public func callAsFunction(_ link: URL) throws -> Connection? { - guard let pairing = try PairingLink(link) else { return nil } + guard let pairing = try read(link) else { return nil } + return apply(pairing) + } + + /// The link's pairing, not yet applied: any app or web page can open a recall:// link, so the person confirms + /// which Mac first (the old address and key stay until they do). + public func read(_ link: URL) throws -> PairingLink? { try PairingLink(link) } + + /// Uses the pairing the person confirmed. + @discardableResult + public func apply(_ pairing: PairingLink) -> Connection { var c = store.load() c.address = pairing.address c.key = pairing.key diff --git a/apps/ios/Packages/RecallKit/Sources/RecallApplication/TrackProcessing.swift b/apps/ios/Packages/RecallKit/Sources/RecallApplication/TrackProcessing.swift index 7763961..3432149 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallApplication/TrackProcessing.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallApplication/TrackProcessing.swift @@ -19,26 +19,35 @@ public struct TrackProcessing { public let state: UploadState public let changes: [ScanChange] public let fresh: Bool - public init(_ state: UploadState, changes: [ScanChange] = [], fresh: Bool = false) { - self.state = state; self.changes = changes; self.fresh = fresh + /// Still processing, and whether the Mac's worker has stopped (so the scan only waits). + public var workerStopped = false + public init(_ state: UploadState, changes: [ScanChange] = [], fresh: Bool = false, workerStopped: Bool = false) { + self.state = state; self.changes = changes; self.fresh = fresh; self.workerStopped = workerStopped } } - /// The scans whose state changed (unreachable Macs change nothing). - public func callAsFunction(_ scans: [Scan], home: String) async -> [String: Outcome] { + /// The scans whose state changed (unreachable Macs change nothing). `announce` false: update quietly (the app just + /// opened; what finished while it was closed shows on the cards instead of talking over the person). + public func callAsFunction(_ scans: [Scan], home: String, announce: Bool = true) async -> [String: Outcome] { var out: [String: Outcome] = [:] + let say = { (line: String) in if announce { speech.say(line, interrupt: false) } } for scan in scans where scan.isProcessing { guard let report = try? await processing.status(scan: scan.id) else { continue } switch report.status { case .processed: out[scan.id] = Outcome(.processed, changes: report.changes, fresh: report.fresh) let things = (try? await memory.things(home: home)) ?? [] - speech.say(Self.announce(scan, report: report, things: things), interrupt: false) + say(Self.announce(scan, report: report, things: things)) case .failed: out[scan.id] = Outcome(.failed(reason: report.detail ?? "The Mac couldn't process this scan.")) - speech.say("Processing \(scan.room) failed. The reason is on the scan.", interrupt: false) + say("Processing \(scan.room) failed. The reason is on the scan.") default: - break + // waiting with no worker: say so once, and again only after it was running in between + let stopped = report.workerAlive == false + if stopped != scan.workerStopped { + out[scan.id] = Outcome(.processing, workerStopped: stopped) + if stopped { say(Scan.workerStoppedMessage) } + } } } return out @@ -47,7 +56,7 @@ public struct TrackProcessing { /// A rescan says what changed; a first scan (or a fresh start) says how much was found. static func announce(_ scan: Scan, report: ScanReport, things: [Thing]) -> String { if report.fresh { - return "I didn't recognise \(scan.room), so its memory started afresh. " + ready(scan.room, things: things) + return ChangeSummary.sayFresh(room: scan.room) + " " + ready(scan.room, things: things) } if scan.alignedTo != nil || !report.changes.isEmpty { return ChangeSummary.say(room: scan.room, changes: report.changes) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift index b4208df..6e2a75e 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ARLocator.swift @@ -14,8 +14,13 @@ public final class ARLocator: Locator { public let scene = SCNScene() private var arrow: SCNNode? private let track = SCNNode() + private let tags = SCNNode() + private var shown: [ArLabel] = [] - public init() { scene.rootNode.addChildNode(track) } + public init() { + scene.rootNode.addChildNode(track) + scene.rootNode.addChildNode(tags) + } public static var isSupported: Bool { ARWorldTrackingConfiguration.isSupported } @@ -122,6 +127,90 @@ public final class ARLocator: Locator { arrow = node } + /// Labels over everything found: a tag (its picture, made by the caller) 35 cm above each thing, facing the camera, + /// a line down to the thing and a ring on the floor under it, in the label's colour. + public func showLabels(_ items: [(label: ArLabel, tag: UIImage)], floorY: Double?) { + clearLabels() + shown = items.map(\.label) + let still = UIAccessibility.isReduceMotionEnabled + for (i, item) in items.enumerated() { + let p = item.label.position + let colour = Self.colour(item.label.mark) + let node = SCNNode() + node.simdPosition = SIMD3(Float(p.x), Float(p.y), Float(p.z)) + + let height: Float = 0.35 + let width = Float(item.tag.size.width / item.tag.size.height) * 0.09 + let plane = SCNPlane(width: CGFloat(width), height: 0.09) + let face = SCNMaterial() + face.lightingModel = .constant + face.diffuse.contents = item.tag + face.isDoubleSided = true + face.readsFromDepthBuffer = false // the tag stays readable in front of furniture + plane.materials = [face] + let tag = SCNNode(geometry: plane) + tag.simdPosition.y = height + tag.constraints = [SCNBillboardConstraint()] + tag.renderingOrder = 10 + node.addChildNode(tag) + + let line = SCNCylinder(radius: 0.003, height: CGFloat(height - 0.045)) + line.materials = [Self.flat(colour)] + let lineNode = SCNNode(geometry: line) + lineNode.simdPosition.y = (height - 0.045) / 2 + node.addChildNode(lineNode) + + let dot = SCNSphere(radius: 0.015) + dot.materials = [Self.flat(colour)] + node.addChildNode(SCNNode(geometry: dot)) + + if let floorY { + let ring = SCNTorus(ringRadius: 0.12, pipeRadius: 0.006) + ring.materials = [Self.flat(colour.withAlphaComponent(0.85))] + let ringNode = SCNNode(geometry: ring) + ringNode.simdPosition.y = Float(floorY - p.y) + 0.005 + node.addChildNode(ringNode) + } + if !still { + node.opacity = 0 + node.runAction(.sequence([.wait(duration: 0.12 * Double(i)), .fadeIn(duration: 0.35)])) + } + tags.addChildNode(node) + } + } + + public func clearLabels() { + tags.childNodes.forEach { $0.removeFromParentNode() } + shown = [] + } + + /// The label the camera points at most directly (within 25°), for "what's this?". + public func labelInView() -> ArLabel? { + guard let cam = session.currentFrame?.camera.transform else { return nil } + let eye = SIMD3(Double(cam.columns.3.x), Double(cam.columns.3.y), Double(cam.columns.3.z)) + let fwd = -SIMD3(Double(cam.columns.2.x), Double(cam.columns.2.y), Double(cam.columns.2.z)) + let scored = shown.map { label -> (ArLabel, Double) in + let to = simd_normalize(label.position - eye) + return (label, acos(max(-1, min(1, simd_dot(to, simd_normalize(fwd))))) * 180 / .pi) + } + return scored.filter { $0.1 < 25 }.min { $0.1 < $1.1 }?.0 + } + + public static func colour(_ mark: ArLabel.Mark) -> UIColor { + switch mark { + case .told: UIColor(red: 0x42 / 255, green: 0xb7 / 255, blue: 0x2a / 255, alpha: 1) // cta green + case .check: UIColor(red: 0xf2 / 255, green: 0xb3 / 255, blue: 0x3d / 255, alpha: 1) // amber + case .seenOnce: UIColor(red: 0xe5 / 255, green: 0x48 / 255, blue: 0x3a / 255, alpha: 1) // red + } + } + + private static func flat(_ colour: UIColor) -> SCNMaterial { + let m = SCNMaterial() + m.lightingModel = .constant + m.diffuse.contents = colour + return m + } + public func showPath(_ path: [SIMD2], floorY: Double?) { track.childNodes.forEach { $0.removeFromParentNode() } let chevrons = ChevronLayout.place(path: path) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallCapture/CaptureSession.swift b/apps/ios/Packages/RecallKit/Sources/RecallCapture/CaptureSession.swift index f2fd325..535d131 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallCapture/CaptureSession.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallCapture/CaptureSession.swift @@ -21,6 +21,21 @@ public struct CaptureUpdate: Sendable, Equatable { public var mapReady: Bool /// Floor and walls the LiDAR saw, and the walked path, for the mini-map. public var surfaces: SurfaceCoverage + /// Walking speed, m/s (the readout turns amber when it's too fast for sharp photos). + public var speed: Double = 0 + /// Distance to whatever is straight ahead, metres (nil: nothing in range). + public var centreDistance: Double? + /// LiDAR points sampled so far, and the newest ones (ARKit world) to paint on the camera; empty between samples. + public var pointsCaptured = 0 + public var newPoints: [SIMD3] = [] + /// The current step's meter ("Wall sides seen 5 of 9"), and the steps finished so far. + public var stepProgress: ScanGuide.StepProgress? + /// The phone's height above the floor, metres (nil until the LiDAR has found the floor). + public var phoneHeight: Double? + /// The floor's height (ARKit world) and which column-and-height zones the LiDAR has seen, for the grid. + public var floorY: Float? + public var zones = ZoneCoverage() + public var stepsDone: [ScanGuide.StepRecord] = [] public init(elapsed: Double, directionsCovered: Int, keyframes: Int, droppedBlurry: Int, tracking: TrackingState, signal: ScanSignals.Event? = nil, lightLux: Double, sectors: Set = [], @@ -60,6 +75,9 @@ public final class CaptureSession: NSObject, ARSessionDelegate, @unchecked Senda private var lastDepthSample: Double = -1 private var lastPosition: simd_float3? private var centreDepth: Double? + private var pointsCaptured = 0 + private var zones = ZoneCoverage() + private var freshPoints: [SIMD3] = [] private var highConfidence: Double? private var keptSharpness: [Float] = [] private var dropped = 0 @@ -87,7 +105,7 @@ public final class CaptureSession: NSObject, ARSessionDelegate, @unchecked Senda queue.sync { self.writer = writer self.heading = heading - startedAt = Date(); dropped = 0; keptSharpness = [] + startedAt = Date(); dropped = 0; keptSharpness = []; pointsCaptured = 0; freshPoints = []; zones = ZoneCoverage() alignTo = map == nil ? nil : rescan?.scanId recognising = alignTo != nil recogniseStarted = nil; alignedTo = nil @@ -125,14 +143,16 @@ public final class CaptureSession: NSObject, ARSessionDelegate, @unchecked Senda return try queue.sync { guard let writer, writer.frameCount > 0 else { throw CaptureError.noFrames } let compass = calibration.summary + let duration = Date().timeIntervalSince(startedAt) let metrics = CaptureMetrics( directionsCovered: coverage.directionsCovered, keyframesKept: writer.frameCount, keyframesDroppedBlurry: dropped, trackingNormalRatio: coverage.trackingNormalRatio, meanSpeedMps: coverage.meanSpeed, peakSpeedMps: coverage.peakSpeed, slowDownCues: signals.cues, areaWalkedM2: coverage.areaWalked, meanLightLux: coverage.meanLight, wallsCovered: surfaces.wallsCovered, floorSeenM2: (surfaces.floorSeenM2 * 100).rounded() / 100, - mapStatus: mapStatus, loopClosed: guide.loopClosed, compassSpreadDeg: compass?.spreadDeg) - let duration = Date().timeIntervalSince(startedAt) + mapStatus: mapStatus, loopClosed: guide.loopClosed, compassSpreadDeg: compass?.spreadDeg, + durationS: duration.rounded(), points: pointsCaptured, steps: guide.records, + zonesSeen: zones.zonesSeen, zonesTotal: zones.zonesTotal) self.writer = nil return try writer.finish(worldMap: mapData, metrics: metrics, compass: compass, alignedTo: alignedTo, duration: duration, device: device, appVersion: appVersion, createdAt: startedAt) @@ -191,6 +211,9 @@ public final class CaptureSession: NSObject, ARSessionDelegate, @unchecked Senda centreDepth = s.centre highConfidence = s.high surfaces.add(samples: s.points, camera: position) + pointsCaptured += s.points.count + freshPoints = s.points + if let floor = surfaces.floorY { zones.add(s.points, floorY: floor) } } if tracking == .normal, t - lastBlurCheck >= selector.minInterval, selector.qualifies(pose, at: t), @@ -228,9 +251,20 @@ public final class CaptureSession: NSObject, ARSessionDelegate, @unchecked Senda signal: pendingSignal, lightLux: lux ?? 0, sectors: coverage.sectors, mapStatus: mapStatus, step: guide.step, guideLine: pendingLine, mapReady: guide.mapReady, surfaces: surfaces) + var live = update + live.speed = speed + live.centreDistance = centreDepth + live.pointsCaptured = pointsCaptured + live.newPoints = freshPoints + live.stepProgress = ScanGuide.progress(of: guide.step, progress) + live.phoneHeight = surfaces.floorY.map { Double(position.y - $0) } + live.floorY = surfaces.floorY + live.zones = zones + live.stepsDone = guide.records + freshPoints = [] pendingSignal = nil pendingLine = nil - onUpdate?(update) + onUpdate?(live) } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallCapture/ScannerFilter.swift b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ScannerFilter.swift new file mode 100644 index 0000000..899fcd8 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallCapture/ScannerFilter.swift @@ -0,0 +1,77 @@ +import CoreImage +import CoreImage.CIFilterBuiltins +import CoreVideo + +/// What the camera looks like in the scan and AR views: the plain camera, the LiDAR depth as heat, or edges like a +/// scanner. Classic image processing (Core Image); no model sees the pictures. +public enum CameraLook: String, CaseIterable, Sendable { + case camera, lidar, edges + + public var title: String { + switch self { + case .camera: "Camera" + case .lidar: "LiDAR" + case .edges: "Edges" + } + } + + public var hint: String { + switch self { + case .camera: "The plain camera." + case .lidar: "The LiDAR's distance as colour: warm is near, cool is far, with the points already captured." + case .edges: "The camera drawn as edges, like a scanner." + } + } +} + +/// Renders the overlays from one ARKit frame, small and fast (about 10 times a second is plenty for a layer). +public final class ScannerFilter: @unchecked Sendable { + private let context = CIContext(options: [.cacheIntermediates: false]) + public init() {} + + /// The camera image as glowing pale-blue edges on near-black, portrait, `longEdge` pixels on its long side. + public func edges(_ image: CVPixelBuffer, longEdge: CGFloat = 480) -> CGImage? { + let input = scaled(CIImage(cvPixelBuffer: image).oriented(.right), longEdge: longEdge) + let edges = CIFilter.edges() + edges.inputImage = input + edges.intensity = 6 + guard let e = edges.outputImage else { return nil } + // grey edges → the site's pale blue (#dfe3ee) glow; everything else stays dark + let mono = CIFilter.maximumComponent() + mono.inputImage = e + let tint = CIFilter.colorMatrix() + tint.inputImage = mono.outputImage + tint.rVector = CIVector(x: 0.87, y: 0, z: 0, w: 0) + tint.gVector = CIVector(x: 0, y: 0.89, z: 0, w: 0) + tint.bVector = CIVector(x: 0, y: 0, z: 0.93, w: 0) + tint.biasVector = CIVector(x: 0.02, y: 0.03, z: 0.06, w: 0) + guard let out = tint.outputImage else { return nil } + return context.createCGImage(out, from: input.extent) + } + + /// LiDAR depth as heat: warm near, cool far (0.3 to 4 m), portrait; pixels with no depth are clear. + public func heat(_ depth: CVPixelBuffer) -> CGImage? { + let d = CIImage(cvPixelBuffer: depth).oriented(.right) + // depth (metres) → 0...1, near = 1 + let norm = CIFilter.colorMatrix() + norm.inputImage = d + let k: CGFloat = -1 / 3.7 + norm.rVector = CIVector(x: k, y: 0, z: 0, w: 0) + norm.gVector = CIVector(x: k, y: 0, z: 0, w: 0) + norm.bVector = CIVector(x: k, y: 0, z: 0, w: 0) + norm.biasVector = CIVector(x: 1 + 0.3 / 3.7, y: 1 + 0.3 / 3.7, z: 1 + 0.3 / 3.7, w: 0) + norm.aVector = CIVector(x: 0, y: 0, z: 0, w: 1) + let colours = CIFilter.falseColor() + colours.inputImage = norm.outputImage?.clampedToExtent().cropped(to: d.extent) + colours.color0 = CIColor(red: 0.16, green: 0.28, blue: 0.49) // far: fb-blue-dark + colours.color1 = CIColor(red: 0.95, green: 0.45, blue: 0.2) // near: warm + guard let out = colours.outputImage else { return nil } + return context.createCGImage(out, from: d.extent) + } + + private func scaled(_ image: CIImage, longEdge: CGFloat) -> CIImage { + let s = longEdge / max(image.extent.width, image.extent.height) + guard s < 1 else { return image } + return image.transformed(by: CGAffineTransform(scaleX: s, y: s)) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Brand.swift b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Brand.swift index d02b078..6de032b 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Brand.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Brand.swift @@ -20,8 +20,10 @@ public struct BrandHeader: View { Spacer(minLength: 0) } .accessibilityElement(children: .combine) + .accessibilityIdentifier("brand-header") Text(title).font(.largeTitle.weight(.bold)).foregroundStyle(Tokens.ink) .accessibilityAddTraits(.isHeader) + .accessibilityIdentifier("screen-title") } .padding(.top, 8) .padding(.bottom, 4) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Components.swift b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Components.swift index e219c22..d71e073 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Components.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Components.swift @@ -94,6 +94,7 @@ public struct SolidPillStyle: ButtonStyle { .opacity(configuration.isPressed ? 0.75 : 1) .scaleEffect(configuration.isPressed && !reduceMotion ? 0.97 : 1) .animation(MotionTokens.animation(MotionTokens.press, reduceMotion: reduceMotion), value: configuration.isPressed) + .onChange(of: configuration.isPressed) { _, down in if down { TapFeedback.play() } } .glassRim(.capsule, .regular.interactive()) } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Feedback.swift b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Feedback.swift new file mode 100644 index 0000000..c0069c0 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Feedback.swift @@ -0,0 +1,52 @@ +import RecallDomain +import SwiftUI +import UIKit + +/// The soft tap (sound and haptic) every press gives. The app connects it to its sound at launch (TapSound); the +/// shared button styles call it, so a new screen gets it by using them. +@MainActor +public enum TapFeedback { + public static var play: () -> Void = {} +} + +/// A plain button (rows, cards) that still gives the soft tap when pressed. +public struct PlainTapStyle: ButtonStyle { + public init() {} + public func makeBody(configuration: Configuration) -> some View { + configuration.label + .opacity(configuration.isPressed ? 0.7 : 1) + .onChange(of: configuration.isPressed) { _, down in if down { TapFeedback.play() } } + } +} + +extension ButtonStyle where Self == PlainTapStyle { + public static var plainTap: PlainTapStyle { PlainTapStyle() } +} + +extension View { + /// Keeps a text field within the Mac's limit, without control characters (or line breaks, unless `multiline`). + /// In a one-line field that grows with large text (`axis: .vertical`), Return types a line break instead of + /// ending the edit: the break is dropped and the keyboard goes away, as Return does in any one-line field. + public func limited(_ text: Binding, to limit: Int, multiline: Bool = false) -> some View { + onChange(of: text.wrappedValue) { _, typed in + let kept = TextInput.clean(typed, limit: limit, multiline: multiline) + if kept != typed { text.wrappedValue = kept } + if TextInput.pressedReturn(typed, multiline: multiline) { Keyboard.hide() } + } + } + + /// The keyboard hides when the list or form is scrolled. A tap outside a field hides it app-wide + /// (KeyboardDismissal), and Return hides it in one-line fields (`limited`), which VoiceOver users reach easily. + /// (No Done bar above the keyboard: SwiftUI keeps its invisible node on screen, which the accessibility audit flags.) + public func keyboardDismissable() -> some View { + scrollDismissesKeyboard(.interactively) + } +} + +/// Hides the keyboard, whichever field has it. +@MainActor +public enum Keyboard { + public static func hide() { + UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Glass.swift b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Glass.swift index 402c728..6259f1d 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Glass.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Glass.swift @@ -150,7 +150,10 @@ public struct FilterChip: View { } public var body: some View { - Button(action: action) { + Button { + TapFeedback.play() + action() + } label: { Text(title) .font(.subheadline.weight(.semibold)) .foregroundStyle(selected ? Tokens.onAccent : Tokens.ink) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Tokens.swift b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Tokens.swift index 6d5d278..c3f8c4d 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDesign/Tokens.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDesign/Tokens.swift @@ -56,6 +56,8 @@ public enum Tokens { // spacing: 16 pt gutters, 12 between cards, 24 between sections public static let gutter: CGFloat = 16 + public static let stack: CGFloat = 12 // between cards and blocks on a screen + public static let sectionGap: CGFloat = 12 // extra space above a section title (12 + 12 = 24 between sections) public static let cardRadius: CGFloat = 20 public static let cardPadding: CGFloat = 16 diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/CaptureMetrics.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/CaptureMetrics.swift index de53912..eb70b27 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/CaptureMetrics.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/CaptureMetrics.swift @@ -16,6 +16,13 @@ public struct CaptureMetrics: Codable, Sendable, Equatable { public var mapStatus: MapStatus? public var loopClosed: Bool? public var compassSpreadDeg: Double? + /// How long the scan took, the LiDAR samples taken, and how each step went (newer scans only). + public var durationS: Double? + public var points: Int? + public var steps: [ScanGuide.StepRecord]? + /// Column-and-height zones the LiDAR saw, of those in the columns the scan reached. + public var zonesSeen: Int? + public var zonesTotal: Int? public var grade: ScanGrade enum CodingKeys: String, CodingKey { @@ -25,12 +32,16 @@ public struct CaptureMetrics: Codable, Sendable, Equatable { case areaWalkedM2 = "area_walked_m2", meanLightLux = "mean_light_lux", grade case wallsCovered = "walls_covered", floorSeenM2 = "floor_seen_m2", mapStatus = "map_status" case loopClosed = "loop_closed", compassSpreadDeg = "compass_spread_deg" + case durationS = "duration_s", points, steps, zonesSeen = "zones_seen", zonesTotal = "zones_total" } public init(directionsCovered: Int, keyframesKept: Int, keyframesDroppedBlurry: Int, trackingNormalRatio: Double, meanSpeedMps: Double, peakSpeedMps: Double, slowDownCues: Int, areaWalkedM2: Double, meanLightLux: Double, wallsCovered: Int? = nil, floorSeenM2: Double? = nil, mapStatus: MapStatus? = nil, - loopClosed: Bool? = nil, compassSpreadDeg: Double? = nil) { + loopClosed: Bool? = nil, compassSpreadDeg: Double? = nil, durationS: Double? = nil, points: Int? = nil, + steps: [ScanGuide.StepRecord]? = nil, zonesSeen: Int? = nil, zonesTotal: Int? = nil) { + self.durationS = durationS; self.points = points; self.steps = steps + self.zonesSeen = zonesSeen; self.zonesTotal = zonesTotal self.directionsCovered = directionsCovered; self.keyframesKept = keyframesKept self.keyframesDroppedBlurry = keyframesDroppedBlurry; self.trackingNormalRatio = trackingNormalRatio self.meanSpeedMps = meanSpeedMps; self.peakSpeedMps = peakSpeedMps; self.slowDownCues = slowDownCues diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/PointTrail.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/PointTrail.swift new file mode 100644 index 0000000..0166734 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/PointTrail.swift @@ -0,0 +1,31 @@ +import Foundation + +/// The LiDAR points captured so far, for painting on the camera while scanning: newest kept, the total capped so +/// drawing stays smooth (the oldest are thinned, every other one, when it's full). +public struct PointTrail: Sendable { + public let cap: Int + public private(set) var points: [SIMD3] = [] + /// every point ever added, including thinned ones: the "Points" readout + public private(set) var total = 0 + + public init(cap: Int = 30_000) { self.cap = cap } + + public mutating func add(_ new: [SIMD3]) { + total += new.count + points.append(contentsOf: new) + if points.count > cap { + let old = points.count - cap / 2 // thin the older part, keep the newest half whole + let thinned = points[.. cap { points.removeFirst(points.count - cap) } + } + } +} + +/// Walking speed for the scan readout, and whether it's too fast for sharp photos. +public enum ScanSpeed { + /// metres per second above which the readout turns amber (the speed coach's own limit) + public static let tooFast = SpeedCoach.Limits().maxSpeed + public static func isTooFast(_ metresPerSecond: Double) -> Bool { metresPerSecond > tooFast } + public static func text(_ metresPerSecond: Double) -> String { String(format: "%.1f m/s", metresPerSecond) } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift index eb28099..d0eaa56 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanSignals.swift @@ -130,6 +130,35 @@ public struct ScanGuide: Sendable { } } + /// Why the step matters, in one sentence: shown before the scan and on the step card. + public var why: String { + switch self { + case .recognise: "This scan then shares the last one's map, so Recall can tell what moved." + case .start: "The door is where the Watch's directions start, and a steady start lets tracking lock on." + case .turnAround: "One full turn shows the whole room from one spot, so nothing is missed between views." + case .alongWalls: "About 1 metre is where the LiDAR and camera see best: closer, the edges are missed and photos blur; further, small things are too small to find." + case .sweepDown: "Your things are on tables, shelves and the floor; this is where SAM 3.1 looks for them." + case .returnToStart: "Ending where you began closes the loop, so the room's shape lines up and doesn't drift." + case .finish: "The room map lets your iPhone recognise the room later, to guide you there." + } + } + + /// How you know the step is done (it also moves on after a while, so nobody gets stuck). + public var doneWhen: String { + switch self { + case .recognise: "Done when the room is recognised, or after 25 seconds." + case .start: "Done when you have held still for a few seconds." + case .turnAround: "Done when 11 of 12 directions are seen." + case .alongWalls: "Done when walls are seen on 9 of 12 sides." + case .sweepDown: "Done when enough floor and table tops are seen." + case .returnToStart: "Done when you are back within 1 metre of the start." + case .finish: "Done when the room map says ready; then press Stop." + } + } + + /// What to do, as shown on the step card: the spoken line. + public var instruction: String { spoken } + public var spoken: String { switch self { case .recognise: "Scanning again. Point the phone slowly around the room so I can recognise it." @@ -160,7 +189,28 @@ public struct ScanGuide: Sendable { } } + /// One step as it went: how long, and whether it was done or moved on after its time ran out. + public struct StepRecord: Sendable, Equatable, Codable { + public let step: Int // Step.rawValue + public let seconds: Double + public let completed: Bool + public let detail: String + public init(step: Int, seconds: Double, completed: Bool, detail: String) { + self.step = step; self.seconds = seconds; self.completed = completed; self.detail = detail + } + public var title: String { Step(rawValue: step)?.title ?? "Step" } + } + + /// Where a step stands right now, for the card's meter: "Wall sides 5 of 9". + public struct StepProgress: Sendable, Equatable { + public let value: Double + public let target: Double + public let text: String + public var fraction: Double { target > 0 ? min(1, value / target) : 0 } + } + public private(set) var step: Step + public private(set) var records: [StepRecord] = [] public private(set) var loopClosed = false public private(set) var mapReady = false private var stepStarted = 0.0 @@ -180,24 +230,33 @@ public struct ScanGuide: Sendable { /// Steps the scan on; returns a line to say when something changed (a new step, or the map becoming ready). public mutating func update(_ p: Progress) -> String? { let inStep = p.elapsed - stepStarted - let done: Bool = switch step { - case .recognise: false // ended by endRecognise + let met: Bool = switch step { + case .recognise, .finish: false // ended by endRecognise / by Stop case .start: inStep >= 4 - case .turnAround: p.directions >= 11 || inStep > 60 - case .alongWalls: p.walls >= 9 || inStep > 120 - case .sweepDown: p.floorSeenM2 >= max(2.5, 0.5 * p.areaWalkedM2) || inStep > 90 - case .returnToStart: (p.walkedM > 3 && p.fromStartM < 1.0) || inStep > 60 - case .finish: false - } - if step == .returnToStart, done, p.fromStartM < 1.0 { loopClosed = true } - if done, let next = Step(rawValue: step.rawValue + 1) { + case .turnAround: p.directions >= 11 + case .alongWalls: p.walls >= 9 + case .sweepDown: p.floorSeenM2 >= Self.floorTarget(p) + case .returnToStart: p.walkedM > 3 && p.fromStartM < 1.0 + } + let timedOut: Bool = switch step { + case .turnAround: inStep > 60 + case .alongWalls: inStep > 120 + case .sweepDown: inStep > 90 + case .returnToStart: inStep > 60 + default: false + } + if step == .returnToStart, met { loopClosed = true } + if met || timedOut, let next = Step(rawValue: step.rawValue + 1) { + let now = Self.progress(of: step, p) + records.append(StepRecord(step: step.rawValue, seconds: inStep.rounded(), completed: met, detail: now.text)) + let missed = met || step == .start ? "" : "Moving on: \(now.text.prefix(1).lowercased() + now.text.dropFirst()). " step = next stepStarted = p.elapsed if next == .finish, p.map == .mapped { mapReady = true - return "Room map ready. Double tap Stop." + return missed + "Room map ready. Double tap Stop." } - return next.spoken + return missed + next.spoken } if step == .finish, !mapReady, p.map == .mapped { mapReady = true @@ -206,3 +265,85 @@ public struct ScanGuide: Sendable { return nil } } + + +public extension ScanGuide { + /// Floor and table tops to see in the sweep: 2.5 m², or half the area walked in a big room. + static func floorTarget(_ p: Progress) -> Double { max(2.5, 0.5 * p.areaWalkedM2) } + + /// The meter for a step: what's measured against what's needed, in words. + static func progress(of step: Step, _ p: Progress) -> StepProgress { + switch step { + case .recognise: return StepProgress(value: 0, target: 1, text: "Looking for the room") + case .start: return StepProgress(value: min(p.elapsed, 4), target: 4, text: "Hold still") + case .turnAround: + return StepProgress(value: Double(p.directions), target: 11, text: "Directions seen \(min(p.directions, 12)) of 11") + case .alongWalls: + return StepProgress(value: Double(p.walls), target: 9, text: "Wall sides seen \(min(p.walls, 12)) of 9") + case .sweepDown: + let target = floorTarget(p) + return StepProgress(value: p.floorSeenM2, target: target, + text: String(format: "Floor and tops seen %.1f of %.1f m²", p.floorSeenM2, target)) + case .returnToStart: + // full when within a metre of the start; the meter fills as you come back + return StepProgress(value: max(0, 4 - p.fromStartM), target: 3, + text: p.fromStartM < 1 ? "Back at the start" : String(format: "%.1f m from the start", p.fromStartM)) + case .finish: + return StepProgress(value: p.map == .mapped ? 1 : p.map == .extending ? 0.6 : 0.3, target: 1, + text: p.map.spoken) + } + } + + /// The walls step's distance check: about 1 m from the wall is right. + enum WallDistance: Equatable { case good, tooClose, tooFar } + static func wallDistance(_ metres: Double) -> WallDistance { + metres < 0.7 ? .tooClose : metres > 1.6 ? .tooFar : .good + } +} + +/// How high to hold the phone in each step, measured (camera height above the floor the LiDAR found), so nobody is +/// asked their own height: chest height for looking round and along the walls, waist height to sweep tables and floor. +public extension ScanGuide.Step { + var phoneHeight: ClosedRange { + switch self { + case .sweepDown: 0.8...1.1 + default: 1.1...1.5 + } + } + + /// "Chest height, 1.1 to 1.5 m" + var heightLabel: String { + let r = phoneHeight + let name = self == .sweepDown ? "Waist height" : "Chest height" + return String(format: "%@, %.1f to %.1f m", name, r.lowerBound, r.upperBound) + } +} + +/// Says "raise" or "lower" only after the phone has been out of the step's band for a while, and not too often. +public struct HeightCoach: Sendable { + public enum Check: Equatable { case good, tooLow, tooHigh } + + public var after: Double = 3 // seconds out of the band before saying anything + public var gap: Double = 12 // seconds between reminders + private var outSince: Double? + private var lastSaid = -Double.infinity + + public init() {} + + public static func check(_ height: Double, for step: ScanGuide.Step) -> Check { + let r = step.phoneHeight + return height < r.lowerBound ? .tooLow : height > r.upperBound ? .tooHigh : .good + } + + /// The line to say now, if any. `height` nil: the floor isn't measured yet, so nothing to say. + public mutating func update(t: Double, height: Double?, step: ScanGuide.Step) -> String? { + guard let height, step != .recognise else { outSince = nil; return nil } + let c = Self.check(height, for: step) + guard c != .good else { outSince = nil; return nil } + outSince = outSince ?? t + guard t - outSince! >= after, t - lastSaid >= gap else { return nil } + lastSaid = t + let place = step == .sweepDown ? "waist height" : "chest height" + return c == .tooLow ? "Raise the phone to \(place)." : "Lower the phone to \(place)." + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift new file mode 100644 index 0000000..beb419e --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ScanTips.swift @@ -0,0 +1,42 @@ +import Foundation + +/// "Next time": up to three concrete things to do better, from what the scan measured. Nothing for a scan that went well. +public enum ScanTips { + public static func make(_ m: CaptureMetrics) -> [String] { + var tips: [String] = [] + if m.peakSpeedMps > SpeedCoach.Limits().maxSpeed + 0.2 { + tips.append(String(format: "You walked at up to %.1f metres a second. Stay under %.1f: slower means sharper photos.", + m.peakSpeedMps, SpeedCoach.Limits().maxSpeed)) + } + if let walls = m.wallsCovered, walls < 9 { + tips.append("Walls were seen on \(walls) of 12 sides. Walk the whole way round, about a metre from each wall.") + } + if let seen = m.zonesSeen, let total = m.zonesTotal, total > 0, Double(seen) / Double(total) < 0.6 { + tips.append("Height zones seen \(seen) of \(total). Point the phone at knee height and up at shelves too, not only straight ahead.") + } + if m.directionsCovered < 10 { + tips.append("\(m.directionsCovered) of 12 directions were seen. Turn slowly all the way round at the start.") + } + if CaptureMetrics.blurryShare(kept: m.keyframesKept, dropped: m.keyframesDroppedBlurry) >= 0.15 { + tips.append("Some photos were blurry. Hold the phone steady with both hands.") + } + if m.meanLightLux > 0, m.meanLightLux < 150 { + tips.append("It was dim. Turn on the lights before scanning.") + } + if m.loopClosed == false { + tips.append("End where you started, by the door: it lines the room up, and the Watch starts its walk there.") + } + if let spread = m.compassSpreadDeg, spread > 25 { + tips.append("The compass wandered. Keep away from big metal things and speakers when you start.") + } + if m.mapStatus.map({ $0 != .mapped }) == true { + tips.append("The room map wasn't ready. Keep moving slowly until it says ready, then press Stop.") + } + return Array(tips.prefix(3)) + } + + /// The light, in a word, for the report's tile. + public static func light(_ lux: Double) -> String { + lux <= 0 ? "Unknown" : lux < 150 ? "Dim" : lux < 1500 ? "OK" : "Bright" + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ZoneCoverage.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ZoneCoverage.swift new file mode 100644 index 0000000..892f1a1 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Capture/ZoneCoverage.swift @@ -0,0 +1,48 @@ +import Foundation + +/// The room cut into zones while scanning: 1 m columns on the floor, each split into Recall's five height bands +/// (floor, knee, waist, chest, head; the same bands answers use). A zone counts as seen once LiDAR points land in it, +/// so the scan shows which parts of the room, at which heights, nobody has looked at yet. +public struct ZoneCoverage: Sendable, Equatable { + /// Band edges above the floor, metres: floor < 0.3 ≤ knee < 0.7 ≤ waist < 1.1 ≤ chest < 1.5 ≤ head < 2.2. + public static let edges: [Float] = [0, 0.3, 0.7, 1.1, 1.5, 2.2] + public static let bandNames = ["floor", "knee", "waist", "chest", "head"] + public static let column: Float = 1.0 + + public struct Column: Hashable, Sendable { + public let x: Int32 + public let z: Int32 + } + + /// For each column: the bands seen in it (0 = floor ... 4 = head). + public private(set) var seen: [Column: Set] = [:] + + public init() {} + + /// LiDAR points (ARKit world) with the floor's height; points above 2.2 m or below the floor are left out. + public mutating func add(_ points: [SIMD3], floorY: Float) { + for p in points { + let h = p.y - floorY + guard h >= -0.05, h < Self.edges.last!, let band = Self.band(Double(max(h, 0))) else { continue } + let c = Column(x: Int32((p.x / Self.column).rounded(.down)), z: Int32((p.z / Self.column).rounded(.down))) + seen[c, default: []].insert(band) + } + } + + public static func band(_ height: Double) -> Int? { + for i in 0..<(edges.count - 1) where height >= Double(edges[i]) && height < Double(edges[i + 1]) { return i } + return nil + } + + /// Zones seen, of all zones in the columns the scan reached (a column counts once anything in it was seen). + public var zonesSeen: Int { seen.values.reduce(0) { $0 + $1.count } } + public var zonesTotal: Int { seen.count * Self.bandNames.count } + + /// The height band seen least, across the columns: what to point at next ("knee height"). + public var weakestBand: String? { + guard !seen.isEmpty else { return nil } + let counts = (0.. Double { + var d = (bearing - heading).truncatingRemainder(dividingBy: 360) + if d > 180 { d -= 360 } + if d < -180 { d += 360 } + return d + } + + public mutating func update(heading: Double, metresWalked: Double) -> Cue { + guard leg < legs.count else { return .arrived } + let target = legs[leg] + let walked = metresWalked - legStart + if walked >= target.metres { + if walked > target.metres + overshootMetres { return .overshot } + leg += 1 + legStart = metresWalked + facing = false + return leg == legs.count ? .arrived : .nextLeg(leg) + } + let off = Self.offset(heading: heading, bearing: target.bearing) + if facing ? abs(off) > turnWhenOff : abs(off) > facingWithin { + facing = false + return .turn(off > 0 ? .right : .left, degrees: abs(off)) + } + facing = true + return .onCourse(metresLeft: max(0, target.metres - walked)) + } + + /// "Turn right" / "Turn left" / "Straight on", then the distance: what the watch says for a leg. + public static func say(_ cue: Cue, leg: CompassLeg?) -> String { + switch cue { + case .turn(let side, let degrees): + return degrees > 135 ? "Turn around" : side == .right ? "Turn right" : "Turn left" + case .onCourse(let left): return "Straight on, \(Self.metres(left))" + case .nextLeg: return leg.map { "Next, \(Self.metres($0.metres))" } ?? "Next" + case .overshot: return "Stop. You've gone past it" + case .arrived: return "You're there" + } + } + + static func metres(_ m: Double) -> String { + let r = max(1, Int(m.rounded())) + return r == 1 ? "1 metre" : "\(r) metres" + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift new file mode 100644 index 0000000..461248b --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ArLabel.swift @@ -0,0 +1,57 @@ +import Foundation + +/// One thing Recall found, placed in its room's saved AR map: what the label over the camera says, and its colour. +public struct ArLabel: Sendable, Equatable, Identifiable { + /// green: Recall tells you about it; amber: it asks you to check; red: seen once, so it waits for you. + public enum Mark: Sendable, Equatable { case told, check, seenOnce } + + public let id: String + public let title: String + public let place: String + public let confidence: Double + public let photos: Int? + public let trusted: Bool + public let needsReview: Bool + /// in the ARKit frame of the scan whose room map the phone relocalises in + public let position: SIMD3 + + public init(id: String, title: String, place: String, confidence: Double, photos: Int?, trusted: Bool, + needsReview: Bool, position: SIMD3) { + self.id = id; self.title = title; self.place = place; self.confidence = confidence; self.photos = photos + self.trusted = trusted; self.needsReview = needsReview; self.position = position + } + + public var mark: Mark { !trusted ? .seenOnce : needsReview ? .check : .told } + + /// The tag's first line: "GREY BAG". + public var headline: String { title.uppercased() } + + /// The tag's second line: "88% · 5 photos". + public var detail: String { + let sure = "\(Int((confidence * 100).rounded()))%" + guard let photos else { return sure } + return "\(sure) · \(photos == 1 ? "1 photo" : "\(photos) photos")" + } + + /// What VoiceOver says: "Grey bag, near the desk, at knee height. 88 percent, seen in 5 photos. Recall tells you." + public var spoken: String { + let seen = photos.map { $0 == 1 ? ", seen in 1 photo" : ", seen in \($0) photos" } ?? "" + let verdict = switch mark { + case .told: "Recall tells you" + case .check: "Check it" + case .seenOnce: "Waits for you to confirm" + } + return "\(title.prefix(1).uppercased() + title.dropFirst()), \(place). \(Int((confidence * 100).rounded())) percent\(seen). \(verdict)." + } +} + +/// The labels for one room and the scan whose room map they are placed in. +public struct ArLabels: Sendable, Equatable { + public let scanId: String + public let room: String + public let floorY: Double? + public let labels: [ArLabel] + public init(scanId: String, room: String, floorY: Double?, labels: [ArLabel]) { + self.scanId = scanId; self.room = room; self.floorY = floorY; self.labels = labels + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift index fac3dc4..6236884 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/AuditEntry.swift @@ -44,6 +44,32 @@ public struct AuditEntry: Sendable, Equatable, Identifiable { return "You erased a scan of \(detail["room"] ?? "a room") from the Mac" case "auth.refused": return "Refused: \(detail["why"] ?? "no valid key") (\(detail["path"] ?? "a request"))" + case "trust.changed": + return "You set how sure Recall must be to \(detail["level"] ?? "a new level")" + case "scan.created": + return "You started uploading a scan of \(detail["room"] ?? "a room")" + case "scan.uploaded": + return refused ? "A scan upload failed its check: \(detail["why"] ?? "damaged")" + : "Your scan of \(detail["room"] ?? "a room") arrived; the Mac is processing it" + case "scan.failed": + return "The Mac couldn't process a scan of \(detail["room"] ?? "a room")" + case "scan.ingested": + return "The Mac remembered a scan of \(detail["room"] ?? "a room")" + case "object.corrected": + return "You corrected \(detail["label"] ?? "a thing")" + case "object.forgotten": + return "You said \(detail["label"] ?? "a thing") isn't there" + case "arrival.verified": + return "The camera checked you'd reached a thing" + case "guide.answered": + return refused ? "Your question couldn't reach the Guide's model" : "Your question went to the Guide's model" + case "voice.transcribed": + return refused ? "Your voice couldn't reach Muse Voice Transcribe" + : "Your voice went to Muse Voice Transcribe (not kept)" + case "rate.limited": + return "Slowed down: too many calls to \(detail["path"] ?? "a route") in a minute" + case "audit.pruned": + return "The Mac dropped \(detail["dropped"] ?? "some") events past the keeping period" default: return action } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Scan.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Scan.swift index fb267b9..5bbe9c2 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Scan.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Scan.swift @@ -52,6 +52,11 @@ public struct Scan: Identifiable, Sendable, Equatable { public var changes: [ScanChange] = [] /// Scanned before but not recognised: the Mac started this room's memory afresh. public var fresh = false + /// Waiting to be processed with nobody to process it: the Mac's scan worker isn't running. + public var workerStopped = false + + /// What to do about a stopped worker, shown and copied on the scan. + public static let workerStoppedMessage = "Your Mac's scan worker isn't running, so this scan is waiting. On the Mac, run make worker (or make live)." public init(id: String, room: String, createdAt: Date, grade: ScanGrade, directions: Int, frames: Int, state: UploadState = .notUploaded, alignedTo: String? = nil) { @@ -102,9 +107,13 @@ public struct ScanReport: Sendable, Equatable { public let detail: String? public let changes: [ScanChange] public let fresh: Bool + /// While processing: whether the Mac's scan worker is running (nil: the Mac didn't say). + public let workerAlive: Bool? - public init(_ status: ScanStatus, detail: String? = nil, changes: [ScanChange] = [], fresh: Bool = false) { + public init(_ status: ScanStatus, detail: String? = nil, changes: [ScanChange] = [], fresh: Bool = false, + workerAlive: Bool? = nil) { self.status = status; self.detail = detail; self.changes = changes; self.fresh = fresh + self.workerAlive = workerAlive } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanChange.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanChange.swift index fed92da..b41c36c 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanChange.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanChange.swift @@ -50,6 +50,18 @@ public struct ScanChange: Sendable, Equatable, Codable, Identifiable { public enum ChangeSummary { public static let most = 3 // per kind; the rest are counted ("and 2 more") + /// A rescan that didn't line up with the last one: a complete new map, nothing lost, but no comparison. Said as + /// what Recall did and what to do next time, never as a failure. + public static func freshTitle(room: String) -> String { "A fresh map of \(room)" } + public static let freshBody = "This walk didn't line up with your last scan, so Recall can't say what moved this " + + "time. Everything it found is mapped, and your earlier scan is kept below." + public static let freshTip = "To see what moved next time, start Scan again where your last scan began." + + /// Spoken: the same, shorter. + public static func sayFresh(room: String) -> String { + "A fresh map of \(room). This walk didn't line up with your last scan, so I can't say what moved this time." + } + public static func say(room: String, changes: [ScanChange]) -> String { guard !changes.isEmpty else { return "\(room) is ready. Nothing has moved since the last scan." } let n = changes.count == 1 ? "1 change" : "\(changes.count) changes" diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanHistory.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanHistory.swift new file mode 100644 index 0000000..b3349e3 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ScanHistory.swift @@ -0,0 +1,36 @@ +import Foundation + +/// A room's newest processed scan, which holds its memory now, and the processed scans before it: its history. +public struct RoomScans: Identifiable, Sendable, Equatable { + public let latest: Scan + public internal(set) var earlier: [Scan] // newest first + + public var id: String { latest.id } + + /// The scan that came right after `scan` in this room ("Replaced by the scan on 1 Nov"). + public func successor(of scan: Scan) -> Scan? { + let all = [latest] + earlier + guard let i = all.firstIndex(where: { $0.id == scan.id }), i > 0 else { return nil } + return all[i - 1] + } +} + +public enum ScanHistory { + /// One card per room, newest first. Only processed scans fold under a newer processed scan of the same room; + /// one still uploading, processing or failed keeps its own card, so its progress and Retry stay in sight. + /// Room names match whatever their case and spacing ("Living room" and "living room "). + public static func byRoom(_ scans: [Scan]) -> [RoomScans] { + var out: [RoomScans] = [] + var current: [String: Int] = [:] // room → its card in `out` + for scan in scans.sorted(by: { $0.createdAt > $1.createdAt }) { + let room = scan.room.trimmingCharacters(in: .whitespaces).lowercased() + if scan.isProcessed, let i = current[room] { + out[i].earlier.append(scan) + continue + } + if scan.isProcessed { current[room] = out.count } + out.append(RoomScans(latest: scan, earlier: [])) + } + return out + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Thing.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Thing.swift index 4533729..bb54c63 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Thing.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/Thing.swift @@ -21,16 +21,22 @@ public struct Thing: Identifiable, Sendable, Equatable { public let confidence: Double public let confirmed: Bool public let needsReview: Bool + /// How many of the scan's photos showed it (nil for sightings kept before the Mac counted them). + public let photos: Int? + /// Seen in enough photos for the owner's level, or confirmed: Find and the Guide use only trusted things. + public let trusted: Bool public let seenAt: Date public init(id: String, label: String, name: String? = nil, color: String? = nil, measuredColor: String? = nil, heightM: Double, userHeightM: Double? = nil, heightBand: String, landmark: String? = nil, relation: String? = nil, room: String? = nil, sizeCm: [Int]? = nil, note: String? = nil, confidence: Double, - confirmed: Bool = false, needsReview: Bool = false, seenAt: Date) { + confirmed: Bool = false, needsReview: Bool = false, photos: Int? = nil, trusted: Bool = true, + seenAt: Date) { self.id = id; self.label = label; self.name = name; self.color = color; self.measuredColor = measuredColor self.heightM = heightM; self.userHeightM = userHeightM; self.heightBand = heightBand; self.landmark = landmark self.relation = relation; self.room = room; self.sizeCm = sizeCm; self.note = note; self.confidence = confidence - self.confirmed = confirmed; self.needsReview = needsReview; self.seenAt = seenAt + self.confirmed = confirmed; self.needsReview = needsReview; self.photos = photos; self.trusted = trusted + self.seenAt = seenAt } /// What VoiceOver and the lists say: "grey bag", or the user's own name for it. @@ -50,6 +56,16 @@ public struct Thing: Identifiable, Sendable, Equatable { return needsReview ? "Unsure, \(sure)" : sure.prefix(1).uppercased() + sure.dropFirst() } + /// The evidence, in words: "Seen in 1 photo, 55 percent sure". + public var evidence: String { + let sure = "\(Int((confidence * 100).rounded())) percent sure" + guard let photos else { return sure.prefix(1).uppercased() + sure.dropFirst() } + return "Seen in \(photos == 1 ? "1 photo" : "\(photos) photos"), \(sure)" + } + + /// Found's short mark for a thing to check: "Seen once" while it waits for a second look, else "Check". + public var checkMark: String { !trusted && photos == 1 ? "Seen once" : "Check" } + /// Unsure things first, then by name: the order of every list. public static func listOrder(_ a: Thing, _ b: Thing) -> Bool { (a.needsReview ? 0 : 1, a.title) < (b.needsReview ? 0 : 1, b.title) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ThingGroups.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ThingGroups.swift index 89ee2e7..0c62bd0 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ThingGroups.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/ThingGroups.swift @@ -55,6 +55,14 @@ public enum ThingGroups { } } + /// The rooms to show maps for: rooms with things (most things first), then rooms scanned that have none yet + /// (in the order given). "Somewhere in the home" isn't a room, so it has no map. + public static func rooms(_ things: [Thing], scanned: [String] = []) -> [String] { + let withThings = byPlace(things).map(\.title).filter { $0 != somewhere } + var seen = Set(withThings) + return withThings + scanned.filter { seen.insert($0).inserted } + } + /// Floor to head height, the five bands in that order. public static func byHeight(_ things: [Thing]) -> [ThingGroup] { let bands = Dictionary(grouping: things, by: \.heightBand) diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/TrustLevel.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/TrustLevel.swift new file mode 100644 index 0000000..763c7a6 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Memory/TrustLevel.swift @@ -0,0 +1,27 @@ +import Foundation + +/// How sure Recall must be before it tells you about a thing, by the photos it was seen in. One photo is one angle, +/// where a book cover can pass for a wallet; seen again from another place, it is very likely there. Kept on the Mac, +/// so the voice Guide, Find, room maps and robots all follow it. A thing you confirmed always counts. +public enum TrustLevel: String, CaseIterable, Identifiable, Sendable { + case careful, balanced, everything + + public var id: String { rawValue } + + public var title: String { + switch self { + case .careful: "Careful" + case .balanced: "Balanced" + case .everything: "Everything" + } + } + + /// What the level means, said once under the choice. + public var explanation: String { + switch self { + case .careful: "Only things seen in at least 3 photos. Fewer mistakes; some real things wait for you to confirm them." + case .balanced: "Things seen in at least 2 photos. A thing seen once waits in Found until you confirm it." + case .everything: "Every find, even from one photo. More mistakes, like a wallet that is really a book." + } + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift index 3dd2d9d..055cf17 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Ports/Ports.swift @@ -12,6 +12,33 @@ public protocol HomeMemory: Sendable { func forget(home: String, thing: String) async throws } +/// Each room's map, with the things in it, for the owner's own screens (not recorded as a robot export). +/// Everything found in a room, placed in that room's saved AR map (labels over the camera). +public protocol ArLabelsSource: Sendable { + func arLabels(home: String, room: String) async throws -> ArLabels +} + +/// Directions by compass for the Watch, from where the room's scan began (the Mac plans them). +public protocol CompassRoutes: Sendable { + func compassRoute(home: String, thing: String) async throws -> CompassRoute +} + +/// The owner's trust level, kept on the Mac. +public protocol TrustSettings: Sendable { + func trustLevel() async throws -> TrustLevel + func setTrustLevel(_ level: TrustLevel) async throws -> TrustLevel +} + +/// One room's package for a robot, made by the Mac: a .zip of the files a robot loads (Nav2 map, things, room +/// outline, 3D points, README). `room` nil is the newest scan's room. +public protocol RobotPackages: Sendable { + func robotPackage(home: String, room: String?) async throws -> Data +} + +public protocol RoomMaps: Sendable { + func roomMap(home: String, room: String) async throws -> RobotMapPackage +} + /// The Guide agent and the route planner. public protocol GuideService: Sendable { /// `about`: the thing the last answer was about, so "why do you think so?" knows what "so" is. @@ -111,6 +138,10 @@ public protocol Clock: Sendable { @MainActor public protocol SpeechOutput: AnyObject { func say(_ text: String, interrupt: Bool) func stop() + /// Speaking right now (false under VoiceOver: those lines are VoiceOver's, stopped with its two-finger tap). + var isSpeaking: Bool { get } + /// Called when the current speech ends or is stopped (a speaker button turns back from Stop). + var onFinished: (() -> Void)? { get set } } @MainActor public protocol HapticsOutput: AnyObject { diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/RecallError.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/RecallError.swift index 939f834..4409791 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/RecallError.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/RecallError.swift @@ -17,6 +17,8 @@ extension RecallError: LocalizedError { case .setup(let issue): issue.message case .server(let s): s case .status(401): "The device key is wrong. Check it in Settings." + case .status(413): "That was too large for the Mac to accept." + case .status(429): "The Mac is getting a lot of requests. Wait a moment and try again." case .status(404): "The Mac doesn't know that yet." case .status(503): "The Mac is missing a setting. Check RECALL_DEVICE_KEY in its .env." case .status(let c): "The Mac answered with an error (\(c))." diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Connection.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Connection.swift index c74b064..97d2cda 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Connection.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Connection.swift @@ -15,11 +15,33 @@ public struct Connection: Sendable, Equatable { public var url: URL? { let text = address.filter { !$0.isNewline }.trimmingCharacters(in: .whitespaces) guard !text.isEmpty else { return nil } - let full = text.contains("://") ? text : "http://\(text)" + // a bare host is https, except on this network (where the Mac serves plain http): the key never crosses the + // internet unencrypted because a scheme was left out + let full = text.contains("://") ? text : "\(Self.isLocal(text) ? "http" : "https")://\(text)" guard let url = URL(string: full), url.scheme?.hasPrefix("http") == true, url.host != nil else { return nil } return url } + /// localhost, a .local name, a private IPv4 address (10/8, 172.16/12, 192.168/16, 127/8, 169.254/16), or Tailscale + /// (a *.ts.net name or 100.64/10): Tailscale encrypts the traffic itself (WireGuard), so plain http is safe there. + static func isLocal(_ hostAndPort: String) -> Bool { + let host = String(hostAndPort.split(separator: "/").first ?? "").split(separator: ":").first.map(String.init) ?? "" + let h = host.lowercased() + if h == "localhost" || h.hasSuffix(".local") || h.hasSuffix(".ts.net") { return true } + let n = h.split(separator: ".").compactMap { Int($0) } + guard n.count == 4, n.allSatisfy({ (0...255).contains($0) }) else { return false } + return n[0] == 10 || n[0] == 127 || (n[0] == 192 && n[1] == 168) || (n[0] == 172 && (16...31).contains(n[1])) + || (n[0] == 169 && n[1] == 254) || (n[0] == 100 && (64...127).contains(n[1])) + } + + /// The address with most of it hidden, enough to recognise it on a shared screen: the scheme, the host's first + /// three characters and the port ("http://rac••••••:8000"). Empty when nothing is typed. + public var maskedAddress: String { + guard let url, let host = url.host else { return address.isEmpty ? "" : "••••••" } + let port = url.port.map { ":\($0)" } ?? "" + return "\(url.scheme ?? "http")://\(host.prefix(3))••••••\(port)" + } + /// The key without the spaces or newline a paste brings along. public var cleanKey: String { key.trimmingCharacters(in: .whitespacesAndNewlines) } @@ -30,8 +52,9 @@ public struct Connection: Sendable, Equatable { return "\(k.count) characters, ending in \(k.suffix(4).map(String.init).joined(separator: " "))." } + /// The home as the Mac names it: "My Flat" is sent as "my-flat"; nothing typed is "flat". public var homeId: String { - let h = home.trimmingCharacters(in: .whitespacesAndNewlines) + let h = TextInput.homeId(home) return h.isEmpty ? "flat" : h } @@ -59,10 +82,19 @@ public enum SetupIssue: Sendable, Equatable { } /// The Mac's pairing link (make pair shows it as a QR code): recall://pair?url=…&key=…&home=… -public struct PairingLink: Sendable, Equatable { +public struct PairingLink: Sendable, Equatable, Identifiable { public let address: String public let key: String public let home: String? + public var id: String { address + key } + + /// What the person confirms, without showing the key: "Pair with 192.168.1.20? Key: 32 characters, ending in …" + public var summary: String { + let c = Connection(address: address, key: key, home: home ?? "") + var host = address + if let url = c.url, let h = url.host { host = url.port.map { "\(h):\($0)" } ?? h } + return "Pair with the Mac at \(host)? Key: \(c.keySummary)" + } /// Nil when the link isn't a pairing link; throws when it is one but is missing the address or key. public init?(_ link: URL) throws { diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift index 0a7943b..9b5dea1 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/Privacy.swift @@ -59,6 +59,8 @@ public enum PrivacyNote { ("Robot Access", "Off by default. Every robot request is in the activity log."), ("Private mode", "Run the Guide on Muse Glimmer on your Mac (LM Studio), so your questions stay home. Scans " + "and voice still use Meta's online models."), + ("People in view", "Scan when nobody else is in the room. Anyone the camera sees is in the scan photos that go to " + + "SAM 3.1 with the rest; Recall never looks for people or faces, and robots never get photos."), ] public static let theirTerms = "These services process what they receive under their own terms; Recall doesn't " diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/TextInput.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/TextInput.swift new file mode 100644 index 0000000..0d9118a --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Setup/TextInput.swift @@ -0,0 +1,43 @@ +import Foundation + +/// What each text field accepts. The limits are the Mac's (resources/schemas; a test checks them against the +/// bundled openapi.json), so the phone never sends what the Mac would refuse. One-line fields drop line breaks; +/// no field keeps control characters (pasted text can carry them). +public enum TextInput { + public static let question = 300 // AskRequest.question + public static let roomName = 60 // ScanCreate.room + public static let thingName = 40 // ObjectCorrection.name + public static let note = 200 // ObjectCorrection.note + public static let homeId = 64 // the home_id pattern + public static let address = 300 + public static let deviceKey = 128 + public static let search = 60 + + /// The text as the field keeps it: no control characters (line breaks only where `multiline`), at most `limit`. + public static func clean(_ text: String, limit: Int, multiline: Bool = false) -> String { + let kept = text.unicodeScalars.filter { s in + (multiline && s == "\n") || !CharacterSet.controlCharacters.contains(s) + } + return String(String(String.UnicodeScalarView(kept)).prefix(limit)) + } + + /// A line break typed into a one-line field: Return was pressed (fields that grow with large text type it). + public static func pressedReturn(_ typed: String, multiline: Bool) -> Bool { + !multiline && typed.contains(where: \.isNewline) + } + + /// A pasted key longer than the field keeps: say so, rather than silently pairing with a cut key. + public static func keyTooLong(_ text: String) -> Bool { + text.trimmingCharacters(in: .whitespacesAndNewlines).count > deviceKey + } + + /// A home name as the Mac accepts it: lowercase letters, digits and dashes ("My Flat" → "my-flat"). + public static func homeId(_ text: String) -> String { + // "Küche" becomes "kuche", "�ódź" "lodz" and "Квартира" "kvartira", rather than vanishing into the default + // home (ICU's Any-Latin, then Latin-ASCII for letters like � that aren't a base letter plus an accent) + let latin = text.applyingTransform(.toLatin, reverse: false) ?? text + let plain = latin.applyingTransform(StringTransform("Latin-ASCII"), reverse: false) ?? latin + let words = plain.lowercased().split { !($0.isASCII && ($0.isLetter || $0.isNumber)) } + return String(words.joined(separator: "-").prefix(homeId)) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Voice/VoiceChoice.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Voice/VoiceChoice.swift new file mode 100644 index 0000000..8026bb2 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Voice/VoiceChoice.swift @@ -0,0 +1,60 @@ +import Foundation + +/// Which kind of voice the owner wants Recall to speak in (Settings → Voice). +public enum VoicePreference: String, CaseIterable, Sendable { + case automatic, female, male + + public var title: String { + switch self { + case .automatic: "Automatic" + case .female: "Female" + case .male: "Male" + } + } +} + +/// One of the iPhone's own voices, as the speech framework describes it (an OS feature, not a model). +public struct VoiceOption: Sendable, Equatable { + public enum Gender: Sendable { case female, male, unspecified } + + public let id: String + public let name: String + public let language: String // "en-GB" + /// 0 default, 1 enhanced, 2 premium: the downloadable voices sound the most natural + public let quality: Int + public let gender: Gender + /// novelty voices (Bells, Bubbles…) and the owner's Personal Voice are never chosen automatically + public let special: Bool + + public init(id: String, name: String, language: String, quality: Int, gender: Gender, special: Bool = false) { + self.id = id; self.name = name; self.language = language; self.quality = quality + self.gender = gender; self.special = special + } + + public var qualityName: String { ["", "Enhanced", "Premium"][min(max(quality, 0), 2)] } + + /// "Ava (Premium)", "Daniel". + public var title: String { qualityName.isEmpty ? name : "\(name) (\(qualityName))" } +} + +public enum VoiceChoice { + /// The best voice for `language` ("en-GB"): the same language, never a novelty or Personal Voice, premium before + /// enhanced before default, then the same region, then by name. With a gender preference, voices of that gender + /// come first; if none is installed, the best of any gender speaks rather than none. + public static func best(_ voices: [VoiceOption], preference: VoicePreference, language: String) -> VoiceOption? { + let base = language.split(separator: "-").first.map(String.init) ?? language + let usable = voices.filter { !$0.special && $0.language.split(separator: "-").first.map(String.init) == base } + let wanted: VoiceOption.Gender? = switch preference { + case .automatic: nil + case .female: .female + case .male: .male + } + let pool = wanted.map { g in usable.filter { $0.gender == g } }.flatMap { $0.isEmpty ? nil : $0 } ?? usable + return pool.max { a, b in + if a.quality != b.quality { return a.quality < b.quality } + let (ra, rb) = (a.language == language, b.language == language) + if ra != rb { return !ra } + return a.name > b.name + } + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/PendingReplies.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/PendingReplies.swift new file mode 100644 index 0000000..c4ed4ea --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/PendingReplies.swift @@ -0,0 +1,43 @@ +import Foundation + +/// Replies the Watch is waiting for from the iPhone, by request id. Each one is answered exactly once: by the +/// iPhone's reply, or by `WatchReply.phoneDidNotAnswer` when `PendingReplies.timeout` passes first. Whichever +/// comes second finds nothing to take and is dropped, so a late reply can't speak over a newer question. +public final class PendingReplies: @unchecked Sendable { + /// How long the Watch waits for the iPhone before saying so (a spoken question goes through the Guide: a few + /// seconds normally, never this long). + public static var timeout: Duration { .seconds(25) } + + private var handlers: [String: Handler] = [:] + private let lock = NSLock() + + public init() {} + + public func put(_ id: String, _ handler: Handler) { + lock.withLock { handlers[id] = handler } + } + + /// The handler for `id`, removed so nothing can call it again; nil if it was already answered. + public func take(_ id: String) -> Handler? { + lock.withLock { handlers.removeValue(forKey: id) } + } + + public var count: Int { lock.withLock { handlers.count } } +} + +public extension WatchReply { + /// Limits on what crosses between the Watch and the iPhone: a question of a few seconds, a reply of a few things. + static let maxAudioBytes = 2 * 1024 * 1024 + static let maxReplyBytes = 256 * 1024 + static let maxThings = 100 + + /// The reply with at most `maxThings` things for the dial and a sentence of reasonable length. + var capped: WatchReply { + var r = self + r.things = things.map { Array($0.prefix(Self.maxThings)) } + r.say = say.map { $0.count > 400 ? String($0.prefix(399)) + "…" : $0 } + return r + } + + static let phoneDidNotAnswer = WatchReply(say: "Your iPhone didn't answer. Try again with it nearby.") +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/WatchMessage.swift b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/WatchMessage.swift new file mode 100644 index 0000000..0c54e24 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Sources/RecallDomain/Watch/WatchMessage.swift @@ -0,0 +1,31 @@ +import Foundation + +/// What the Watch asks the iPhone. The Watch holds no key and never talks to the Mac: the phone does, and answers. +public enum WatchRequest: Sendable, Equatable, Codable { + /// the things to pick from with the Digital Crown + case things + /// a picked thing: the way there + case route(thingId: String) + /// a spoken question; the audio travels as a file with this id + case ask(audioFile: String) +} + +/// A thing on the Watch's dial: short enough to say as the crown turns. +public struct WatchThing: Sendable, Equatable, Codable, Identifiable { + public let id: String + public let title: String + public let place: String + public init(id: String, title: String, place: String) { self.id = id; self.title = title; self.place = place } +} + +/// The iPhone's answer to a WatchRequest. +public struct WatchReply: Sendable, Equatable, Codable { + public var things: [WatchThing]? + /// what to say: the Guide's grounded answer, or why it can't help + public var say: String? + public var route: CompassRoute? + + public init(things: [WatchThing]? = nil, say: String? = nil, route: CompassRoute? = nil) { + self.things = things; self.say = say; self.route = route + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift index ea23576..c0906be 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/RecallAPI.swift @@ -13,9 +13,22 @@ public struct RecallAPI: Sendable { let client: Client public init(baseURL: URL, deviceKey: String) { - self.init(baseURL: baseURL, deviceKey: deviceKey, transport: URLSessionTransport()) + self.init(baseURL: baseURL, deviceKey: deviceKey, + transport: URLSessionTransport(configuration: .init(session: Self.session))) } + /// One session for every call: a request that hears nothing for 30 s fails (instead of iOS's 60 s), so a voice + /// question never hangs while someone waits. Uploads use their own background session. + static let session: URLSession = { + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 30 + return URLSession(configuration: config) + }() + + /// The longest error sentence shown or spoken from the Mac, and the largest robot package accepted. + static let maxDetail = 300 + static let maxPackageBytes = 50 * 1024 * 1024 + init(baseURL: URL, deviceKey: String, transport: any ClientTransport) { self.baseURL = baseURL self.deviceKey = deviceKey @@ -32,6 +45,7 @@ public struct RecallAPI: Sendable { func request(_ path: String, method: String, contentType: String, body: Data) -> URLRequest { var req = URLRequest(url: baseURL.appending(path: path)) req.httpMethod = method + req.timeoutInterval = 30 req.setValue("Bearer \(deviceKey)", forHTTPHeaderField: "Authorization") req.setValue(contentType, forHTTPHeaderField: "Content-Type") req.httpBody = body @@ -40,14 +54,20 @@ public struct RecallAPI: Sendable { /// Sends a plain request; non-200 answers become RecallError with the Mac's `detail` when it gives one. func send(_ req: URLRequest) async throws -> [String: Any] { - let (data, response) = try await URLSession.shared.data(for: req) + let (data, response) = try await Self.session.data(for: req) let code = (response as? HTTPURLResponse)?.statusCode ?? 0 let json = (try? JSONSerialization.jsonObject(with: data) as? [String: Any]) ?? [:] guard (200..<300).contains(code) else { - throw (json["detail"] as? String).map { RecallError.server($0) } ?? RecallError.status(code) + throw (json["detail"] as? String).map { RecallError.server(Self.clean($0)) } ?? RecallError.status(code) } return json } + + /// The Mac's sentence as the person will hear it: no control characters, and never a wall of text. + static func clean(_ detail: String) -> String { + let text = String(detail.unicodeScalars.filter { !CharacterSet.controlCharacters.contains($0) }) + return text.count > maxDetail ? String(text.prefix(maxDetail - 1)) + "…" : text + } } // MARK: HomeMemory @@ -58,6 +78,7 @@ extension RecallAPI: HomeMemory { switch r { case .ok(let ok): return try ok.body.json.map(Thing.init) case .notFound: return [] // no scan of this home processed yet + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -74,6 +95,7 @@ extension RecallAPI: HomeMemory { switch r { case .noContent: return case .notFound: throw RecallError.server("The Mac doesn't know this thing any more.") + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -89,6 +111,9 @@ extension RecallAPI: GuideService, Transcriber { let a = try ok.body.json return Answer(text: a.text, thingId: a.objectId, grounded: a.grounded, answerId: a.answerId) case .serviceUnavailable: throw RecallError.server("The Guide isn't available right now. Try again in a moment.") + case .unauthorized: throw RecallError.status(401) + case .contentTooLarge: throw RecallError.status(413) + case .tooManyRequests: throw RecallError.status(429) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -107,6 +132,8 @@ extension RecallAPI: GuideService, Transcriber { target: g.targetArkit.flatMap { $0.count == 3 ? SIMD3($0[0], $0[1], $0[2]) : nil }, bearingDeg: g.bearingDeg, floorY: g.floorYArkit) case .notFound(let e): throw RecallError.server(((try? e.body.json.detail) ?? nil) ?? "The Mac can't plan a route yet.") + case .unauthorized: throw RecallError.status(401) + case .contentTooLarge: throw RecallError.status(413) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -160,6 +187,7 @@ extension RecallAPI: PrivacySource { return PrivacyStatus(storedOn: p.storedOn, sent: p.sent.map { DataFlow(what: $0.what, to: $0.to, when: $0.when, onThisMac: $0.onThisMac ?? false) }, robotAccess: p.robotAccess, auditDays: p.auditDays) + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -181,6 +209,7 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { case .noContent, .notFound: return // gone already is fine case .conflict(let e): throw RecallError.server(try e.body.json.detail ?? "The Mac is still processing this scan.") + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -195,8 +224,9 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { name: c.name, color: c.color, from: c.from, to: c.to) } return ScanReport(ScanStatus(rawValue: s.status.rawValue) ?? .failed, detail: s.detail, changes: changes, - fresh: s.fresh ?? false) + fresh: s.fresh ?? false, workerAlive: s.workerAlive) case .notFound: throw RecallError.server("The Mac doesn't know this scan.") + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -209,13 +239,17 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { case .ok(let ok): return UploadPlan(parts: try ok.body.json.parts.map { .init(number: $0.number, path: $0.url, received: $0.received) }) case .unprocessableContent(let e): throw RecallError.server(try e.body.json.detail ?? "The scan was refused.") + case .unauthorized: throw RecallError.status(401) + case .contentTooLarge: throw RecallError.status(413) case .undocumented(let code, _): throw RecallError.status(code) } } /// The part upload as a plain request, for the background URLSession (which needs a request and a file). + /// The device key only ever goes to this Mac: an absolute part URL is used only when it is on the same scheme, + /// host and port; otherwise just its path is taken, on this Mac. public func partRequest(path: String, sha256: String) -> URLRequest { - var req = URLRequest(url: path.hasPrefix("http") ? URL(string: path)! : baseURL.appending(path: path)) + var req = URLRequest(url: partURL(path)) req.httpMethod = "PUT" req.setValue("Bearer \(deviceKey)", forHTTPHeaderField: "Authorization") req.setValue(sha256, forHTTPHeaderField: "X-Content-SHA256") @@ -223,6 +257,14 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { return req } + func partURL(_ path: String) -> URL { + if let url = URL(string: path), url.host != nil { + let same = url.scheme == baseURL.scheme && url.host == baseURL.host && url.port == baseURL.port + return same ? url : baseURL.appending(path: url.path) + } + return baseURL.appending(path: path) + } + /// The manifest (domain type → the generated type through the same JSON, since both follow the spec). public func putManifest(scan: String, manifest: Manifest) async throws { let json = try Manifest.encoder().encode(manifest) @@ -234,6 +276,8 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { case .ok: return case .notFound: throw RecallError.server("The Mac doesn't know this scan.") case .unprocessableContent(let e): throw RecallError.server(try e.body.json.detail ?? "The manifest was refused.") + case .unauthorized: throw RecallError.status(401) + case .contentTooLarge: throw RecallError.status(413) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -246,6 +290,9 @@ extension RecallAPI: ConnectionCheck, ScanProcessing, ScanServer { return (ScanStatus(rawValue: s.status.rawValue) ?? .failed, s.detail) case .notFound: throw RecallError.server("The Mac doesn't know this scan.") case .conflict(let e): throw RecallError.server(try e.body.json.detail ?? "The scan isn't ready to complete.") + case .unauthorized: throw RecallError.status(401) + case .contentTooLarge: throw RecallError.status(413) + case .tooManyRequests: throw RecallError.status(429) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -260,7 +307,8 @@ extension Thing { heightM: o.position.count == 3 ? o.position[2] : 0, userHeightM: o.corrections?.heightM, heightBand: o.heightBand.rawValue, landmark: o.landmark, relation: o.relation?.rawValue, room: o.room, sizeCm: o.sizeM?.map { Int(($0 * 100).rounded()) }, note: o.corrections?.note, confidence: o.confidence, - confirmed: o.corrections?.confirmed ?? false, needsReview: o.needsReview ?? false, seenAt: o.seenAt) + confirmed: o.corrections?.confirmed ?? false, needsReview: o.needsReview ?? false, photos: o.photos, + trusted: o.trusted ?? true, seenAt: o.seenAt) } } @@ -331,6 +379,7 @@ extension RecallAPI: EvidenceSource { return Evidence(question: e.answer.question, text: e.answer.text, model: e.answer.model, latencyMs: e.answer.latencyMs, tools: e.steps.compactMap(\.tool), sightings: sightings) case .notFound: throw RecallError.server("The Mac no longer has this answer's evidence.") + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } @@ -341,11 +390,109 @@ extension RecallAPI: EvidenceSource { switch r { case .ok(let ok): return try await Data(collecting: ok.body.jpeg, upTo: 5 * 1024 * 1024) case .notFound: throw RecallError.server("No camera view is kept for this sighting.") + case .unauthorized: throw RecallError.status(401) case .undocumented(let code, _): throw RecallError.status(code) } } } +// MARK: ArLabelsSource + +extension RecallAPI: ArLabelsSource { + public func arLabels(home: String, room: String) async throws -> ArLabels { + var req = request("homes/\(home)/ar-labels", method: "GET", contentType: "application/json", body: Data()) + if let url = req.url, var parts = URLComponents(url: url, resolvingAgainstBaseURL: false) { + parts.queryItems = [URLQueryItem(name: "room", value: room)] + req.url = parts.url + } + return try Self.arLabels(try await send(req)) + } + + static func arLabels(_ j: [String: Any]) throws -> ArLabels { + guard let scan = j["scan_id"] as? String, let room = j["room"] as? String, + let rows = j["labels"] as? [[String: Any]] else { + throw RecallError.server("The Mac sent labels this app can't read. Update the app.") + } + let labels: [ArLabel] = rows.compactMap { r in + guard let id = r["object_id"] as? String, let title = r["title"] as? String, let place = r["place"] as? String, + let p = r["position_arkit"] as? [Double], p.count == 3 else { return nil } + return ArLabel(id: id, title: title, place: place, confidence: r["confidence"] as? Double ?? 0, + photos: r["photos"] as? Int, trusted: r["trusted"] as? Bool ?? true, + needsReview: r["needs_review"] as? Bool ?? false, position: SIMD3(p[0], p[1], p[2])) + } + return ArLabels(scanId: scan, room: room, floorY: j["floor_y_arkit"] as? Double, labels: labels) + } +} + +// MARK: CompassRoutes + +extension RecallAPI: CompassRoutes { + public func compassRoute(home: String, thing: String) async throws -> CompassRoute { + let req = request("homes/\(home)/objects/\(thing)/compass-route", method: "GET", contentType: "application/json", + body: Data()) + let (data, response) = try await Self.session.data(for: req) + let code = (response as? HTTPURLResponse)?.statusCode ?? 0 + let json = (try? JSONSerialization.jsonObject(with: data) as? [String: Any]) ?? [:] + guard code == 200 else { + // the Mac's reason, in words ("This room was scanned without a reliable compass; scan it again.") + let detail = (json["detail"] as? [String: Any])?["detail"] as? String ?? json["detail"] as? String + throw detail.map { RecallError.server(Self.clean($0)) } ?? RecallError.status(code) + } + return try Self.compassRoute(json) + } + + static func compassRoute(_ j: [String: Any]) throws -> CompassRoute { + guard let id = j["object_id"] as? String, let label = j["label"] as? String, let start = j["start"] as? String, + let place = j["place"] as? String, let legs = j["legs"] as? [[String: Any]] else { + throw RecallError.server("The Mac sent a route this app can't read. Update the app.") + } + return CompassRoute(thingId: id, label: label, color: j["color"] as? String, start: start, place: place, + legs: legs.compactMap { leg in + guard let b = leg["bearing_deg"] as? Double, let m = leg["metres"] as? Double else { return nil } + return CompassLeg(bearing: b, metres: m) + }) + } +} + +// MARK: RobotPackages + +extension RecallAPI: RobotPackages { + public func robotPackage(home: String, room: String?) async throws -> Data { + var req = request("homes/\(home)/robot-package", method: "GET", contentType: "application/zip", body: Data()) + if let room, let url = req.url, var parts = URLComponents(url: url, resolvingAgainstBaseURL: false) { + parts.queryItems = [URLQueryItem(name: "room", value: room)] + req.url = parts.url + } + let (data, response) = try await Self.session.data(for: req) + let code = (response as? HTTPURLResponse)?.statusCode ?? 0 + guard code == 200 else { throw RecallError.status(code) } + guard data.count <= Self.maxPackageBytes else { + throw RecallError.server("The robot package is larger than this phone will share (50 MB).") + } + return data + } +} + +// MARK: TrustSettings + +extension RecallAPI: TrustSettings { + public func trustLevel() async throws -> TrustLevel { + try Self.level(try await send(request("trust", method: "GET", contentType: "application/json", body: Data()))) + } + + public func setTrustLevel(_ level: TrustLevel) async throws -> TrustLevel { + let body = try JSONSerialization.data(withJSONObject: ["level": level.rawValue]) + return try Self.level(try await send(request("trust", method: "PUT", contentType: "application/json", body: body))) + } + + static func level(_ j: [String: Any]) throws -> TrustLevel { + guard let raw = j["level"] as? String, let level = TrustLevel(rawValue: raw) else { + throw RecallError.server("The Mac answered with a trust level this app doesn't know. Update the app.") + } + return level + } +} + // MARK: RobotSharing extension RecallAPI: RobotSharing { @@ -361,7 +508,7 @@ extension RecallAPI: RobotSharing { public func activity(limit: Int) async throws -> [AuditEntry] { var req = request("audit", method: "GET", contentType: "application/json", body: Data()) req.url = req.url.flatMap { URL(string: $0.absoluteString + "?limit=\(limit)") } - let (data, response) = try await URLSession.shared.data(for: req) + let (data, response) = try await Self.session.data(for: req) guard (response as? HTTPURLResponse)?.statusCode == 200 else { throw RecallError.status((response as? HTTPURLResponse)?.statusCode ?? 0) } return Self.entries(data) } @@ -394,6 +541,17 @@ extension RecallAPI: RobotSharing { return try Self.package(j) } + /// GET /homes/{id}/room-map?room=: the same document, for the owner's Find screen. Not a robot export, so it + /// works with robot access off and isn't in the robot activity. + public func roomMap(home: String, room: String) async throws -> RobotMapPackage { + var req = request("homes/\(home)/room-map", method: "GET", contentType: "application/json", body: Data()) + if let url = req.url, var parts = URLComponents(url: url, resolvingAgainstBaseURL: false) { + parts.queryItems = [URLQueryItem(name: "room", value: room)] + req.url = parts.url + } + return try Self.package(try await send(req)) + } + static func package(_ j: [String: Any]) throws -> RobotMapPackage { guard let m = j["map"] as? [String: Any], let b64 = m["pgm_base64"] as? String, let pgm = Data(base64Encoded: b64), let resolution = m["resolution"] as? Double, let origin = m["origin"] as? [Double], origin.count == 3, diff --git a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json index 25d898b..13b9233 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json +++ b/apps/ios/Packages/RecallKit/Sources/RecallNetworking/openapi.json @@ -114,6 +114,12 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, "422": { "$ref": "#/components/responses/Invalid" } @@ -143,6 +149,9 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -164,6 +173,9 @@ "204": { "description": "Erased" }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" }, @@ -226,9 +238,15 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, "422": { "$ref": "#/components/responses/Invalid" } @@ -268,9 +286,15 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, "422": { "$ref": "#/components/responses/Invalid" } @@ -300,11 +324,20 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" }, "409": { "$ref": "#/components/responses/Conflict" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" } } } @@ -342,8 +375,14 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/TooLarge" } } } @@ -371,6 +410,9 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -403,6 +445,9 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -426,6 +471,9 @@ } } } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" } } }, @@ -455,6 +503,74 @@ } } } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + } + } + } + }, + "/trust": { + "get": { + "tags": [ + "homes" + ], + "operationId": "getTrust", + "summary": "How sure Recall must be before it tells you about a thing", + "responses": { + "200": { + "description": "The trust level", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trust" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + } + }, + "put": { + "tags": [ + "homes" + ], + "operationId": "setTrust", + "summary": "Set how sure Recall must be (the owner's device key only)", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TrustUpdate" + } + } + } + }, + "responses": { + "200": { + "description": "The trust level after the change", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trust" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "422": { + "$ref": "#/components/responses/Invalid" } } } @@ -524,6 +640,12 @@ } } } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "422": { + "$ref": "#/components/responses/Invalid" } } } @@ -545,6 +667,9 @@ } } } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" } } } @@ -577,6 +702,9 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -612,12 +740,179 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" } } } }, + "/homes/{home_id}/room-map": { + "get": { + "tags": [ + "homes" + ], + "operationId": "getRoomMap", + "summary": "One room's map and the things in it, for the owner's phone (the same document a robot gets)", + "description": "The owner's view of a room: the occupancy map, the things on it and the room outline, in the robot export's format. Device key only; works whether robot access is on or off, and isn't recorded as a robot export. `room` picks a room; the default is the newest processed scan's.\n", + "parameters": [ + { + "$ref": "#/components/parameters/home_id" + }, + { + "name": "room", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The room's map", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RobotExport" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "description": "The scan's point cloud is missing; process the scan again", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/homes/{home_id}/ar-labels": { + "get": { + "tags": [ + "homes" + ], + "operationId": "getArLabels", + "summary": "Everything Recall found in a room, placed in that room's saved AR map (for labels over the camera)", + "description": "Positions are in the ARKit frame of the room's newest processed scan (scan_id): the phone relocalises in that scan's room map, then draws one label per thing. trusted and needs_review follow the owner's trust level.\n", + "parameters": [ + { + "$ref": "#/components/parameters/home_id" + }, + { + "name": "room", + "in": "query", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 60 + } + } + ], + "responses": { + "200": { + "description": "The labels", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArLabels" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "422": { + "$ref": "#/components/responses/Invalid" + } + } + } + }, + "/homes/{home_id}/robot-package": { + "get": { + "tags": [ + "robots" + ], + "operationId": "getRobotPackage", + "summary": "One room's memory for a robot as a .zip of files it loads (Nav2 map, things, room, 3D points, README)", + "description": "The same content as the robot export, as files: map.pgm and map.yaml (Nav2 map server), objects.json, room.json, cloud.ply (x y z, metres, z up) and README.md. Same access as the robot export: the owner's device key always (to share it from the phone), a robot's key only while robot access is on. Recorded in the audit.\n", + "security": [ + { + "deviceKey": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/home_id" + }, + { + "name": "room", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The package", + "content": { + "application/zip": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "description": "Robot access is off", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "description": "The scan's point cloud is missing; process the scan again", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/homes/{home_id}/robot-export": { "get": { "tags": [ @@ -655,6 +950,9 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "403": { "description": "Robot access is off", "content": { @@ -778,9 +1076,15 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, "422": { "$ref": "#/components/responses/Invalid" } @@ -804,8 +1108,50 @@ "204": { "description": "Forgotten" }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "404": { + "$ref": "#/components/responses/NotFound" + } + } + } + }, + "/homes/{home_id}/objects/{object_id}/compass-route": { + "get": { + "tags": [ + "guidance" + ], + "operationId": "getCompassRoute", + "summary": "Directions to a thing by compass bearing and metres, from where the room's scan began (for the Watch)", + "description": "For walking without the phone's camera: the route starts where the scan of the thing's room began (usually the doorway) and is given as legs of a magnetic compass bearing and a distance, using the north the phone measured while scanning. 409 no_compass when that scan had no reliable compass; 409 no_path when no route is wide enough for a person.\n", + "parameters": [ + { + "$ref": "#/components/parameters/home_id" + }, + { + "$ref": "#/components/parameters/object_id" + } + ], + "responses": { + "200": { + "description": "The route", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CompassRoute" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" } } } @@ -838,6 +1184,15 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, "503": { "$ref": "#/components/responses/Unavailable" } @@ -872,8 +1227,14 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/TooLarge" } } } @@ -916,8 +1277,17 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, "404": { "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" } } } @@ -960,6 +1330,15 @@ } } }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "413": { + "$ref": "#/components/responses/TooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, "503": { "$ref": "#/components/responses/Unavailable" } @@ -1205,6 +1584,13 @@ "type": "string", "description": "Why it failed, in one sentence" }, + "worker_alive": { + "type": [ + "boolean", + "null" + ], + "description": "While processing: whether the Mac's scan worker is running (its heartbeat is under 90 s old). false means the scan waits until someone runs make worker; null when not processing" + }, "frame_count": { "type": "integer" }, @@ -1332,6 +1718,51 @@ "usable", "redo" ] + }, + "duration_s": { + "type": "number", + "description": "How long the scan took, seconds" + }, + "points": { + "type": "integer", + "description": "LiDAR depth samples taken while scanning (for the report)" + }, + "steps": { + "type": "array", + "description": "How each voice-led step went: done, or moved on when its time ran out", + "items": { + "type": "object", + "required": [ + "step", + "seconds", + "completed", + "detail" + ], + "properties": { + "step": { + "type": "integer", + "description": "0 start, 1 turn around, 2 along the walls, 3 sweep down, 4 back to start, 5 finish; -1 recognise" + }, + "seconds": { + "type": "number" + }, + "completed": { + "type": "boolean" + }, + "detail": { + "type": "string", + "description": "e.g. Wall sides seen 9 of 9" + } + } + } + }, + "zones_seen": { + "type": "integer", + "description": "1 m floor columns x 5 height bands the LiDAR saw" + }, + "zones_total": { + "type": "integer", + "description": "Zones in the columns the scan reached (columns x 5)" } } }, @@ -1623,7 +2054,11 @@ ], "properties": { "label": { - "type": "string" + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "^[^\\u0000-\\u001f\\u007f]+$", + "description": "What SAM 3.1 was asked for (\"mug\", \"sofa\"); no control characters" }, "centroid": { "type": "array", @@ -1690,6 +2125,7 @@ "string", "null" ], + "maxLength": 30, "description": "Basic colour name measured from the photo pixels inside the box (objects only)" }, "size_m": { @@ -1927,9 +2363,20 @@ "corrections": { "$ref": "#/components/schemas/ObjectCorrection" }, + "photos": { + "type": [ + "integer", + "null" + ], + "description": "How many of the scan's photos showed it (null for sightings kept before this was recorded)" + }, + "trusted": { + "type": "boolean", + "description": "Seen in enough photos for the owner's trust level (or confirmed): the Guide, Find, room maps and robots use it" + }, "needs_review": { "type": "boolean", - "description": "The models were unsure (confidence under 0.7) and the user hasn't confirmed it" + "description": "Unconfirmed and either unsure (confidence under 0.7) or not trusted yet" } } }, @@ -1970,6 +2417,47 @@ } } }, + "Trust": { + "type": "object", + "description": "How sure Recall must be before it tells the owner (or a robot) about a thing, by the photos it was seen in", + "required": [ + "level", + "min_photos" + ], + "properties": { + "level": { + "type": "string", + "enum": [ + "careful", + "balanced", + "everything" + ], + "description": "careful: 3 photos; balanced: 2 (default); everything: 1" + }, + "min_photos": { + "type": "integer", + "minimum": 1, + "description": "Photos a thing must be seen in, at this level (a confirmed thing always counts)" + } + } + }, + "TrustUpdate": { + "type": "object", + "required": [ + "level" + ], + "additionalProperties": false, + "properties": { + "level": { + "type": "string", + "enum": [ + "careful", + "balanced", + "everything" + ] + } + } + }, "AuditEvent": { "type": "object", "description": "One recorded event: who (by the key they used), what, how it ended. Append-only, kept on the Mac for RECALL_AUDIT_RETENTION_DAYS (90). Never a key, token or password; free text is trimmed.\n", @@ -1995,6 +2483,7 @@ "owner", "robot", "pipeline", + "system", "unknown" ] }, @@ -2002,14 +2491,22 @@ "type": "string", "enum": [ "robot_access.changed", + "trust.changed", "robot_export.read", "home_memory.query", "auth.refused", "object.corrected", "object.forgotten", + "scan.created", + "scan.uploaded", + "scan.failed", "scan.ingested", "scan.erased", - "arrival.verified" + "arrival.verified", + "guide.answered", + "voice.transcribed", + "audit.pruned", + "rate.limited" ] }, "outcome": { @@ -2548,6 +3045,138 @@ } } }, + "ArLabels": { + "type": "object", + "required": [ + "scan_id", + "room", + "labels" + ], + "properties": { + "scan_id": { + "type": "string", + "description": "The scan whose room map the positions are in" + }, + "room": { + "type": "string" + }, + "floor_y_arkit": { + "type": [ + "number", + "null" + ], + "description": "The floor's height in that ARKit frame, metres" + }, + "labels": { + "type": "array", + "items": { + "type": "object", + "required": [ + "object_id", + "title", + "place", + "confidence", + "trusted", + "needs_review", + "position_arkit" + ], + "properties": { + "object_id": { + "type": "string" + }, + "title": { + "type": "string", + "description": "e.g. grey bag, or the owner's own name for it" + }, + "place": { + "type": "string", + "description": "e.g. near the desk, at knee height" + }, + "confidence": { + "type": "number" + }, + "photos": { + "type": [ + "integer", + "null" + ] + }, + "trusted": { + "type": "boolean" + }, + "needs_review": { + "type": "boolean" + }, + "position_arkit": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 3, + "maxItems": 3 + } + } + } + } + } + }, + "CompassRoute": { + "type": "object", + "description": "A walk to a thing from where its room's scan began, as compass legs (for a watch with no camera)", + "required": [ + "object_id", + "label", + "start", + "place", + "legs" + ], + "properties": { + "object_id": { + "type": "string" + }, + "label": { + "type": "string", + "description": "e.g. keys" + }, + "color": { + "type": [ + "string", + "null" + ] + }, + "start": { + "type": "string", + "description": "Where to stand first, in words, e.g. where you started the scan of Living room" + }, + "place": { + "type": "string", + "description": "Where the thing is, in words, e.g. near the desk, at knee height" + }, + "legs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "bearing_deg", + "metres" + ], + "properties": { + "bearing_deg": { + "type": "number", + "minimum": 0, + "maximum": 360, + "description": "Magnetic bearing, clockwise from north" + }, + "metres": { + "type": "number", + "minimum": 0 + } + } + } + } + } + }, "AskRequest": { "type": "object", "required": [ @@ -2556,7 +3185,8 @@ ], "properties": { "home_id": { - "type": "string" + "type": "string", + "pattern": "^[a-z0-9-]{1,64}$" }, "question": { "type": "string", @@ -2568,6 +3198,7 @@ "string", "null" ], + "pattern": "^[A-Za-z0-9_-]{1,64}$", "description": "A guidance session, if the user is already being guided" }, "about": { @@ -2575,6 +3206,7 @@ "string", "null" ], + "pattern": "^[A-Za-z0-9_-]{1,64}$", "description": "The thing the last answer was about, for a follow-up (\"why do you think so?\")" } } @@ -2869,6 +3501,26 @@ } }, "responses": { + "Unauthorized": { + "description": "Missing or wrong device key", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "TooLarge": { + "description": "The body is larger than this route accepts (a part 64 MB, a result 50 MB, an arrival frame 25 MB, audio 10 MB, anything else 1 MB)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, "Invalid": { "description": "Invalid input", "content": { @@ -2899,6 +3551,24 @@ } } }, + "RateLimited": { + "description": "Too many calls from this key to a route that reaches a model or the pipeline; try again after Retry-After seconds", + "headers": { + "Retry-After": { + "schema": { + "type": "integer" + }, + "description": "Seconds to wait" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, "Unavailable": { "description": "A required model or service is not configured or not reachable", "content": { diff --git a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/Keychain.swift b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/Keychain.swift index 800d119..baf7a14 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallPersistence/Keychain.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallPersistence/Keychain.swift @@ -21,7 +21,7 @@ public struct Keychain: SecretStore { SecItemDelete(q as CFDictionary) var add = q add[kSecValueData as String] = Data(value.utf8) - add[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlock + add[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly // never in a backup or another device SecItemAdd(add as CFDictionary, nil) } } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift index a24c6bb..2424d99 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallTestSupport/Fakes.swift @@ -114,8 +114,12 @@ public final class FakeClock: Clock, @unchecked Sendable { @MainActor public final class FakeSpeech: SpeechOutput { public private(set) var lines: [String] = [] public init() {} - public func say(_ text: String, interrupt: Bool) { lines.append(text) } - public func stop() {} + public private(set) var isSpeaking = false + public var onFinished: (() -> Void)? + public func say(_ text: String, interrupt: Bool) { lines.append(text); isSpeaking = true } + public func stop() { isSpeaking = false; onFinished?() } + /// The test's stand-in for the line ending by itself. + public func finish() { isSpeaking = false; onFinished?() } } @MainActor public final class FakeHaptics: HapticsOutput { @@ -273,3 +277,39 @@ public final class FakeRobotSharing: RobotSharing, @unchecked Sendable { exportURL: "http://mac.local:8000/homes/flat/robot-export") } } + +/// Room maps by room name; a room not in `maps` throws, as the Mac does for a room with no processed scan. +public final class FakeRoomMaps: RoomMaps, @unchecked Sendable { + public var maps: [String: RobotMapPackage] + public private(set) var asked: [String] = [] + public init(maps: [String: RobotMapPackage] = [:]) { self.maps = maps } + public func roomMap(home: String, room: String) async throws -> RobotMapPackage { + asked.append(room) + guard let map = maps[room] else { throw RecallError.status(404) } + return map + } +} + +/// Hears what it's told to: `text` (or throws `error`). +public final class FakeWords: Transcriber, @unchecked Sendable { + public var text: String + public var error: Error? + public init(_ text: String = "where are my keys", error: Error? = nil) { self.text = text; self.error = error } + public func transcribe(wav: Data) async throws -> String { + if let error { throw error } + return text + } +} + +/// A one-leg route to whatever is asked for (or throws `error`). +public final class FakeCompassRoutes: CompassRoutes, @unchecked Sendable { + public var error: Error? + public private(set) var asked: [String] = [] + public init(error: Error? = nil) { self.error = error } + public func compassRoute(home: String, thing: String) async throws -> CompassRoute { + asked.append(thing) + if let error { throw error } + return CompassRoute(thingId: thing, label: "keys", color: "red", start: "where you started the scan of Living room", + place: "on the desk, at waist height", legs: [CompassLeg(bearing: 90, metres: 3)]) + } +} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallUpload/FolderZip.swift b/apps/ios/Packages/RecallKit/Sources/RecallUpload/FolderZip.swift deleted file mode 100644 index cf55472..0000000 --- a/apps/ios/Packages/RecallKit/Sources/RecallUpload/FolderZip.swift +++ /dev/null @@ -1,22 +0,0 @@ -import Foundation - -/// Zips a folder with the system's own archiver (NSFileCoordinator .forUploading), as the scan upload does. -public enum FolderZip { - /// Writes `files` into `//` and returns `/.zip` (rebuilt each time). - public static func make(name: String, files: [String: Data], in directory: URL) throws -> URL { - let fm = FileManager.default - let folder = directory.appending(path: name) - let zip = directory.appending(path: name + ".zip") - try? fm.removeItem(at: folder) - try? fm.removeItem(at: zip) - try fm.createDirectory(at: folder, withIntermediateDirectories: true) - for (file, data) in files { try data.write(to: folder.appending(path: file), options: .atomic) } - var coordError: NSError? - var copyError: Error? - NSFileCoordinator().coordinate(readingItemAt: folder, options: .forUploading, error: &coordError) { tmp in - do { try fm.copyItem(at: tmp, to: zip) } catch { copyError = error } - } - if let e = coordError ?? copyError { throw e } - return zip - } -} diff --git a/apps/ios/Packages/RecallKit/Sources/RecallVoice/Announcer.swift b/apps/ios/Packages/RecallKit/Sources/RecallVoice/Announcer.swift index 1d77d3d..b9a4bcb 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallVoice/Announcer.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallVoice/Announcer.swift @@ -3,18 +3,59 @@ import RecallDomain import UIKit /// Speaks short lines. With VoiceOver on, lines go through VoiceOver (the user's own voice and rate); otherwise through -/// AVSpeechSynthesizer, an OS feature, not a speech model (CLAUDE.md rule 8). +/// AVSpeechSynthesizer, an OS feature, not a speech model (CLAUDE.md rule 8), in the iPhone's best voice for the +/// owner's choice (Settings → Voice: automatic, female or male; premium voices first when downloaded). @MainActor -public final class Announcer: SpeechOutput { +public final class Announcer: NSObject, SpeechOutput, AVSpeechSynthesizerDelegate { private let synth = AVSpeechSynthesizer() public var rate: Float = AVSpeechUtteranceDefaultSpeechRate /// Sees every line as it is spoken (the UI tests' log of what was said; nil in the app). public var onSay: ((String) -> Void)? + public var onFinished: (() -> Void)? - public init() { + public override init() { + super.init() + synth.delegate = self try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio, options: [.duckOthers]) } + public var isSpeaking: Bool { synth.isSpeaking } + + nonisolated public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + Task { @MainActor in if !self.synth.isSpeaking { self.onFinished?() } } + } + + nonisolated public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + Task { @MainActor in if !self.synth.isSpeaking { self.onFinished?() } } + } + + /// The owner's choice in Settings → Voice, kept on this iPhone. + public static let voiceKey = "recall.voice" + + public var voicePreference: VoicePreference { + get { VoicePreference(rawValue: UserDefaults.standard.string(forKey: Self.voiceKey) ?? "") ?? .automatic } + set { UserDefaults.standard.set(newValue.rawValue, forKey: Self.voiceKey) } + } + + /// Every voice on this iPhone, as Recall's domain describes them. + public static func installedVoices() -> [VoiceOption] { + AVSpeechSynthesisVoice.speechVoices().map { v in + VoiceOption( + id: v.identifier, name: v.name, language: v.language, + quality: v.quality == .premium ? 2 : v.quality == .enhanced ? 1 : 0, + gender: v.gender == .female ? .female : v.gender == .male ? .male : .unspecified, + special: v.voiceTraits.contains(.isNoveltyVoice) || v.voiceTraits.contains(.isPersonalVoice) + ) + } + } + + /// The voice Recall would speak in for `preference`: the best installed one in the iPhone's language. + public static func chosenVoice(_ preference: VoicePreference) -> VoiceOption? { + VoiceChoice.best(installedVoices(), preference: preference, language: AVSpeechSynthesisVoice.currentLanguageCode()) + } + + public func currentVoice() -> VoiceOption? { Self.chosenVoice(voicePreference) } + public func say(_ text: String, interrupt: Bool = false) { onSay?(text) if UIAccessibility.isVoiceOverRunning { @@ -26,6 +67,7 @@ public final class Announcer: SpeechOutput { try? AVAudioSession.sharedInstance().setActive(true) let u = AVSpeechUtterance(string: text) u.rate = rate + if let id = currentVoice()?.id { u.voice = AVSpeechSynthesisVoice(identifier: id) } // else the system default synth.speak(u) } diff --git a/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift b/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift index 50a1e69..0b1367e 100644 --- a/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift +++ b/apps/ios/Packages/RecallKit/Sources/RecallVoice/WavRecorder.swift @@ -37,7 +37,9 @@ public final class WavRecorder: NSObject, VoiceInput { try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio, options: [.duckOthers]) let handler = finished finished = nil - handler?(ok ? try? Data(contentsOf: file) : nil) + let audio = ok ? try? Data(contentsOf: file) : nil + try? FileManager.default.removeItem(at: file) // the question is in memory now; no copy is left on the phone + handler?(audio) } } diff --git a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/AnswerWatchTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/AnswerWatchTests.swift new file mode 100644 index 0000000..0f82a4a --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/AnswerWatchTests.swift @@ -0,0 +1,49 @@ +import Foundation +import Testing +@testable import RecallApplication +import RecallDomain +import RecallTestSupport + +@Suite("The iPhone answering its Watch") +struct AnswerWatchTests { + private func answer(memory: FakeHomeMemory = FakeHomeMemory(things: [.sample()]), guide: FakeGuide = FakeGuide(), + words: FakeWords = FakeWords(), routes: FakeCompassRoutes = FakeCompassRoutes()) -> AnswerWatch { + AnswerWatch(memory: memory, guide: guide, transcriber: words, routes: routes) + } + + @Test func theDialListsTrustedThingsByName() async { + let seenOnce = Thing(id: "w", label: "wallet", color: "green", heightM: 0.4, heightBand: "knee", confidence: 0.55, + photos: 1, trusted: false, seenAt: .now) + let reply = await answer(memory: FakeHomeMemory(things: [.sample(), seenOnce]))(.things, home: "flat") + #expect(reply.things?.map(\.title) == ["Red keys"]) // the wallet seen once waits in Found + } + + @Test func aPickedThingComesWithItsRouteAndWhereToStart() async { + let routes = FakeCompassRoutes() + let reply = await answer(routes: routes)(.route(thingId: "obj_1"), home: "flat") + #expect(routes.asked == ["obj_1"]) + #expect(reply.route?.legs.count == 1) + #expect(reply.say == "Your red keys: on the desk, at waist height. To walk there, stand where you started the scan of Living room, then double tap.") + } + + @Test func aSpokenQuestionIsTranscribedAnsweredAndRouted() async { + let reply = await answer()(.ask(audioFile: "q.wav"), home: "flat", audio: Data([1, 2, 3])) + #expect(reply.say?.hasPrefix("On the desk.") == true) + #expect(reply.route?.thingId == "obj_1") + } + + @Test func notFoundIsSaidWithoutARoute() async { + let guide = FakeGuide(answer: Answer(text: "I haven't seen your passport yet.", thingId: nil, grounded: true)) + let reply = await answer(guide: guide)(.ask(audioFile: "q.wav"), home: "flat", audio: Data([1])) + #expect(reply.say == "I haven't seen your passport yet.") + #expect(reply.route == nil) + } + + @Test func troubleComesBackAsWords() async { + let routes = FakeCompassRoutes(error: RecallError.server("This room was scanned without a reliable compass; scan it again.")) + let reply = await answer(routes: routes)(.route(thingId: "obj_1"), home: "flat") + #expect(reply.say == "This room was scanned without a reliable compass; scan it again.") + let silent = await answer()(.ask(audioFile: "q.wav"), home: "flat", audio: Data()) + #expect(silent.say == "I didn't hear anything. Try again.") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift index 6602ef1..612a55e 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallApplicationTests/UseCaseTests.swift @@ -200,13 +200,14 @@ struct UseCaseTests { #expect(speech.lines.last == "Hall is ready. Nothing has moved since the last scan.") } - @Test func aRescanThatWasNotRecognisedSaysItStartedAfresh() async { + @Test func aRescanThatDidNotLineUpSaysItIsAFreshMap() async { let speech = FakeSpeech() let scan = Scan(id: "f", room: "Kitchen", createdAt: .now, grade: .good, directions: 10, frames: 50, state: .processing) let out = await TrackProcessing(processing: FakeProcessing(reports: ["f": ScanReport(.processed, fresh: true)]), memory: FakeHomeMemory(things: [keys]), speech: speech)([scan], home: "flat") #expect(out["f"]?.fresh == true) - #expect(speech.lines == ["I didn't recognise Kitchen, so its memory started afresh. Kitchen is ready. 1 thing found."]) + #expect(speech.lines == ["A fresh map of Kitchen. This walk didn't line up with your last scan, so I can't say what " + + "moved this time. Kitchen is ready. 1 thing found."]) } @Test func uploadNeedsSetupAndAManifest() async throws { @@ -231,6 +232,16 @@ struct UseCaseTests { #expect(try pair(URL(string: "https://example.com")!) == nil) } + @Test func readingAPairingLinkChangesNothingUntilItIsApplied() throws { + let store = FakeConnectionStore() + let before = store.load() + let pair = PairDevice(store: store) + let pairing = try #require(try pair.read(URL(string: "recall://pair?url=https://evil.example&key=theirs")!)) + #expect(store.load() == before) // a link alone never re-points the phone + let c = pair.apply(pairing) + #expect(c.address == "https://evil.example" && store.connection == c) + } + @Test func testConnectionSaysTheModel() async throws { #expect(try await TestConnection(check: FakeConnectionCheck(.success("baseten_spark_open")))() == "Connected. The Mac's Guide runs on baseten_spark_open.") @@ -249,3 +260,54 @@ struct UseCaseTests { #expect(memory.things.map(\.id) == ["2"]) } } + +@Suite("A scan waiting with no worker") +@MainActor +struct WorkerStoppedTests { + private let waiting = Scan(id: "w", room: "Living room", createdAt: .now, grade: .good, directions: 12, frames: 160, + state: .processing) + + @Test func itSaysSoOnceWhenTheWorkerIsntRunning() async { + let speech = FakeSpeech() + let track = TrackProcessing(processing: FakeProcessing(reports: ["w": ScanReport(.processing, workerAlive: false)]), + memory: FakeHomeMemory(), speech: speech) + let out = await track([waiting], home: "flat") + #expect(out["w"]?.workerStopped == true && out["w"]?.state == .processing) + #expect(speech.lines == [Scan.workerStoppedMessage]) + var known = waiting + known.workerStopped = true + #expect(await track([known], home: "flat").isEmpty) // already said: nothing new + #expect(speech.lines.count == 1) + } + + @Test func itClearsOnceTheWorkerRunsAgain() async { + var known = waiting + known.workerStopped = true + let track = TrackProcessing(processing: FakeProcessing(reports: ["w": ScanReport(.processing, workerAlive: true)]), + memory: FakeHomeMemory(), speech: FakeSpeech()) + #expect(await track([known], home: "flat")["w"]?.workerStopped == false) + } + + @Test func aMacThatDoesntSayChangesNothing() async { + let track = TrackProcessing(processing: FakeProcessing(reports: ["w": ScanReport(.processing)]), + memory: FakeHomeMemory(), speech: FakeSpeech()) + #expect(await track([waiting], home: "flat").isEmpty) + } +} + +@Suite("Quiet when the app opens") +@MainActor +struct QuietOpenTests { + @Test func aScanThatFinishedWhileClosedUpdatesWithoutTalking() async { + let speech = FakeSpeech() + let scan = Scan(id: "q", room: "Living room", createdAt: .now, grade: .good, directions: 12, frames: 160, + state: .processing) + let track = TrackProcessing(processing: FakeProcessing(reports: ["q": ScanReport(.processed, fresh: true)]), + memory: FakeHomeMemory(), speech: speech) + let quiet = await track([scan], home: "flat", announce: false) + #expect(quiet["q"]?.state == .processed && quiet["q"]?.fresh == true) // the card still updates + #expect(speech.lines.isEmpty) + _ = await track([scan], home: "flat") // while the app is open, a finished scan is still said + #expect(speech.lines.count == 1) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift new file mode 100644 index 0000000..40183bc --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ArLabelTests.swift @@ -0,0 +1,39 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Labels over the camera, and the live scan readouts") +struct ArLabelTests { + private func label(trusted: Bool = true, review: Bool = false, photos: Int? = 5) -> ArLabel { + ArLabel(id: "o", title: "grey bag", place: "near the desk, at knee height", confidence: 0.876, photos: photos, + trusted: trusted, needsReview: review, position: .zero) + } + + @Test func eachLabelIsColouredByWhatRecallDoesWithIt() { + #expect(label().mark == .told) + #expect(label(review: true).mark == .check) + #expect(label(trusted: false, review: true, photos: 1).mark == .seenOnce) + } + + @Test func theTagSaysTheThingTheScoreAndThePhotos() { + #expect(label().headline == "GREY BAG") + #expect(label().detail == "88% · 5 photos") + #expect(label(photos: 1).detail == "88% · 1 photo") + #expect(label(photos: nil).detail == "88%") + #expect(label().spoken == "Grey bag, near the desk, at knee height. 88 percent, seen in 5 photos. Recall tells you.") + } + + @Test func thePointTrailKeepsTheNewestAndStaysUnderItsCap() { + var trail = PointTrail(cap: 100) + for i in 0..<12 { trail.add((0..<20).map { SIMD3(Float(i), Float($0), 0) }) } + #expect(trail.total == 240) + #expect(trail.points.count <= 100) + #expect(trail.points.last == SIMD3(11, 19, 0)) // the newest point is always kept + } + + @Test func walkingFasterThanTheCoachAllowsTurnsTheSpeedAmber() { + #expect(!ScanSpeed.isTooFast(0.4)) + #expect(ScanSpeed.isTooFast(0.7)) + #expect(ScanSpeed.text(0.42) == "0.4 m/s") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/AuditEntryTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/AuditEntryTests.swift index c4065c6..40cba09 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/AuditEntryTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/AuditEntryTests.swift @@ -28,6 +28,17 @@ struct AuditEntryTests { == "Refused: wrong key (/a2a/home-memory)") } + @Test func everyActionTheMacRecordsReadsAsASentence() { + let actions = ["trust.changed", "scan.created", "scan.uploaded", "scan.failed", "scan.ingested", "object.corrected", + "object.forgotten", "arrival.verified", "guide.answered", "voice.transcribed", "audit.pruned"] + for action in actions { + #expect(entry(action, "owner").summary != action, "\(action) has no sentence") + } + #expect(entry("object.forgotten", "owner", "ok", ["label": "wallet"]).summary == "You said wallet isn't there") + #expect(entry("voice.transcribed", "owner", "failed").summary == "Your voice couldn't reach Muse Voice Transcribe") + #expect(entry("trust.changed", "owner", "ok", ["level": "careful"]).summary == "You set how sure Recall must be to careful") + } + @Test func onlyRobotMattersAreShown() { #expect(entry("home_memory.query", "robot").isAboutRobots) #expect(!entry("object.corrected", "owner").isAboutRobots) diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/CompassWalkTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/CompassWalkTests.swift new file mode 100644 index 0000000..588125f --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/CompassWalkTests.swift @@ -0,0 +1,50 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Walking by compass") +struct CompassWalkTests { + private let route = [CompassLeg(bearing: 90, metres: 3), CompassLeg(bearing: 180, metres: 2)] + + @Test func theOffsetTakesTheShortWayRound() { + #expect(CompassWalk.offset(heading: 350, bearing: 10) == 20) // right, across north + #expect(CompassWalk.offset(heading: 10, bearing: 350) == -20) // left + #expect(CompassWalk.offset(heading: 0, bearing: 180) == 180) + } + + @Test func itSaysWhichWayToTurnUntilFacingTheLeg() { + var walk = CompassWalk(legs: route) + #expect(walk.update(heading: 0, metresWalked: 0) == .turn(.right, degrees: 90)) + #expect(walk.update(heading: 180, metresWalked: 0) == .turn(.left, degrees: 90)) + #expect(walk.update(heading: 85, metresWalked: 0) == .onCourse(metresLeft: 3)) + } + + @Test func smallWobblesWhileWalkingDontSayTurn() { + var walk = CompassWalk(legs: route) + _ = walk.update(heading: 90, metresWalked: 0) + #expect(walk.update(heading: 110, metresWalked: 1) == .onCourse(metresLeft: 2)) // 20° off: still on course + if case .turn = walk.update(heading: 130, metresWalked: 1.5) {} else { Issue.record("40° off should say turn") } + } + + @Test func legsEndByDistanceThenItArrives() { + var walk = CompassWalk(legs: route) + _ = walk.update(heading: 90, metresWalked: 0) + #expect(walk.update(heading: 90, metresWalked: 3.1) == .nextLeg(1)) + #expect(walk.update(heading: 90, metresWalked: 3.1) == .turn(.right, degrees: 90)) // the next leg goes south + #expect(walk.update(heading: 180, metresWalked: 5.2) == .arrived) + #expect(walk.update(heading: 0, metresWalked: 9) == .arrived) + } + + @Test func walkingFarPastALegSaysStop() { + var walk = CompassWalk(legs: route) + _ = walk.update(heading: 90, metresWalked: 0) + #expect(walk.update(heading: 90, metresWalked: 6) == .overshot) + } + + @Test func eachCueHasShortWords() { + #expect(CompassWalk.say(.turn(.left, degrees: 40), leg: nil) == "Turn left") + #expect(CompassWalk.say(.turn(.right, degrees: 170), leg: nil) == "Turn around") + #expect(CompassWalk.say(.onCourse(metresLeft: 2.4), leg: nil) == "Straight on, 2 metres") + #expect(CompassWalk.say(.nextLeg(1), leg: CompassLeg(bearing: 0, metres: 1)) == "Next, 1 metre") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ModelTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ModelTests.swift index 69c5c8a..27adc03 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ModelTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ModelTests.swift @@ -10,6 +10,37 @@ struct ConnectionTests { #expect(Connection(address: "ftp://x", key: "k", home: "").url == nil) } + @Test func aBareHostIsHttpsUnlessItIsOnThisNetwork() { + let url = { (a: String) in Connection(address: a, key: "k", home: "").url?.absoluteString } + #expect(url("mac.example.com:8445") == "https://mac.example.com:8445") // across the internet: encrypted + #expect(url("recall.trycloudflare.com") == "https://recall.trycloudflare.com") + #expect(url("10.0.0.5:8000") == "http://10.0.0.5:8000") + #expect(url("172.20.1.1:8000") == "http://172.20.1.1:8000") + #expect(url("172.40.1.1:8000") == "https://172.40.1.1:8000") // not a private range + #expect(url("rachits-mac.local:8000") == "http://rachits-mac.local:8000") + #expect(url("localhost:8000") == "http://localhost:8000") + // Tailscale: encrypted by the tailnet itself, and the Mac serves plain http there + #expect(url("100.80.5.54:8000") == "http://100.80.5.54:8000") + #expect(url("rachits-macbook-pro-2.tail67b272.ts.net:8000") == "http://rachits-macbook-pro-2.tail67b272.ts.net:8000") + #expect(url("100.200.1.1:8000") == "https://100.200.1.1:8000") // outside 100.64/10 + #expect(url("http://mac.example.com") == "http://mac.example.com") // a typed scheme is kept + } + + @Test func theAddressIsMaskedButRecognisable() { + let masked = { (a: String) in Connection(address: a, key: "k", home: "").maskedAddress } + #expect(masked("100.80.5.54:8000") == "http://100••••••:8000") + #expect(masked("rachits-macbook-pro-2.tail67b272.ts.net:8000") == "http://rac••••••:8000") + #expect(masked("https://mac.example.com") == "https://mac••••••") + #expect(!masked("rachits-macbook-pro-2.tail67b272.ts.net:8000").contains("tail67b272")) + #expect(masked("") == "") + } + + @Test func aPairingLinkSaysWhereWithoutShowingTheKey() throws { + let p = try #require(try PairingLink(URL(string: "recall://pair?url=https://mac:8445&key=abcdefgh1234&home=loft")!)) + #expect(p.summary == "Pair with the Mac at mac:8445? Key: 12 characters, ending in 1 2 3 4.") + #expect(!p.summary.contains("abcdefgh")) + } + @Test func saysWhatIsMissing() { #expect(Connection(address: "", key: "", home: "").issue == .nothingSet) #expect(Connection(address: "::", key: "k", home: "").issue == .badAddress) diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PendingRepliesTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PendingRepliesTests.swift new file mode 100644 index 0000000..249522c --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PendingRepliesTests.swift @@ -0,0 +1,42 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Waiting for the iPhone") +struct PendingRepliesTests { + @Test func aReplyIsTakenOnceThenGone() { + let pending = PendingReplies() + pending.put("a", "handler") + #expect(pending.take("a") == "handler") + #expect(pending.take("a") == nil) // the timeout, coming second, finds nothing + #expect(pending.count == 0) + } + + @Test func aTimedOutRequestDropsTheLateReply() { + let pending = PendingReplies() + pending.put("a", "handler") + #expect(pending.take("a") != nil) // timeout fires first and answers + #expect(pending.take("a") == nil) // the iPhone's late reply is dropped + } + + @Test func otherRequestsAreUntouched() { + let pending = PendingReplies() + pending.put("a", 1) + pending.put("b", 2) + _ = pending.take("a") + #expect(pending.take("b") == 2) + } + + @Test func theWaitIsLongEnoughForTheGuideButNotForever() { + #expect(PendingReplies.timeout == .seconds(25)) + #expect(WatchReply.phoneDidNotAnswer.say?.contains("didn't answer") == true) + } + + @Test func aReplyFromThePhoneIsCapped() { + let many = (0..<500).map { WatchThing(id: "t\($0)", title: "thing \($0)", place: "here") } + let r = WatchReply(things: many, say: String(repeating: "a", count: 1000)).capped + #expect(r.things?.count == WatchReply.maxThings) + #expect(r.say?.count == 400) + #expect(WatchReply(say: "Fine.").capped == WatchReply(say: "Fine.")) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PrivacyTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PrivacyTests.swift index 7440188..6c743f1 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PrivacyTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/PrivacyTests.swift @@ -14,6 +14,6 @@ struct PrivacyTests { @Test func theNoteNamesWhatLeavesAndWhatNeverDoes() { #expect(PrivacyNote.short.contains("own computer, a Mac today") && PrivacyNote.short.contains("Meta's models online")) #expect(PrivacyNote.never.contains { $0.contains("No Recall server") }) - #expect(PrivacyNote.controls.map(\.title) == ["Delete a scan", "Forget a thing", "Robot Access", "Private mode"]) + #expect(PrivacyNote.controls.map(\.title) == ["Delete a scan", "Forget a thing", "Robot Access", "Private mode", "People in view"]) } } diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanHistoryTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanHistoryTests.swift new file mode 100644 index 0000000..0f7d0bd --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanHistoryTests.swift @@ -0,0 +1,50 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Earlier scans fold under a room's newest") +struct ScanHistoryTests { + private func scan(_ id: String, _ room: String, day: Double, _ state: UploadState = .processed) -> Scan { + Scan(id: id, room: room, createdAt: Date(timeIntervalSince1970: day * 86_400), grade: .good, directions: 12, + frames: 60, state: state) + } + + @Test func aRescanHoldsTheRoomAndTheFirstScanIsItsHistory() { + let rooms = ScanHistory.byRoom([scan("a", "Living room", day: 1), scan("b", "Living room", day: 3)]) + #expect(rooms.map(\.latest.id) == ["b"]) + #expect(rooms[0].earlier.map(\.id) == ["a"]) + #expect(rooms[0].successor(of: rooms[0].earlier[0])?.id == "b") + #expect(rooms[0].successor(of: rooms[0].latest) == nil) + } + + @Test func eachRoomHasItsOwnCardNewestFirst() { + let rooms = ScanHistory.byRoom([ + scan("k", "Kitchen", day: 2), scan("l1", "Living room", day: 1), scan("l2", "Living room", day: 3), + ]) + #expect(rooms.map(\.latest.id) == ["l2", "k"]) + #expect(rooms[1].earlier.isEmpty) + } + + @Test func aScanStillUploadingKeepsItsOwnCardAboveTheRoomsMemory() { + let rooms = ScanHistory.byRoom([ + scan("a", "Living room", day: 1), scan("b", "Living room", day: 2), + scan("c", "Living room", day: 3, .uploading(fraction: 0.4)), + ]) + #expect(rooms.map(\.latest.id) == ["c", "b"]) + #expect(rooms[1].earlier.map(\.id) == ["a"]) + } + + @Test func aFailedOrLocalScanNeverFolds() { + let rooms = ScanHistory.byRoom([ + scan("a", "Kitchen", day: 1, .failed(reason: "network")), scan("b", "Kitchen", day: 2), + scan("c", "Kitchen", day: 0, .notUploaded), + ]) + #expect(rooms.map(\.latest.id) == ["b", "a", "c"]) + } + + @Test func roomNamesMatchWhateverTheirCaseAndSpaces() { + let rooms = ScanHistory.byRoom([scan("a", "living room ", day: 1), scan("b", "Living room", day: 2)]) + #expect(rooms.count == 1) + #expect(rooms[0].earlier.map(\.id) == ["a"]) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift new file mode 100644 index 0000000..ef76787 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ScanStepsTests.swift @@ -0,0 +1,120 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Scan steps that explain themselves, and the report's tips") +struct ScanStepsTests { + private func p(_ t: Double, dirs: Int = 0, walls: Int = 0, floor: Double = 0, walked: Double = 0, fromStart: Double = 3, + map: MapStatus = .extending) -> ScanGuide.Progress { + .init(elapsed: t, directions: dirs, walls: walls, floorSeenM2: floor, areaWalkedM2: 4, walkedM: walked, + fromStartM: fromStart, map: map) + } + + @Test func everyStepSaysWhatWhyAndWhenItsDone() { + for step in ScanGuide.Step.allCases { + #expect(!step.instruction.isEmpty && !step.why.isEmpty && step.doneWhen.hasPrefix("Done when")) + } + #expect(ScanGuide.Step.alongWalls.instruction.contains("about a metre away")) + #expect(ScanGuide.Step.alongWalls.why.contains("1 metre")) + } + + @Test func eachStepHasAMeter() { + #expect(ScanGuide.progress(of: .turnAround, p(10, dirs: 7)).text == "Directions seen 7 of 11") + #expect(ScanGuide.progress(of: .alongWalls, p(10, walls: 5)).fraction == 5.0 / 9) + #expect(ScanGuide.progress(of: .sweepDown, p(10, floor: 1.6)).text == "Floor and tops seen 1.6 of 2.5 m²") + #expect(ScanGuide.progress(of: .returnToStart, p(10, fromStart: 2.3)).text == "2.3 m from the start") + #expect(ScanGuide.progress(of: .returnToStart, p(10, fromStart: 0.5)).text == "Back at the start") + #expect(ScanGuide.progress(of: .finish, p(10, map: .mapped)).fraction == 1) + } + + @Test func aboutAMetreFromTheWallIsRight() { + #expect(ScanGuide.wallDistance(1.1) == .good) + #expect(ScanGuide.wallDistance(0.4) == .tooClose) + #expect(ScanGuide.wallDistance(2.5) == .tooFar) + } + + @Test func eachStepIsRecordedDoneOrMovedOn() { + var g = ScanGuide() + _ = g.update(p(4)) + _ = g.update(p(20, dirs: 11)) + let line = g.update(p(141, dirs: 12, walls: 6)) // the walls step ran out of time + #expect(line?.hasPrefix("Moving on: wall sides seen 6 of 9.") == true) + #expect(g.records.map(\.completed) == [true, true, false]) + #expect(g.records[2].detail == "Wall sides seen 6 of 9" && g.records[2].seconds == 121) + #expect(g.records[1].title == "Turn all the way round") + } + + private func metrics(peak: Double = 0.4, walls: Int = 11, dirs: Int = 12, lux: Double = 400, loop: Bool = true, + spread: Double = 5) -> CaptureMetrics { + CaptureMetrics(directionsCovered: dirs, keyframesKept: 100, keyframesDroppedBlurry: 2, trackingNormalRatio: 0.98, + meanSpeedMps: 0.3, peakSpeedMps: peak, slowDownCues: 0, areaWalkedM2: 8, meanLightLux: lux, + wallsCovered: walls, floorSeenM2: 4, mapStatus: .mapped, loopClosed: loop, compassSpreadDeg: spread) + } + + @Test func aGoodScanNeedsNoTips() { #expect(ScanTips.make(metrics()).isEmpty) } + + @Test func missedHeightZonesGetATip() { + var m = metrics() + m.zonesSeen = 12 + m.zonesTotal = 40 + #expect(ScanTips.make(m) == ["Height zones seen 12 of 40. Point the phone at knee height and up at shelves too, not only straight ahead."]) + } + + @Test func tipsSayWhatToDoBetterAtMostThree() { + let tips = ScanTips.make(metrics(peak: 0.9, walls: 5, dirs: 7, lux: 50, loop: false, spread: 40)) + #expect(tips.count == 3) + #expect(tips[0].hasPrefix("You walked at up to 0.9 metres a second")) + #expect(tips[1].hasPrefix("Walls were seen on 5 of 12 sides")) + #expect(ScanTips.light(80) == "Dim" && ScanTips.light(400) == "OK") + } + + @Test func scansSavedBeforeStepsWereRecordedStillOpen() throws { + let old = #"{"directions_covered":10,"keyframes_kept":50,"keyframes_dropped_blurry":3,"tracking_normal_ratio":0.95,"mean_speed_mps":0.3,"peak_speed_mps":0.5,"slow_down_cues":1,"area_walked_m2":6,"mean_light_lux":300,"grade":"good"}"# + let m = try JSONDecoder().decode(CaptureMetrics.self, from: Data(old.utf8)) + #expect(m.steps == nil && m.points == nil && m.directionsCovered == 10) + } +} + +@Suite("How high to hold the phone") +struct PhoneHeightTests { + @Test func chestHeightToLookRoundWaistHeightToSweep() { + #expect(ScanGuide.Step.alongWalls.heightLabel == "Chest height, 1.1 to 1.5 m") + #expect(ScanGuide.Step.sweepDown.heightLabel == "Waist height, 0.8 to 1.1 m") + #expect(HeightCoach.check(1.35, for: .alongWalls) == .good) // a 180 cm person's chest + #expect(HeightCoach.check(0.9, for: .alongWalls) == .tooLow) + #expect(HeightCoach.check(1.35, for: .sweepDown) == .tooHigh) + } + + @Test func itWaitsBeforeSayingAnythingAndDoesntNag() { + var coach = HeightCoach() + #expect(coach.update(t: 0, height: nil, step: .alongWalls) == nil) // floor not found yet + #expect(coach.update(t: 1, height: 0.8, step: .alongWalls) == nil) + #expect(coach.update(t: 4.5, height: 0.8, step: .alongWalls) == "Raise the phone to chest height.") + #expect(coach.update(t: 6, height: 0.8, step: .alongWalls) == nil) // not again so soon + #expect(coach.update(t: 7, height: 1.3, step: .alongWalls) == nil) // back in the band + #expect(coach.update(t: 30, height: 1.4, step: .sweepDown) == nil) + #expect(coach.update(t: 34, height: 1.4, step: .sweepDown) == "Lower the phone to waist height.") + } +} + +@Suite("Height zones and columns") +struct ZoneCoverageTests { + @Test func pointsLandInTheirColumnAndHeightBand() { + var z = ZoneCoverage() + z.add([SIMD3(0.5, -1.4, 0.5), SIMD3(0.5, -0.6, 0.2), SIMD3(1.5, -1.0, 0.5)], floorY: -1.5) // floor, waist; knee + #expect(z.seen[.init(x: 0, z: 0)] == [0, 2]) + #expect(z.seen[.init(x: 1, z: 0)] == [1]) + #expect((z.zonesSeen, z.zonesTotal) == (3, 10)) + } + + @Test func bandsAreRecallsHeightBands() { + #expect(ZoneCoverage.band(0.1) == 0 && ZoneCoverage.band(0.5) == 1 && ZoneCoverage.band(0.9) == 2) + #expect(ZoneCoverage.band(1.3) == 3 && ZoneCoverage.band(1.8) == 4 && ZoneCoverage.band(2.5) == nil) + } + + @Test func itSaysWhichHeightIsLeastSeen() { + var z = ZoneCoverage() + z.add([SIMD3(0.5, 0.1, 0.5), SIMD3(0.5, 0.9, 0.5), SIMD3(0.5, 1.3, 0.5), SIMD3(0.5, 1.8, 0.5)], floorY: 0) + #expect(z.weakestBand == "knee") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TextInputTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TextInputTests.swift new file mode 100644 index 0000000..98e98e4 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TextInputTests.swift @@ -0,0 +1,47 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("What text fields keep") +struct TextInputTests { + @Test func aFieldStopsAtItsLimit() { + #expect(TextInput.clean(String(repeating: "a", count: 500), limit: TextInput.question).count == 300) + #expect(TextInput.clean("Living room", limit: TextInput.roomName) == "Living room") + } + + @Test func pastedControlCharactersAndLineBreaksGo() { + #expect(TextInput.clean("Living\u{0007} room\n", limit: 60) == "Living room") + #expect(TextInput.clean("in the\nblue pouch\u{0000}", limit: 200, multiline: true) == "in the\nblue pouch") + } + + @Test func bidiControlsArePastedAway() { + // U+202E would show "txt.exe" as "exe.txt": a Cf format character, which controlCharacters includes + #expect(TextInput.clean("key\u{202E}fdp.exe", limit: 60) == "keyfdp.exe") + #expect(TextInput.clean("a\u{200F}b\u{2066}c", limit: 60) == "abc") + } + + @Test func returnInAOneLineFieldIsNoticedNotInANote() { + #expect(TextInput.pressedReturn("keys\n", multiline: false)) + #expect(!TextInput.pressedReturn("keys", multiline: false)) + #expect(!TextInput.pressedReturn("in the\nblue pouch", multiline: true)) // a note may have lines + } + + @Test func aKeyLongerThanTheFieldIsNoticed() { + #expect(TextInput.keyTooLong(String(repeating: "k", count: 129))) + #expect(!TextInput.keyTooLong(String(repeating: "k", count: 128) + "\n")) + } + + @Test func emojiAndAccentsCountAsOneCharacterEach() { + #expect(TextInput.clean("Café 🔑🔑", limit: 6) == "Café 🔑") + } + + @Test func aHomeNameIsSentAsTheMacSpellsIt() { + #expect(TextInput.homeId("My Flat") == "my-flat") + #expect(TextInput.homeId(" Grandma's house #2 ") == "grandma-s-house-2") + #expect(TextInput.homeId("�ódź") == "lodz") // spelt in the ASCII letters the Mac accepts, not dropped + #expect(TextInput.homeId("Küche 2") == "kuche-2") + #expect(TextInput.homeId("Квартира") == "kvartira") + #expect(Connection(address: "", key: "", home: " ").homeId == "flat") + #expect(Connection(address: "", key: "", home: "Studio Flat").homeId == "studio-flat") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ThingGroupsTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ThingGroupsTests.swift index 147f547..d1c32ef 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ThingGroupsTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/ThingGroupsTests.swift @@ -78,6 +78,12 @@ struct ThingGroupsTests { #expect(ThingGroups.group(things, by: .kind) == ThingGroups.byKind(things)) } + @Test func roomsForMapsMostThingsFirstThenScannedOnes() { + // "Somewhere in the home" has no map; a scanned room with nothing found yet still gets one, once + #expect(ThingGroups.rooms(things, scanned: ["Hallway", "Kitchen"]) == ["Living room", "Kitchen", "Hallway"]) + #expect(ThingGroups.rooms([], scanned: ["Hallway", "Hallway"]) == ["Hallway"]) + } + @Test func nothingToGroup() { #expect(ThingGroups.byPlace([]).isEmpty && ThingGroups.byHeight([]).isEmpty && ThingGroups.byKind([]).isEmpty) } diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TrustTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TrustTests.swift new file mode 100644 index 0000000..41be118 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/TrustTests.swift @@ -0,0 +1,28 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("What Recall trusts, in words") +struct TrustTests { + private func thing(photos: Int?, trusted: Bool, confidence: Double = 0.55) -> Thing { + Thing(id: "w", label: "wallet", color: "green", heightM: 0.4, heightBand: "knee", confidence: confidence, + needsReview: !trusted, photos: photos, trusted: trusted, seenAt: Date(timeIntervalSince1970: 0)) + } + + @Test func theEvidenceSaysHowManyPhotosAndHowSure() { + #expect(thing(photos: 1, trusted: false).evidence == "Seen in 1 photo, 55 percent sure") + #expect(thing(photos: 19, trusted: true, confidence: 0.63).evidence == "Seen in 19 photos, 63 percent sure") + #expect(thing(photos: nil, trusted: true).evidence == "55 percent sure") // kept before photos were counted + } + + @Test func aThingSeenOnceSaysSoInsteadOfCheck() { + #expect(thing(photos: 1, trusted: false).checkMark == "Seen once") + #expect(thing(photos: 2, trusted: false).checkMark == "Check") // Careful wants 3 + #expect(thing(photos: 4, trusted: true).checkMark == "Check") // trusted but unsure + } + + @Test func everyLevelExplainsItself() { + #expect(TrustLevel.allCases.map(\.title) == ["Careful", "Balanced", "Everything"]) + #expect(TrustLevel.allCases.allSatisfy { !$0.explanation.isEmpty }) + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/VoiceChoiceTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/VoiceChoiceTests.swift new file mode 100644 index 0000000..0d1d2b6 --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallDomainTests/VoiceChoiceTests.swift @@ -0,0 +1,38 @@ +import Foundation +import Testing +@testable import RecallDomain + +@Suite("Choosing the voice") +struct VoiceChoiceTests { + private let voices = [ + VoiceOption(id: "daniel", name: "Daniel", language: "en-GB", quality: 0, gender: .male), + VoiceOption(id: "ava-p", name: "Ava", language: "en-US", quality: 2, gender: .female), + VoiceOption(id: "serena-p", name: "Serena", language: "en-GB", quality: 2, gender: .female), + VoiceOption(id: "jamie-e", name: "Jamie", language: "en-GB", quality: 1, gender: .male), + VoiceOption(id: "bells", name: "Bells", language: "en-US", quality: 2, gender: .unspecified, special: true), + VoiceOption(id: "anna", name: "Anna", language: "de-DE", quality: 2, gender: .female), + ] + + @Test func automaticPicksTheBestQualityInTheOwnersRegion() { + #expect(VoiceChoice.best(voices, preference: .automatic, language: "en-GB")?.id == "serena-p") + #expect(VoiceChoice.best(voices, preference: .automatic, language: "en-US")?.id == "ava-p") + } + + @Test func aGenderPreferenceIsHonouredAndFallsBackRatherThanGoingSilent() { + #expect(VoiceChoice.best(voices, preference: .male, language: "en-GB")?.id == "jamie-e") // enhanced beats default + #expect(VoiceChoice.best(voices, preference: .female, language: "en-GB")?.id == "serena-p") + let onlyFemale = voices.filter { $0.gender != .male } + #expect(VoiceChoice.best(onlyFemale, preference: .male, language: "en-GB")?.id == "serena-p") + } + + @Test func noveltyVoicesAndOtherLanguagesAreNeverChosen() { + let funny = [VoiceOption(id: "bells", name: "Bells", language: "en-US", quality: 2, gender: .unspecified, special: true)] + #expect(VoiceChoice.best(funny, preference: .automatic, language: "en-US") == nil) + #expect(VoiceChoice.best(voices, preference: .automatic, language: "de-DE")?.id == "anna") + } + + @Test func aVoiceSaysItsQuality() { + #expect(voices[1].title == "Ava (Premium)") + #expect(voices[0].title == "Daniel") + } +} diff --git a/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/ApiCallsTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/ApiCallsTests.swift index 3074a1f..1b2c816 100644 --- a/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/ApiCallsTests.swift +++ b/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/ApiCallsTests.swift @@ -26,6 +26,26 @@ private func api(_ t: StubTransport) -> RecallAPI { @Suite("Calls to the Mac") struct ApiCallsTests { + @Test func theDeviceKeyOnlyGoesToThisMac() { + let a = api(StubTransport(.ok)) + #expect(a.partRequest(path: "/scans/s/parts/1", sha256: "x").url?.absoluteString == "http://mac.local:8000/scans/s/parts/1") + #expect(a.partRequest(path: "http://mac.local:8000/scans/s/parts/2", sha256: "x").url?.host == "mac.local") + let elsewhere = a.partRequest(path: "https://evil.example/steal", sha256: "x") + #expect(elsewhere.url?.host == "mac.local" && elsewhere.url?.path == "/steal") // only its path, on this Mac + #expect(a.partRequest(path: "http://mac.local:9999/x", sha256: "x").url?.port == 8000) // another port too + } + + @Test func theMacsSentenceIsCleanedAndCapped() { + #expect(RecallAPI.clean("No such scan.\u{1B}[2J\n") == "No such scan.[2J") + let long = RecallAPI.clean(String(repeating: "a", count: 1000)) + #expect(long.count == RecallAPI.maxDetail && long.hasSuffix("…")) + } + + @Test func callsGiveUpAfterThirtySeconds() { + #expect(RecallAPI.session.configuration.timeoutIntervalForRequest == 30) + #expect(api(StubTransport(.ok)).request("ask", method: "POST", contentType: "x", body: Data()).timeoutInterval == 30) + } + @Test func eraseSendsDeleteAndAcceptsGoneAlready() async throws { let done = StubTransport(.noContent) try await api(done).erase(scan: "s1") @@ -40,6 +60,20 @@ struct ApiCallsTests { } } + @Test func roomMapDecodesTheExportDocument() throws { + // a 2 x 1 PGM: one free cell, one occupied; one thing at the free cell + let pgm = Data("P5\n2 1\n255\n".utf8) + Data([254, 0]) + let j: [String: Any] = [ + "room": "Kitchen", + "map": ["pgm_base64": pgm.base64EncodedString(), "resolution": 0.1, "origin": [0.0, 0.0, 0.0]], + "objects": [["label": "mug", "color": "white", "position": [0.05, 0.05, 0.9]]], + "room_model": [:], + ] + let p = try RecallAPI.package(j) + #expect(p.room == "Kitchen" && p.map.width == 2 && p.map.cell(row: 0, column: 1) == .occupied) + #expect(p.things.map(\.title) == ["white mug"]) + } + @Test func privacyDecodes() async throws { let json = """ {"stored_on": "This Mac", "robot_access": false, "audit_days": 90, "sent": [ diff --git a/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/LimitsMatchTheSpecTests.swift b/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/LimitsMatchTheSpecTests.swift new file mode 100644 index 0000000..418137d --- /dev/null +++ b/apps/ios/Packages/RecallKit/Tests/RecallNetworkingTests/LimitsMatchTheSpecTests.swift @@ -0,0 +1,28 @@ +import Foundation +import RecallDomain +import Testing + +/// The phone's text limits are the Mac's: read from the openapi.json the client is generated from. +@Suite("Text limits match the Mac's spec") +struct LimitsMatchTheSpecTests { + private func schema(_ name: String) throws -> [String: Any] { + let url = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent().appending(path: "Sources/RecallNetworking/openapi.json") + let spec = try #require(try JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any]) + let schemas = try #require((spec["components"] as? [String: Any])?["schemas"] as? [String: Any]) + return try #require((schemas[name] as? [String: Any])?["properties"] as? [String: Any]) + } + + private func maxLength(_ schema: [String: Any], _ field: String) -> Int? { + (schema[field] as? [String: Any])?["maxLength"] as? Int + } + + @Test func everyLimitIsTheMacs() throws { + #expect(maxLength(try schema("AskRequest"), "question") == TextInput.question) + #expect(maxLength(try schema("ScanCreate"), "room") == TextInput.roomName) + #expect(maxLength(try schema("ObjectCorrection"), "name") == TextInput.thingName) + #expect(maxLength(try schema("ObjectCorrection"), "note") == TextInput.note) + let homePattern = (try schema("AskRequest")["home_id"] as? [String: Any])?["pattern"] as? String + #expect(homePattern == "^[a-z0-9-]{1,\(TextInput.homeId)}$") + } +} diff --git a/apps/ios/Recall/App/AppContainer.swift b/apps/ios/Recall/App/AppContainer.swift index df5d406..c26ec85 100644 --- a/apps/ios/Recall/App/AppContainer.swift +++ b/apps/ios/Recall/App/AppContainer.swift @@ -30,9 +30,12 @@ final class AppContainer { let settings: SettingsModel let things: ThingsModel let scans: ScansModel + let roomMaps: RoomMapsModel + let trust: TrustModel + private var watch: WatchRelay? init() { - let store = DefaultsConnectionStore() + let store: any ConnectionStore = UITestSamples.enabled ? UITestSamples.Connections() : DefaultsConnectionStore() settings = SettingsModel(store: store, server: server, pair: PairDevice(store: store), test: TestConnection(check: server), speech: speech) things = ThingsModel(load: LoadThings(memory: server), correct: CorrectThing(memory: server), @@ -41,6 +44,15 @@ final class AppContainer { upload: UploadScan(files: files, uploads: Self.uploader, server: server, speech: speech), track: TrackProcessing(processing: server, memory: server, speech: speech), processing: server, settings: settings, things: things, speech: speech, clock: clock) + roomMaps = RoomMapsModel(source: UITestSamples.enabled ? UITestSamples.Maps() : server, settings: settings) + trust = TrustModel(source: UITestSamples.enabled ? UITestSamples.Trust() : server) + if !UITestSamples.enabled { + // the Watch asks through the phone: the same connection, key and trust level as Find + let settings = settings + watch = WatchRelay(answer: AnswerWatch(memory: server, guide: server, transcriber: server, routes: server), + home: { await MainActor.run { settings.connection.homeId } }) + watch?.start() + } if UITestSamples.logSpeech { speech.onSay = { line in UITestSamples.SpokenLog.shared.lines.append(line) } } @@ -50,6 +62,22 @@ final class AppContainer { } } + /// "See it in AR" for one room: the shared locator (one ARKit session at a time), stopped from guiding first. + func arLabelsModel(room: String) -> ARLabelsModel { + idle?.cancel() + let scans = scans + return ARLabelsModel(room: room, home: settings.connection.homeId, source: arLabelsSource(), locator: locator, + roomMap: { scans.roomMap(scan: $0) }, speech: speech) + } + + func arLabelsSource() -> any ArLabelsSource { + UITestSamples.enabled ? UITestSamples.Labels() : server + } + + func robotPackages() -> any RobotPackages { + UITestSamples.enabled ? UITestSamples.Packages() : server + } + func evidenceSource() -> any EvidenceSource { UITestSamples.enabled ? UITestSamples.EvidenceSample() : server } @@ -108,7 +136,7 @@ struct SystemClock: Clock { /// The Mac, as every Mac-facing port, reached with the current connection settings. Changing Settings swaps the /// client underneath; a call made before setup throws the setup issue, in words. final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, ConnectionCheck, ScanServer, RobotSharing, - PrivacySource, EvidenceSource, @unchecked Sendable { + RoomMaps, RobotPackages, CompassRoutes, ArLabelsSource, TrustSettings, PrivacySource, EvidenceSource, @unchecked Sendable { private let lock = NSLock() private var api: RecallAPI? private var issue: SetupIssue = .nothingSet @@ -145,9 +173,23 @@ final class LiveServer: HomeMemory, GuideService, Transcriber, ScanProcessing, C func verify(session: String, frame: ArrivalFrame) async throws -> ArrivalCheck { try await current().verify(session: session, frame: frame) } + func robotPackage(home: String, room: String?) async throws -> Data { + try await current().robotPackage(home: home, room: room) + } + func arLabels(home: String, room: String) async throws -> ArLabels { + try await current().arLabels(home: home, room: room) + } + func compassRoute(home: String, thing: String) async throws -> CompassRoute { + try await current().compassRoute(home: home, thing: thing) + } + func trustLevel() async throws -> TrustLevel { try await current().trustLevel() } + func setTrustLevel(_ level: TrustLevel) async throws -> TrustLevel { try await current().setTrustLevel(level) } func robotAccess() async throws -> RobotAccessStatus { try await current().robotAccess() } func setRobotAccess(_ on: Bool) async throws -> RobotAccessStatus { try await current().setRobotAccess(on) } func robotMap(home: String) async throws -> RobotMapPackage { try await current().robotMap(home: home) } + func roomMap(home: String, room: String) async throws -> RobotMapPackage { + try await current().roomMap(home: home, room: room) + } func activity(limit: Int) async throws -> [AuditEntry] { try await current().activity(limit: limit) } func transcribe(wav: Data) async throws -> String { try await current().transcribe(wav: wav) } func status(scan: String) async throws -> ScanReport { try await current().status(scan: scan) } diff --git a/apps/ios/Recall/App/RecallApp.swift b/apps/ios/Recall/App/RecallApp.swift index 2f4ee78..31b80d6 100644 --- a/apps/ios/Recall/App/RecallApp.swift +++ b/apps/ios/Recall/App/RecallApp.swift @@ -9,6 +9,10 @@ struct RecallApp: App { @UIApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate @State private var container = AppContainer() + init() { + TapFeedback.play = { TapSound.shared.play(.press) } // every shared button style gives the soft tap + } + var body: some Scene { WindowGroup { ThemedRoot() @@ -16,6 +20,8 @@ struct RecallApp: App { .environment(container.settings) .environment(container.things) .environment(container.scans) + .environment(container.roomMaps) + .environment(container.trust) .environment(\.container, container) } } @@ -34,12 +40,18 @@ private struct ThemedRoot: View { @Environment(ScansModel.self) private var scans @Environment(\.container) private var container @Environment(\.colorSchemeContrast) private var systemContrast + @Environment(\.scenePhase) private var scenePhase @AppStorage(Theme.appearanceKey) private var appearance = Appearance.system @AppStorage(Theme.highContrastKey) private var highContrast = false var body: some View { @Bindable var router = router - TabView(selection: $router.tab) { + // a tap on the bar plays the tap sound; switching tabs in code ("Open Settings") stays quiet + let tabs = Binding(get: { router.tab }, set: { tab in + if tab != router.tab { TapSound.shared.play() } + router.tab = tab + }) + TabView(selection: tabs) { // first in the bar, but the app opens on Find: asking stays one step away Tab("Devices", systemImage: "apps.iphone", value: AppTab.devices) { DevicesView() } Tab("Find", systemImage: "location.magnifyingglass", value: AppTab.find) { FindView() } @@ -61,7 +73,21 @@ private struct ThemedRoot: View { .id(highContrast) .onChange(of: highContrast, initial: true) { Theme.highContrast = highContrast } .task { await scans.start() } + // back from the background: what finished meanwhile updates the cards without talking + .onChange(of: scenePhase) { _, phase in + if phase == .active { scans.becameActive(); KeyboardDismissal.shared.install() } + } + .task { KeyboardDismissal.shared.install() } // a tap outside a text field hides the keyboard, everywhere .onOpenURL { settings.pair(with: $0) } + .alert("Pair with this Mac?", isPresented: Binding(get: { settings.pendingPairing != nil }, + set: { if !$0 { settings.cancelPairing() } }), + presenting: settings.pendingPairing) { _ in + Button("Pair") { settings.confirmPairing() } + .accessibilityIdentifier("pairing.confirm") + Button("Cancel", role: .cancel) { settings.cancelPairing() } + } message: { pairing in + Text(pairing.summary + " Only pair with your own Mac.") + } } } diff --git a/apps/ios/Recall/App/UITestSamples.swift b/apps/ios/Recall/App/UITestSamples.swift index 4064e4b..0450c79 100644 --- a/apps/ios/Recall/App/UITestSamples.swift +++ b/apps/ios/Recall/App/UITestSamples.swift @@ -9,6 +9,15 @@ enum UITestSamples { /// The usability tests read what was spoken from one invisible element ("spoken-log"). static let logSpeech = ProcessInfo.processInfo.arguments.contains("-UITestSpokenLog") + /// A paired-looking Mac, whatever an earlier run left in the simulator's settings or Keychain (the key outlives an + /// uninstall there), so every screen starts the same and shows no setup notice. Nothing listens on port 9, so a + /// request that slips past the samples fails at once instead of reaching a real Mac. Changes last until the app quits. + final class Connections: ConnectionStore, @unchecked Sendable { + private var connection = Connection(address: "127.0.0.1:9", key: "ui-test-sample-key", home: "flat") + func load() -> Connection { connection } + func save(_ connection: Connection) { self.connection = connection } + } + /// Every line spoken since launch, newest last. @MainActor @Observable final class SpokenLog { static let shared = SpokenLog() @@ -94,6 +103,32 @@ enum UITestSamples { } } + /// Labels for "See it in AR" in the UI tests: one of each colour (the simulator has no ARKit, so only the list shows). + struct Labels: ArLabelsSource { + func arLabels(home: String, room: String) async throws -> ArLabels { + ArLabels(scanId: "scan-2", room: room, floorY: 0, labels: [ + ArLabel(id: "s1", title: "beige keys", place: "near the desk, at waist height", confidence: 0.85, photos: 6, + trusted: true, needsReview: false, position: SIMD3(0.5, 0.8, -1)), + ArLabel(id: "s2", title: "grey bag", place: "near the desk, at knee height", confidence: 0.6, photos: 3, + trusted: true, needsReview: true, position: SIMD3(-0.4, 0.4, -1.2)), + ArLabel(id: "s9", title: "green wallet", place: "near the cabinet, at knee height", confidence: 0.55, + photos: 1, trusted: false, needsReview: true, position: SIMD3(1.2, 0.4, -2)), + ]) + } + } + + /// A robot package for the UI tests: a few bytes standing in for the Mac's .zip. + struct Packages: RobotPackages { + func robotPackage(home: String, room: String?) async throws -> Data { Data("sample robot package".utf8) } + } + + /// The trust level for the UI tests: Balanced, kept in memory. + final class Trust: TrustSettings, @unchecked Sendable { + private var level = TrustLevel.balanced + func trustLevel() async throws -> TrustLevel { level } + func setTrustLevel(_ level: TrustLevel) async throws -> TrustLevel { self.level = level; return level } + } + /// Robot Access for the UI tests: kept in memory, with a small made-up map. final class Sharing: RobotSharing, @unchecked Sendable { private var on = false @@ -124,4 +159,20 @@ enum UITestSamples { exportURL: "http://mac.local:8000/homes/flat/robot-export") } } + + /// Room maps for the UI tests: a small made-up map for each sample room, its things as dots. + struct Maps: RoomMaps { + func roomMap(home: String, room: String) async throws -> RobotMapPackage { + guard room == "Living room" || room == "Kitchen" else { throw RecallError.status(404) } + // a 10 x 6 room: a wall along the top, unseen along the bottom, one piece of furniture + var shades = [UInt8](repeating: 0, count: 10) + [UInt8](repeating: 254, count: 40) + [UInt8](repeating: 128, count: 10) + shades[room == "Kitchen" ? 22 : 25] = 0 + let map = RobotMap(width: 10, height: 6, resolution: 0.25, origin: (0, 0), shades: shades) + let here = things.filter { $0.room == room } + let dots = here.enumerated().map { i, t in + RobotMapPackage.Thing(label: t.label, color: t.color, x: 0.4 + Double(i) * 0.5, y: 0.6) + } + return RobotMapPackage(room: room, map: map, things: dots, files: [:]) + } + } } diff --git a/apps/ios/Recall/App/WatchRelay.swift b/apps/ios/Recall/App/WatchRelay.swift new file mode 100644 index 0000000..f0c338a --- /dev/null +++ b/apps/ios/Recall/App/WatchRelay.swift @@ -0,0 +1,57 @@ +import Foundation +import RecallApplication +import RecallDomain +import WatchConnectivity + +/// The Watch's way to the Mac: requests arrive over WatchConnectivity (a spoken question as a file, everything else as +/// a message), AnswerWatch answers them with the phone's own connection, and the reply goes back to the Watch. The +/// phone app wakes in the background for this, so the phone can stay in a pocket. +final class WatchRelay: NSObject, WCSessionDelegate, @unchecked Sendable { + private let answer: AnswerWatch + private let home: @Sendable () async -> String + + init(answer: AnswerWatch, home: @escaping @Sendable () async -> String) { + self.answer = answer; self.home = home + } + + func start() { + guard WCSession.isSupported() else { return } + WCSession.default.delegate = self + WCSession.default.activate() + } + + // a request with a reply handler: the things for the dial, or a route to a picked thing + func session(_ session: WCSession, didReceiveMessage message: [String: Any], replyHandler: @escaping ([String: Any]) -> Void) { + let reply = Reply(replyHandler) + guard let data = message["request"] as? Data, let request = try? JSONDecoder().decode(WatchRequest.self, from: data) else { + return reply(WatchReply(say: "The phone didn't understand the Watch. Update both apps.")) + } + Task { reply(await answer(request, home: await home())) } + } + + // a spoken question: the recording arrives as a file; the answer goes back as a message carrying the same id + func session(_ session: WCSession, didReceive file: WCSessionFile) { + let id = file.metadata?["id"] as? String ?? "" + // a question is a few seconds of 16 kHz audio; anything far bigger isn't one, and isn't read into memory + let size = (try? file.fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + let audio = size <= WatchReply.maxAudioBytes ? try? Data(contentsOf: file.fileURL) : nil + Task { + let reply = await answer(.ask(audioFile: id), home: await home(), audio: audio) + guard let data = try? JSONEncoder().encode(reply) else { return } + WCSession.default.sendMessage(["reply": data, "id": id], replyHandler: nil) + } + } + + func session(_ session: WCSession, activationDidCompleteWith activationState: WCSessionActivationState, error: Error?) {} + func sessionDidBecomeInactive(_ session: WCSession) {} + func sessionDidDeactivate(_ session: WCSession) { WCSession.default.activate() } // a new Watch was paired +} + +/// WatchConnectivity's reply handler, carried into the task that answers (it is called once, from any thread). +private struct Reply: @unchecked Sendable { + let send: ([String: Any]) -> Void + init(_ send: @escaping ([String: Any]) -> Void) { self.send = send } + func callAsFunction(_ reply: WatchReply) { + send(["reply": (try? JSONEncoder().encode(reply)) ?? Data()]) + } +} diff --git a/apps/ios/Recall/Features/AR/ARLabelsModel.swift b/apps/ios/Recall/Features/AR/ARLabelsModel.swift new file mode 100644 index 0000000..da326b2 --- /dev/null +++ b/apps/ios/Recall/Features/AR/ARLabelsModel.swift @@ -0,0 +1,103 @@ +import ARKit +import Foundation +import Observation +import RecallCapture +import RecallDomain +import SwiftUI + +/// "See it in AR": recognise the room in its saved map, then float a label over everything Recall found there. The +/// labels are SAM 3.1's results from the Mac, placed by MapAnything's geometry; nothing is recognised on the phone. +@MainActor @Observable +final class ARLabelsModel { + enum Phase: Equatable { case loading, locating, showing, listOnly(String), failed(String) } + + private(set) var phase = Phase.loading + private(set) var labels: [ArLabel] = [] + var look = CameraLook.camera + let room: String + let locator: ARLocator + + private let source: any ArLabelsSource + private let home: String + private let roomMap: (String) -> Data? + private let speech: any SpeechOutput + private var task: Task? + + init(room: String, home: String, source: any ArLabelsSource, locator: ARLocator, roomMap: @escaping (String) -> Data?, + speech: any SpeechOutput) { + self.room = room; self.home = home; self.source = source; self.locator = locator; self.roomMap = roomMap + self.speech = speech + } + + func start() { + task = Task { [weak self] in await self?.run() } + } + + func stop() { + task?.cancel() + locator.clearLabels() + locator.stop() + } + + /// What the camera points at, said aloud (Magic Tap). + func sayInView() { + guard let label = locator.labelInView() else { + speech.say("Point the phone at a label.", interrupt: true) + return + } + speech.say(label.spoken, interrupt: true) + } + + private func run() async { + do { + let found = try await source.arLabels(home: home, room: room) + labels = found.labels + guard ARLocator.isSupported, let map = roomMap(found.scanId) else { + return phase = .listOnly("The room map of this scan isn't on this iPhone, so here is the list.") + } + phase = .locating + speech.say("Point the phone around \(room) so I can recognise it.", interrupt: true) + locator.start(roomMap: map) + // ARKit says the room is recognised when tracking turns normal in the saved map + for _ in 0..<50 { + try await Task.sleep(for: .milliseconds(500)) + if locator.reading()?.tracking == .normal { break } + } + guard locator.reading()?.tracking == .normal else { + return phase = .listOnly("I couldn't recognise \(room) from here, so here is the list.") + } + let tags = labels.map { (label: $0, tag: Self.tagImage($0)) } + locator.showLabels(tags, floorY: found.floorY) + phase = .showing + let n = labels.count == 1 ? "1 thing" : "\(labels.count) things" + speech.say("\(n) in \(room). Point at a label and double tap with two fingers to hear it.", interrupt: false) + } catch is CancellationError { + } catch { + phase = .failed(UserMessage.explain(error)) + } + } + + /// The tag as a picture: "GREY BAG" over "88% · 5 photos", dark glass with an edge in the label's colour. + static func tagImage(_ label: ArLabel) -> UIImage { + let renderer = ImageRenderer(content: ARTag(label: label)) + renderer.scale = 3 + return renderer.uiImage ?? UIImage() + } +} + +/// The scanner-style tag drawn into the AR view. +struct ARTag: View { + let label: ArLabel + var body: some View { + let colour = Color(ARLocator.colour(label.mark)) + VStack(alignment: .leading, spacing: 2) { + Text(label.headline).font(.system(size: 15, weight: .bold, design: .monospaced)) + Text(label.detail).font(.system(size: 12, weight: .medium, design: .monospaced)).opacity(0.85) + } + .foregroundStyle(.white) + .padding(.horizontal, 10).padding(.vertical, 6) + .background(Color(red: 0.05, green: 0.08, blue: 0.14).opacity(0.88), in: .rect(cornerRadius: 4)) + .overlay(RoundedRectangle(cornerRadius: 4).stroke(colour, lineWidth: 2)) + .overlay(alignment: .leading) { Rectangle().fill(colour).frame(width: 4).padding(.vertical, 2) } + } +} diff --git a/apps/ios/Recall/Features/AR/ARLabelsView.swift b/apps/ios/Recall/Features/AR/ARLabelsView.swift new file mode 100644 index 0000000..1f20511 --- /dev/null +++ b/apps/ios/Recall/Features/AR/ARLabelsView.swift @@ -0,0 +1,90 @@ +import ARKit +import RecallCapture +import RecallDesign +import RecallDomain +import SceneKit +import SwiftUI + +/// "See it in AR": the camera with a label over everything Recall found in the room, and the same things as a list +/// underneath, for VoiceOver and for anyone who can't hold the phone up. +struct ARLabelsView: View { + @State var model: ARLabelsModel + let close: () -> Void + + var body: some View { + VStack(spacing: 0) { + ZStack(alignment: .top) { + if ARLocator.isSupported { + LabelsCamera(session: model.locator.session, scene: model.locator.scene) + .ignoresSafeArea(edges: .top) + .accessibilityHidden(true) + LiveLook(session: model.locator.session, look: model.look) + } else { + Color.black.ignoresSafeArea(edges: .top) + } + VStack(spacing: 10) { + HStack { + Text(status).font(.subheadline.weight(.semibold)).foregroundStyle(.white) + .padding(.horizontal, 12).padding(.vertical, 8) + .background(.black.opacity(0.6), in: Capsule()) + .accessibilityIdentifier("ar-status") + Spacer() + Button("Done", action: close).buttonStyle(.glass).accessibilityIdentifier("ar-done") + } + if ARLocator.isSupported { + LookChips(look: $model.look, choices: [.camera, .edges]) + } + } + .padding(16) + } + .frame(maxHeight: .infinity) + List { + Section { + ForEach(model.labels) { label in + HStack(spacing: 12) { + Circle().fill(Color(ARLocator.colour(label.mark))).frame(width: 12, height: 12) + VStack(alignment: .leading, spacing: 2) { + Text(label.title.prefix(1).uppercased() + label.title.dropFirst()).font(.headline) + Text("\(label.place) · \(label.detail)").font(.subheadline).foregroundStyle(Tokens.muted) + } + } + .accessibilityElement(children: .ignore) + .accessibilityLabel(label.spoken) + .accessibilityIdentifier("ar-label-\(label.id)") + } + } header: { + Text("Found in \(model.room) · green: Recall tells you · amber: check · red: seen once") + .font(.footnote).textCase(nil) + } + } + .frame(maxHeight: 260) + } + .background(Tokens.canvas) + .onAppear { model.start() } + .onDisappear { model.stop() } + .accessibilityAction(.magicTap) { model.sayInView() } + } + + private var status: String { + switch model.phase { + case .loading: "Loading what Recall found…" + case .locating: "Point at the room…" + case .showing: "\(model.labels.count) things in \(model.room)" + case .listOnly(let why), .failed(let why): why + } + } +} + +/// The locator's camera feed with the labels drawn into its scene. +private struct LabelsCamera: UIViewRepresentable { + let session: ARSession + let scene: SCNScene + func makeUIView(context: Context) -> ARSCNView { + let v = ARSCNView(frame: .zero) + v.session = session + v.scene = scene + v.automaticallyUpdatesLighting = false + return v + } + func updateUIView(_ uiView: ARSCNView, context: Context) {} +} diff --git a/apps/ios/Recall/Features/Devices/DevicesView.swift b/apps/ios/Recall/Features/Devices/DevicesView.swift index 6a9f09f..d506198 100644 --- a/apps/ios/Recall/Features/Devices/DevicesView.swift +++ b/apps/ios/Recall/Features/Devices/DevicesView.swift @@ -1,7 +1,7 @@ import RecallDesign import SwiftUI -/// Where Recall runs: the iPhone today; the Apple Watch and Meta's AI glasses next. Every card says plainly whether it +/// Where Recall runs: the iPhone today; the Apple Watch and robots in preview; Meta's AI glasses planned. Every card says plainly whether it /// works yet (a word and an icon, never colour alone), so nothing planned reads as finished. No other company's logos, /// only their product names. struct DevicesView: View { @@ -10,47 +10,45 @@ struct DevicesView: View { var body: some View { NavigationStack { ScrollView { - VStack(alignment: .leading, spacing: 16) { - BrandHeader("Devices", subtitle: "One memory of your home, wherever you are") - Pitch() - SectionTitle("Where Recall runs", count: "\(Device.all.count) devices") - } - .padding(.horizontal, Tokens.gutter) - DeviceCarousel() - NavigationLink { RobotAccessView() } label: { - LinkCard(icon: "gearshape.2.fill", title: "Robot Access", - text: "Share your home map with a robot, or keep it private", id: "robot-access-link") - } - .buttonStyle(.plain) - .padding(.horizontal, Tokens.gutter) - .padding(.top, 8) - .accessibilityHint("Share your home map with robots, or turn it off.") - .padding(.top, 4) - NavigationLink { PrivacyView() } label: { - LinkCard(icon: "hand.raised.fill", title: "Your privacy", - text: "Your memory lives on your Mac. See what is sent where", id: "privacy-link") - } - .buttonStyle(.plain) + // one column on the shared rhythm: Tokens.stack between blocks, Tokens.sectionGap more above a section + VStack(alignment: .leading, spacing: Tokens.stack) { + Group { + BrandHeader("Devices", subtitle: "One memory, on every device") + Pitch() + SectionTitle("Where Recall runs", count: "\(Device.all.count) devices") + .padding(.top, Tokens.sectionGap) + } .padding(.horizontal, Tokens.gutter) - .padding(.top, 8) - .accessibilityHint("What stays on your Mac and iPhone, and what is sent where.") - VStack(alignment: .leading, spacing: 16) { - SectionTitle("Why it works") - Promises() - HStack(spacing: 12) { - Button { router.tab = .find } label: { Label("Ask now", systemImage: "location.magnifyingglass") } - .buttonStyle(SolidPillStyle(fill: Tokens.go, text: Tokens.onGo)) - .accessibilityHint("Opens Find, to ask where something is.") - Button { router.tab = .scans } label: { Label("Scan a room", systemImage: "viewfinder") } - .buttonStyle(SolidPillStyle(fill: Tokens.card, text: Tokens.accent, edge: Tokens.line)) - .accessibilityHint("Opens Scans, to record a room.") + DeviceCarousel() // full width: its cards peek in from the edges + Group { + NavigationLink { RobotAccessView() } label: { + LinkCard(icon: "gearshape.2.fill", title: "Robot Access", + text: "Share your home map with a robot, or keep it private", id: "robot-access-link") + } + .buttonStyle(.plainTap) + .accessibilityHint("Share your home map with robots, or turn it off.") + NavigationLink { PrivacyView() } label: { + LinkCard(icon: "hand.raised.fill", title: "Your privacy", + text: "Your memory lives on your Mac. See what is sent where", id: "privacy-link") + } + .buttonStyle(.plainTap) + .accessibilityHint("What stays on your Mac and iPhone, and what is sent where.") + SectionTitle("Why it works").padding(.top, Tokens.sectionGap) + Promises() + HStack(spacing: Tokens.stack) { + Button { router.tab = .find } label: { Label("Ask now", systemImage: "location.magnifyingglass") } + .buttonStyle(SolidPillStyle(fill: Tokens.go, text: Tokens.onGo)) + .accessibilityHint("Opens Find, to ask where something is.") + Button { router.tab = .scans } label: { Label("Scan a room", systemImage: "viewfinder") } + .buttonStyle(SolidPillStyle(fill: Tokens.card, text: Tokens.accent, edge: Tokens.line)) + .accessibilityHint("Opens Scans, to record a room.") + } + Text("The iPhone app works today; the Watch and robots are in preview, and the glasses are planned. Recall is an independent project, not affiliated with Apple or Meta.") + .font(.footnote).foregroundStyle(Tokens.muted) + .fixedSize(horizontal: false, vertical: true) } - Text("Only the iPhone app works today; the Watch and glasses are being built, and robots can use the shared map in a preview. Recall is an independent project, not affiliated with Apple or Meta.") - .font(.footnote).foregroundStyle(Tokens.muted) - .fixedSize(horizontal: false, vertical: true) + .padding(.horizontal, Tokens.gutter) } - .padding(.horizontal, Tokens.gutter) - .padding(.top, 8) .padding(.bottom, 24) } .tabScreen() @@ -88,7 +86,7 @@ private struct DeviceCarousel: View { } .contentShape(.rect) } - .buttonStyle(.plain) + .buttonStyle(.plainTap) .accessibilityLabel(device.name) .accessibilityHint("Shows it below.") .accessibilityAddTraits(on ? .isSelected : []) @@ -186,11 +184,11 @@ struct Device: Identifiable { points: ["Scan a room with LiDAR, led step by step by voice", "Ask out loud; hear clock-face turns with compass points", "A blue arrow over the thing on the camera, for a helper"]), - Device(id: "watch", name: "Apple Watch Ultra 4", short: "Watch", systemImage: "applewatch", status: .next, + Device(id: "watch", name: "Apple Watch Ultra 4", short: "Watch", systemImage: "applewatch", status: .preview, pitch: "Directions you can feel, with the phone in your pocket.", - points: ["A tap on the wrist for each turn, a buzz when you arrive", - "Ask from your wrist and hear the answer", - "Hands free for a cane or a guide dog"]), + points: ["Say what to find, or turn the crown to pick it", + "A tap on the wrist for each turn, a click each metre, a buzz when you arrive", + "Walks by its compass from where the room's scan began"]), Device(id: "glasses", name: "Meta AI glasses", short: "Glasses", systemImage: "eyeglasses", status: .planned, pitch: "Ask hands-free and be guided by what the glasses see.", points: ["Ask through the glasses; hear the answer in their speakers", diff --git a/apps/ios/Recall/Features/Find/FindView.swift b/apps/ios/Recall/Features/Find/FindView.swift index 2425975..05752ea 100644 --- a/apps/ios/Recall/Features/Find/FindView.swift +++ b/apps/ios/Recall/Features/Find/FindView.swift @@ -3,41 +3,44 @@ import RecallDesign import RecallDomain import SwiftUI -/// Find: ask where something is (typed or spoken), hear the answer, and be walked there; or pick a thing from the -/// list and go. +/// Find: ask where something is (typed or spoken), hear the answer, and be walked there; or pick a room's map and a +/// thing from the list below it, grouped by room and the furniture it's on or near, and go. struct FindView: View { @Environment(\.container) private var container @Environment(ThingsModel.self) private var things + @Environment(ScansModel.self) private var scans + @Environment(RoomMapsModel.self) private var maps @Environment(Router.self) private var router @State private var model: FindModel? + /// The room picked on its map card; nil shows every room. + @State private var room: String? + @State private var folded: Set = [] var body: some View { NavigationStack { ScrollView { - VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: Tokens.stack) { BrandHeader("Find", subtitle: "Ask where something is") if let model { AskCard(model: model) { router.guiding = $0 } } - SectionTitle("Your things", count: things.things.isEmpty ? nil : count) - .padding(.top, 12) + if !rooms.isEmpty { + SectionTitle("Your rooms", count: rooms.count == 1 ? "1 room" : "\(rooms.count) rooms") + .padding(.top, Tokens.sectionGap) + RoomCarousel(rooms: rooms, things: trusted, picked: $room) + } + + SectionTitle("Your things", count: trusted.isEmpty ? nil : count) + .padding(.top, Tokens.sectionGap) if let error = things.error { ErrorBox(text: error) - } else if things.things.isEmpty, things.loaded { + } else if trusted.isEmpty, things.loaded { EmptyState("shippingbox", "Nothing remembered yet. Scan a room and upload it first.") } - // one room: a plain list; several: a heading per room - let rooms = ThingGroups.byPlace(things.things) - ForEach(rooms) { room in - if rooms.count > 1 { - Text(room.title).font(.headline).foregroundStyle(Tokens.muted) - .padding(.top, 4) - .accessibilityAddTraits(.isHeader) - } - ForEach(room.things) { thing in - Button { router.guiding = thing } label: { ThingRow(thing: thing) } - .buttonStyle(.plain) - .accessibilityHint("Walks you there with spoken directions.") - } + // like Found: a heading per room (even just one), then a card per piece of furniture + ForEach(ThingGroups.byPlace(shown)) { group in + GroupSection(group: group, folded: folded.contains(group.id), inSpots: true, mode: .find, + toggle: { toggle(group.id) }, open: { router.guiding = $0 }, + guide: { router.guiding = $0 }) } } .padding(.horizontal, Tokens.gutter) @@ -48,11 +51,154 @@ struct FindView: View { if model == nil { model = container?.findModel() } if !things.loaded { await things.refresh() } } - .refreshable { await things.refresh() } + .refreshable { + maps.forget() + await things.refresh() + } + // a room that's gone (erased, or renamed) stops filtering the list + .onChange(of: rooms) { if let room, !rooms.contains(room) { self.room = nil } } + // a scan finished processing (a rescan, or a new room): its room's map has changed on the Mac + .onChange(of: processed) { maps.forget() } + .keyboardDismissable() // scroll or Done hides the keyboard; a tap outside does too (KeyboardDismissal) + } + } + + /// Rooms with things, most first, then rooms scanned on this phone that have nothing found yet. + private var rooms: [String] { + ThingGroups.rooms(trusted, scanned: scans.scans.filter { $0.state == .processed }.map(\.room)) + } + + private var processed: [String] { scans.scans.filter { $0.state == .processed }.map(\.id) } + + /// Find only offers what the owner's trust level counts (Found shows the rest, to check). + private var trusted: [Thing] { things.things.filter(\.trusted) } + + private var shown: [Thing] { room.map { r in trusted.filter { $0.room == r } } ?? trusted } + + private var count: String { + let n = shown.count + return (n == 1 ? "1 thing" : "\(n) things") + (room.map { " in \($0)" } ?? "") + } + + private func toggle(_ id: String) { + if folded.contains(id) { folded.remove(id) } else { folded.insert(id) } + } +} + +/// One map card per room, side by side: swipe to see the next room, tap one to show only its things below. +private struct RoomCarousel: View { + let rooms: [String] + let things: [Thing] + @Binding var picked: String? + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + if rooms.count > 1 { + FilterChip("All rooms", selected: picked == nil) { pick(nil) } + .accessibilityHint("Shows every room's things.") + } + ScrollView(.horizontal, showsIndicators: false) { + LazyHStack(spacing: 12) { + ForEach(rooms, id: \.self) { room in + RoomMapCard(room: room, count: things.filter { $0.room == room }.count, picked: picked == room) { + pick(picked == room ? nil : room) + } + // one room fills the width; with more, the next one peeks in so it's clear there's more + .containerRelativeFrame(.horizontal) { width, _ in rooms.count == 1 ? width : width * 0.86 } + } + } + .scrollTargetLayout() + } + .scrollTargetBehavior(.viewAligned) + .scrollClipDisabled() + .accessibilityIdentifier("room-maps") + RoomMapLegend().padding(.top, 2) } } - private var count: String { things.things.count == 1 ? "1 thing" : "\(things.things.count) things" } + private func pick(_ room: String?) { + withAnimation(reduceMotion ? nil : .snappy) { picked = room } + } +} + +/// A room's map in a card: its name, how many things, and the map with the things as dots. +private struct RoomMapCard: View { + let room: String + let count: Int + let picked: Bool + let tap: () -> Void + @Environment(RoomMapsModel.self) private var maps + @Environment(\.container) private var container + @State private var showingAR = false + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + Button(action: tap) { + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .firstTextBaseline) { + Text(room).font(.headline).foregroundStyle(Tokens.ink) + .fixedSize(horizontal: false, vertical: true) // wraps at large text instead of clipping + Spacer(minLength: 8) + Text(things).font(.subheadline).foregroundStyle(Tokens.muted) + .fixedSize(horizontal: false, vertical: true) + } + switch maps.maps[room] { + case .ready(let package): RoomMapCanvas(package: package).frame(height: 190) + case .notReady: + // grows with large text instead of clipping; the map itself keeps a fixed height + Text("Map not ready yet. It appears once your Mac has processed a scan of this room.") + .font(.subheadline).foregroundStyle(Tokens.muted).multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + .padding().frame(maxWidth: .infinity, minHeight: 190) + .background(Tokens.tile, in: .rect(cornerRadius: 14)) + case .loading, nil: + ProgressView().frame(maxWidth: .infinity).frame(height: 190) + .background(Tokens.tile, in: .rect(cornerRadius: 14)) + } + } + .contentShape(.rect) + } + .buttonStyle(.plainTap) + .accessibilityElement(children: .ignore) + .accessibilityLabel("\(room) map, \(things)\(mapState)") + .accessibilityValue(picked ? "Selected" : "Not selected") + .accessibilityHint(picked ? "Shows every room's things again." : "Shows only this room's things.") + .accessibilityAddTraits(picked ? [.isButton, .isSelected] : .isButton) + .accessibilityIdentifier("room-map-\(room)") + if case .ready = maps.maps[room] { + // the actions wrap under each other at large text sizes + ViewThatFits(in: .horizontal) { + HStack(spacing: 8) { seeInAR; RobotPackageShare(room: room) } + VStack(alignment: .leading, spacing: 8) { seeInAR; RobotPackageShare(room: room) } + } + } + } + .card() + .overlay(RoundedRectangle(cornerRadius: Tokens.cardRadius).stroke(Tokens.accent, lineWidth: picked ? 3 : 0)) + .task(id: maps.generation) { await maps.load(room) } + } + + private var seeInAR: some View { + Button { showingAR = true } label: { Label("See it in AR", systemImage: "arkit") } + .buttonStyle(SolidPillStyle(fill: Tokens.accentFill, text: Tokens.onAccent, compact: true)) + .accessibilityHint("Labels over everything Recall found in \(room), in the camera view, with a list.") + .accessibilityIdentifier("see-in-ar-\(room)") + .fullScreenCover(isPresented: $showingAR) { + if let container { ARLabelsView(model: container.arLabelsModel(room: room)) { showingAR = false } } + } + } + + private var things: String { count == 1 ? "1 thing" : "\(count) things" } + + /// What the card shows instead of a map, said too: VoiceOver hears every word a sighted person reads. + private var mapState: String { + switch maps.maps[room] { + case .notReady: ". Map not ready yet. It appears once your Mac has processed a scan of this room." + case .loading, nil: ". Loading the map." + case .ready: "" + } + } } /// The question, the two ways to ask it, and the answer. @@ -68,6 +214,7 @@ private struct AskCard: View { TextField("Your question", text: $model.question, prompt: Text("e.g. where are my keys?").foregroundStyle(Tokens.muted), axis: .vertical) .font(.body).foregroundStyle(Tokens.ink) + .limited($model.question, to: TextInput.question) .focused($typing) .submitLabel(.search) .onSubmit { Task { await model.submit() } } @@ -142,29 +289,3 @@ private struct AnswerCard: View { .card() } } - -/// A thing in the list: what it is, where it is, and a Go arrow. -private struct ThingRow: View { - let thing: Thing - - var body: some View { - HStack(spacing: 12) { - ThingIcon(thing: thing) - VStack(alignment: .leading, spacing: 3) { - Text(thing.title.capitalizedFirst).font(.headline).foregroundStyle(Tokens.ink) - Text(thing.place.capitalizedFirst).font(.subheadline).foregroundStyle(Tokens.muted) - .fixedSize(horizontal: false, vertical: true) - } - Spacer(minLength: 8) - Label("Go", systemImage: "figure.walk") - .font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.onAccent) - .padding(.horizontal, 14).frame(minHeight: 40) - .background(Tokens.accentFill, in: Capsule()) - .fixedSize() - } - .frame(minHeight: Tokens.minTarget) - .card(padding: 14) - // VoiceOver reads the row's own words: "Grey bag, near the desk, knee height, Go" - .accessibilityElement(children: .combine) - } -} diff --git a/apps/ios/Recall/Features/Found/EditThingView.swift b/apps/ios/Recall/Features/Found/EditThingView.swift index 0b9da6e..078e996 100644 --- a/apps/ios/Recall/Features/Found/EditThingView.swift +++ b/apps/ios/Recall/Features/Found/EditThingView.swift @@ -37,6 +37,11 @@ struct EditThingView: View { Text("About \(size[0]) by \(size[1]) by \(size[2]) centimetres.").font(.subheadline) .foregroundStyle(Tokens.muted) } + Text("\(thing.evidence) (SAM 3.1).").font(.subheadline).foregroundStyle(Tokens.muted) + if !thing.trusted { + Text("Recall won't mention it yet. Save to confirm it's there, or tell it it's not.") + .font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.danger) + } } .fixedSize(horizontal: false, vertical: true) } @@ -45,6 +50,7 @@ struct EditThingView: View { TextField("Your name for it", text: $name, prompt: Text("e.g. work bag").foregroundStyle(Tokens.muted)) .textInputAutocapitalization(.never) .accessibilityIdentifier("objectName") + .limited($name, to: TextInput.thingName) } header: { FormHeader("Your name for it") } footer: { FormNote("Optional. You can then ask for it by this name.") } @@ -75,14 +81,15 @@ struct EditThingView: View { Section { TextField("Note", text: $note, prompt: Text("e.g. in the blue pouch").foregroundStyle(Tokens.muted), axis: .vertical) - } header: { FormHeader("Note") } + .limited($note, to: TextInput.note, multiline: true) + } header: { FormHeader("Note") } footer: { FormNote("\(note.count) of \(TextInput.note) characters.") } if let error { Section { ErrorBox(text: error) } } Section { Button("Not there, forget it", systemImage: "trash", role: .destructive) { confirmRemove = true } .foregroundStyle(Tokens.danger) - .accessibilityHint("Removes it from memory.") + .accessibilityHint("Removes it, and later scans won't bring it back here.") .disabled(busy) } } @@ -107,6 +114,7 @@ struct EditThingView: View { .confirmationDialog("Forget the \(thing.title)?", isPresented: $confirmRemove, titleVisibility: .visible) { Button("Forget it", role: .destructive) { Task { await remove() } } } + .keyboardDismissable() // scroll or Done hides the keyboard; a tap outside does too (KeyboardDismissal) } } diff --git a/apps/ios/Recall/Features/Found/FoundView.swift b/apps/ios/Recall/Features/Found/FoundView.swift index 1924fff..b263559 100644 --- a/apps/ios/Recall/Features/Found/FoundView.swift +++ b/apps/ios/Recall/Features/Found/FoundView.swift @@ -26,7 +26,7 @@ struct FoundView: View { var body: some View { NavigationStack { ScrollView { - VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: Tokens.stack) { BrandHeader("Found", subtitle: "Check what Recall found") if things.loading && !things.loaded { ProgressView("Loading what Recall found…").frame(maxWidth: .infinity).padding(.vertical, 32) @@ -51,7 +51,7 @@ struct FoundView: View { } ForEach(ThingGroups.group(shown, by: groupBy)) { group in GroupSection(group: group, folded: folded.contains(group.id), inSpots: groupBy == .place, - toggle: { toggle(group.id) }, open: { editing = $0 }, + mode: .review, toggle: { toggle(group.id) }, open: { editing = $0 }, guide: { router.guiding = $0 }) } } @@ -63,6 +63,7 @@ struct FoundView: View { .task { if !things.loaded { await things.refresh() } } .refreshable { await things.refresh() } .sheet(item: $editing) { EditThingView(thing: $0) } + .keyboardDismissable() // scroll or Done hides the keyboard; a tap outside does too (KeyboardDismissal) } } @@ -96,86 +97,6 @@ extension ThingGroups.By { } } -/// A room (or height, or kind): its heading with counts, folding away with a tap, then its things in compact cards, -/// one card per piece of furniture when grouped by place. -private struct GroupSection: View { - let group: ThingGroup - let folded: Bool - let inSpots: Bool - let toggle: () -> Void - let open: (Thing) -> Void - let guide: (Thing) -> Void - @Environment(\.accessibilityReduceMotion) private var reduceMotion - - var body: some View { - VStack(alignment: .leading, spacing: 10) { - Button { - withAnimation(reduceMotion ? nil : .snappy) { toggle() } - } label: { - HStack(alignment: .firstTextBaseline, spacing: 8) { - Text(group.title).font(.title3.weight(.bold)).foregroundStyle(Tokens.ink) - .fixedSize(horizontal: false, vertical: true) - Spacer(minLength: 8) - Text(counts).font(.subheadline).foregroundStyle(Tokens.muted) - Image(systemName: "chevron.down").font(.footnote.weight(.bold)).foregroundStyle(Tokens.muted) - .rotationEffect(.degrees(folded ? -90 : 0)) - } - .frame(minHeight: 44) - .contentShape(.rect) - } - .buttonStyle(.plain) - .accessibilityElement(children: .ignore) - .accessibilityLabel(group.summary) - .accessibilityValue(folded ? "Folded" : "Open") - .accessibilityHint(folded ? "Shows its things." : "Hides its things.") - .accessibilityAddTraits([.isHeader, .isButton]) - .accessibilityIdentifier("group-\(group.id)") - if !folded { - if inSpots && !group.spots.isEmpty { - ForEach(group.spots) { spot in - Text(spot.title.uppercased()).font(.caption.weight(.bold)).foregroundStyle(Tokens.muted) - .padding(.top, 2) - .accessibilityAddTraits(.isHeader) - RowsCard(things: spot.things, atFurniture: true, open: open, guide: guide) - } - } else { - RowsCard(things: group.things, atFurniture: false, open: open, guide: guide) - } - } - } - .padding(.top, 6) - } - - private var counts: String { - let n = group.count == 1 ? "1 thing" : "\(group.count) things" - return group.toCheck == 0 ? n : "\(n) · \(group.toCheck) to check" - } -} - -/// Several things in one card, a divider between them. -private struct RowsCard: View { - let things: [Thing] - let atFurniture: Bool - let open: (Thing) -> Void - let guide: (Thing) -> Void - - var body: some View { - VStack(spacing: 0) { - ForEach(Array(things.enumerated()), id: \.element.id) { i, thing in - if i > 0 { Divider().overlay(Tokens.line.opacity(0.5)).padding(.leading, 58) } - Button { open(thing) } label: { FoundRow(thing: thing, atFurniture: atFurniture) } - .buttonStyle(.plain) - .accessibilityHint("Opens it to correct or confirm.") - .accessibilityAction(named: "Take me there") { guide(thing) } - .contextMenu { Button("Take me there", systemImage: "figure.walk") { guide(thing) } } - } - } - .padding(.horizontal, 14).padding(.vertical, 4) - .background(Tokens.card, in: .rect(cornerRadius: Tokens.cardRadius)) - .overlay(RoundedRectangle(cornerRadius: Tokens.cardRadius).stroke(Tokens.line.opacity(0.6), lineWidth: 0.5)) - } -} - /// Search within the list (in the page, so it shows with the brand header instead of a navigation bar). private struct SearchField: View { @Binding var text: String @@ -187,6 +108,7 @@ private struct SearchField: View { axis: .vertical) // grows with large text instead of clipping .foregroundStyle(Tokens.ink) .textInputAutocapitalization(.never) + .limited($text, to: TextInput.search) .submitLabel(.search) .accessibilityIdentifier("search") if !text.isEmpty { @@ -201,45 +123,3 @@ private struct SearchField: View { .overlay(Capsule().stroke(Tokens.line, lineWidth: 1)) } } - -/// One thing, compact: its icon, its name with a small Check mark when unsure, and where (just "on, waist height" -/// under a furniture heading that already names the place). -private struct FoundRow: View { - let thing: Thing - let atFurniture: Bool - - var body: some View { - HStack(spacing: 12) { - ThingIcon(thing: thing, side: 44) - VStack(alignment: .leading, spacing: 2) { - HStack(alignment: .firstTextBaseline, spacing: 6) { - Text(thing.title.capitalizedFirst).font(.headline).foregroundStyle(Tokens.ink) - .fixedSize(horizontal: false, vertical: true) - if thing.needsReview && !thing.confirmed { - Label("Check", systemImage: "questionmark.circle.fill").labelStyle(.titleAndIcon) - .font(.caption.weight(.bold)).foregroundStyle(Tokens.danger).fixedSize() - } else if thing.confirmed { - Image(systemName: "checkmark.seal.fill").font(.caption).foregroundStyle(Tokens.goText) - .accessibilityLabel("Checked") - } - } - Text(line).font(.subheadline).foregroundStyle(Tokens.muted) - .fixedSize(horizontal: false, vertical: true) - } - Spacer(minLength: 4) - Image(systemName: "chevron.right").font(.footnote.weight(.semibold)).foregroundStyle(Tokens.muted) - .accessibilityHidden(true) - } - .frame(minHeight: Tokens.minTarget) - .padding(.vertical, 6) - .contentShape(.rect) - .accessibilityElement(children: .ignore) - .accessibilityLabel("\(thing.title.capitalizedFirst). \(thing.place.capitalizedFirst). \(thing.certainty).") - } - - private var line: String { - guard atFurniture, thing.landmark != nil else { return thing.place.capitalizedFirst } - let rel = (thing.relation ?? "near").replacingOccurrences(of: "_", with: " ") - return "\(rel.capitalizedFirst) · \(thing.heightBand) height" - } -} diff --git a/apps/ios/Recall/Features/Home/HomeView.swift b/apps/ios/Recall/Features/Home/HomeView.swift index ae94dd3..961d891 100644 --- a/apps/ios/Recall/Features/Home/HomeView.swift +++ b/apps/ios/Recall/Features/Home/HomeView.swift @@ -13,9 +13,10 @@ struct HomeView: View { @State private var scanning: ScanRequest? var body: some View { + let rooms = ScanHistory.byRoom(scans.scans) // one card per room; its earlier scans fold inside it NavigationStack { ScrollView { - VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: Tokens.stack) { BrandHeader("Scans", subtitle: "Where are your things?") ScanHero { askingRoom = true } .padding(.top, 4) @@ -25,13 +26,13 @@ struct HomeView: View { } SectionTitle("Your scans", count: count) - .padding(.top, 12) + .padding(.top, Tokens.sectionGap) if scans.scans.isEmpty { EmptyState("viewfinder", "No scans yet. Scan a room to start.") } - ForEach(scans.scans) { scan in - ScanRow(scan: scan, review: { router.tab = .found }, - again: { scanning = ScanRequest(room: scan.room, again: scan) }) + ForEach(rooms) { room in + ScanRow(history: room, review: { router.tab = .found }, + again: { scanning = ScanRequest(room: room.latest.room, again: room.latest) }) } } .padding(.horizontal, Tokens.gutter) @@ -89,7 +90,7 @@ private struct ScanHero: View { .frame(maxWidth: .infinity, minHeight: Tokens.minTarget) .background(hc ? Tokens.accentFill : .white, in: Capsule()) } - .buttonStyle(.plain) + .buttonStyle(.plainTap) .glassRim(.capsule, .regular.interactive()) .accessibilityHint("Walk slowly around one room with the phone.") } @@ -140,11 +141,14 @@ struct ScanRequest: Identifiable { private struct ScanRow: View { @Environment(ScansModel.self) private var scans - let scan: Scan + let history: RoomScans let review: () -> Void let again: () -> Void @State private var shareURL: URL? @State private var confirmingDelete = false + @State private var showingEarlier = false + + private var scan: Scan { history.latest } var body: some View { VStack(alignment: .leading, spacing: 12) { @@ -163,6 +167,7 @@ private struct ScanRow: View { Text("\(scan.frames) frames · \(scan.directions) of 12 directions") .font(.footnote).foregroundStyle(Tokens.muted) if scan.state != .notUploaded { UploadStatus(state: scan.state) } + if scan.isProcessing && scan.workerStopped { WorkerStopped() } } .contentShape(.rect) .accessibilityElement(children: .combine) @@ -192,16 +197,126 @@ private struct ScanRow: View { ShareLink(item: url) { Image(systemName: "square.and.arrow.up").foregroundStyle(Tokens.accent) } .buttonStyle(.glass).buttonBorderShape(.circle).controlSize(.large) .accessibilityLabel("Share the scan") - .accessibilityHint("Send this scan with AirDrop or save it to Files.") + .accessibilityHint("Sends the scan's photos of your home, its depth and path, with AirDrop or to Files. Share only with people you trust.") } } + if !history.earlier.isEmpty { + EarlierScans(history: history, open: $showingEarlier) + } } .card() .task(id: scan.id) { shareURL = await scans.archive(scan) } // built once, off the main thread .accessibilityElement(children: .contain) .accessibilityAction(named: "Delete scan") { askToDelete() } - // an alert, not a confirmation dialog: iOS 26 shows that as a popover with no Cancel, hard to leave by VoiceOver - .alert(deleteTitle, isPresented: $confirmingDelete) { + .deleteScanAlert(scan, isPresented: $confirmingDelete) + } + + /// Asks first; while the Mac is still busy with the scan, says so instead. + private func askToDelete() { + if scan.deleteReach == .waitForMac { Task { await scans.delete(scan) } } else { confirmingDelete = true } + } +} + +/// A room's earlier scans, folded at the foot of its card: the memory now comes from the newest one, so these are +/// history, kept to share or delete. No Check or Scan again here; those belong to the current scan. +private struct EarlierScans: View { + let history: RoomScans + @Binding var open: Bool + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Divider().overlay(Tokens.line.opacity(0.5)) + Button { + withAnimation(reduceMotion ? nil : .snappy) { open.toggle() } + } label: { + HStack(spacing: 8) { + Label("Earlier scans", systemImage: "clock.arrow.circlepath") + .font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.ink) + Spacer(minLength: 8) + Text("\(history.earlier.count)").font(.subheadline).foregroundStyle(Tokens.muted) + Image(systemName: "chevron.down").font(.footnote.weight(.bold)).foregroundStyle(Tokens.muted) + .rotationEffect(.degrees(open ? 0 : -90)) + } + .frame(minHeight: 44) + .contentShape(.rect) + } + .buttonStyle(.plainTap) + .accessibilityElement(children: .ignore) + .accessibilityLabel("Earlier scans of \(history.latest.room), \(scansText)") + .accessibilityValue(open ? "Open" : "Folded") + .accessibilityHint(open ? "Hides them." : "Shows them.") + .accessibilityAddTraits(.isButton) + .accessibilityIdentifier("earlier-scans") + if open { + ForEach(history.earlier) { earlier in + EarlierScanRow(scan: earlier, after: history.successor(of: earlier)) + } + } + } + } + + private var scansText: String { history.earlier.count == 1 ? "1 scan" : "\(history.earlier.count) scans" } +} + +/// One earlier scan, compact: when, how good, and which scan took over from it. Long press to share or delete. +private struct EarlierScanRow: View { + @Environment(ScansModel.self) private var scans + let scan: Scan + let after: Scan? + @State private var shareURL: URL? + @State private var confirmingDelete = false + + var body: some View { + HStack(spacing: 12) { + VStack(alignment: .leading, spacing: 2) { + Text(scan.createdAt.formatted(date: .abbreviated, time: .shortened)) + .font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.ink) + Text("\(scan.frames) frames · \(took)").font(.footnote).foregroundStyle(Tokens.muted) + .fixedSize(horizontal: false, vertical: true) + } + Spacer(minLength: 8) + GradeBadge(grade: scan.grade) + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Tokens.canvas, in: .rect(cornerRadius: 12)) + .contentShape(.rect) + .contextMenu { + if let url = shareURL { ShareLink("Share scan", item: url) } + Button("Delete scan", systemImage: "trash", role: .destructive) { confirmingDelete = true } + } + .task(id: scan.id) { shareURL = await scans.archive(scan) } + .accessibilityElement(children: .ignore) + .accessibilityLabel("\(scan.room), \(scan.createdAt.formatted(date: .abbreviated, time: .shortened)), \(took)") + .accessibilityAction(named: "Delete scan") { confirmingDelete = true } + .accessibilityIdentifier("earlier-scan-\(scan.id)") + .deleteScanAlert(scan, isPresented: $confirmingDelete) + } + + // a recognised rescan built on this one; one that didn't line up made a fresh map of the room + private var took: String { + guard let after else { return "earlier scan" } + let day = after.createdAt.formatted(.dateTime.day().month()) + return after.alignedTo == nil ? "replaced by the scan on \(day)" : "updated by the scan on \(day)" + } +} + +private extension View { + /// Asks before deleting a scan, saying what it reaches. An alert, not a confirmation dialog: iOS 26 shows that as + /// a popover with no Cancel, hard to leave by VoiceOver. + func deleteScanAlert(_ scan: Scan, isPresented: Binding) -> some View { + modifier(DeleteScanAlert(scan: scan, isPresented: isPresented)) + } +} + +private struct DeleteScanAlert: ViewModifier { + @Environment(ScansModel.self) private var scans + let scan: Scan + @Binding var isPresented: Bool + + func body(content: Content) -> some View { + content.alert(title, isPresented: $isPresented) { Button(scan.deleteReach == .phoneAndMac ? "Delete from iPhone and Mac" : "Delete", role: .destructive) { Task { await scans.delete(scan) } } @@ -214,14 +329,9 @@ private struct ScanRow: View { } } - private var deleteTitle: String { + private var title: String { scan.deleteReach == .phoneAndMac ? "Delete this scan from your iPhone and your Mac?" : "Delete this scan?" } - - /// Asks first; while the Mac is still busy with the scan, says so instead. - private func askToDelete() { - if scan.deleteReach == .waitForMac { Task { await scans.delete(scan) } } else { confirmingDelete = true } - } } /// An SF Symbol for a room name, so the list can be scanned at a glance. @@ -291,7 +401,7 @@ private struct UploadStatus: View { } /// After a rescan: what moved, what's new, what's gone, each with its icon and the places in words; or that the room -/// wasn't recognised and its memory started afresh. "Hear it again" says the summary. +/// didn't line up with the last one, so it is a fresh map (nothing lost, nothing compared). "Hear it again" says it. private struct WhatChanged: View { @Environment(ScansModel.self) private var scans let scan: Scan @@ -299,17 +409,23 @@ private struct WhatChanged: View { var body: some View { VStack(alignment: .leading, spacing: 10) { HStack { - Label("What changed", systemImage: "arrow.left.arrow.right").font(.headline).foregroundStyle(Tokens.ink) + Label(scan.fresh ? ChangeSummary.freshTitle(room: scan.room) : "What changed", + systemImage: scan.fresh ? "map" : "arrow.left.arrow.right") + .font(.headline).foregroundStyle(Tokens.ink) Spacer(minLength: 8) - Button { scans.sayChanges(scan) } label: { - Label("Hear it again", systemImage: "speaker.wave.2.fill").labelStyle(.iconOnly) + let speaking = scans.speakingScan == scan.id + Button { scans.toggleChanges(scan) } label: { + Label(speaking ? "Stop" : "Hear it again", systemImage: speaking ? "speaker.slash.fill" : "speaker.wave.2.fill") + .labelStyle(.iconOnly) .font(.title3).foregroundStyle(Tokens.accent).frame(minWidth: 44, minHeight: 44) } - .accessibilityLabel("Hear what changed again") + .accessibilityLabel(speaking ? "Stop reading" : "Hear what changed again") } if scan.fresh { - Text("The room wasn't recognised, so Recall started its memory afresh.") + Text(ChangeSummary.freshBody) .font(.subheadline).foregroundStyle(Tokens.ink).fixedSize(horizontal: false, vertical: true) + Label(ChangeSummary.freshTip, systemImage: "lightbulb") + .font(.subheadline).foregroundStyle(Tokens.muted).fixedSize(horizontal: false, vertical: true) } else if scan.changes.isEmpty { Text("Nothing has moved since the last scan.").font(.subheadline).foregroundStyle(Tokens.ink) } else { @@ -365,3 +481,25 @@ private struct ChangeRow: View { } } } + + +/// A scan waiting with nobody to process it: the Mac's scan worker isn't running. Says what to run, and copies it. +private struct WorkerStopped: View { + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Label(Scan.workerStoppedMessage, systemImage: "exclamationmark.triangle.fill") + .font(.subheadline).foregroundStyle(Tokens.danger) + .fixedSize(horizontal: false, vertical: true) + Button { UIPasteboard.general.string = "make worker" } label: { + Label("Copy \"make worker\"", systemImage: "doc.on.doc").font(.subheadline.weight(.semibold)) + .foregroundStyle(Tokens.accent).frame(minHeight: 44) + } + .accessibilityHint("Copies the command to run in the recall folder on your Mac.") + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Tokens.canvas, in: .rect(cornerRadius: 12)) + .accessibilityElement(children: .contain) + .accessibilityIdentifier("worker-stopped") + } +} diff --git a/apps/ios/Recall/Features/Report/ReportView.swift b/apps/ios/Recall/Features/Report/ReportView.swift index 123149d..e371527 100644 --- a/apps/ios/Recall/Features/Report/ReportView.swift +++ b/apps/ios/Recall/Features/Report/ReportView.swift @@ -31,6 +31,34 @@ struct ReportView: View { StatTile(String(format: "%.1f m/s", metrics.meanSpeedMps), "average pace") StatTile("\(metrics.slowDownCues)", "slow-down cues") StatTile(String(format: "%.0f m²", metrics.areaWalkedM2), "area walked") + if let seen = metrics.zonesSeen, let total = metrics.zonesTotal, total > 0 { + StatTile("\(seen)/\(total)", "height zones seen") + } + if let walls = metrics.wallsCovered { StatTile("\(walls)/12", "wall sides seen") } + if let floor = metrics.floorSeenM2 { StatTile(String(format: "%.1f m²", floor), "floor seen") } + if let points = metrics.points { + StatTile(points >= 1000 ? "\(points / 1000) k" : "\(points)", "LiDAR points") + } + StatTile(String(format: "%.1f m/s", metrics.peakSpeedMps), "fastest pace") + StatTile("\(Int((CaptureMetrics.blurryShare(kept: metrics.keyframesKept, dropped: metrics.keyframesDroppedBlurry) * 100).rounded())) %", + "blurry photos") + if let loop = metrics.loopClosed { StatTile(loop ? "Yes" : "No", "back at the start") } + if let spread = metrics.compassSpreadDeg { StatTile(spread <= 25 ? "Good" : "Poor", "compass") } + StatTile(ScanTips.light(metrics.meanLightLux), "light") + } + if let steps = metrics.steps, !steps.isEmpty { StepTimeline(steps: steps) } + let tips = ScanTips.make(metrics) + if !tips.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Next time").font(.headline).foregroundStyle(Tokens.ink).accessibilityAddTraits(.isHeader) + ForEach(tips, id: \.self) { tip in + Label(tip, systemImage: "lightbulb").font(.subheadline).foregroundStyle(Tokens.ink) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .card() + .accessibilityIdentifier("report-tips") } BigButton("Upload to the Mac", systemImage: "arrow.up.circle.fill", hint: "Send the scan to your Mac so Recall can remember the room.") { @@ -48,3 +76,33 @@ struct ReportView: View { .recallScreen() } } + + +/// How each step went: done (a tick) or moved on when its time ran out, how long it took, and what it measured. +private struct StepTimeline: View { + let steps: [ScanGuide.StepRecord] + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + Text("How each step went").font(.headline).foregroundStyle(Tokens.ink).accessibilityAddTraits(.isHeader) + ForEach(Array(steps.enumerated()), id: \.offset) { _, r in + HStack(alignment: .top, spacing: 10) { + Image(systemName: r.completed ? "checkmark.circle.fill" : "arrow.right.circle") + .foregroundStyle(r.completed ? Tokens.goText : Tokens.danger) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: 2) { + Text(r.title).font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.ink) + Text("\(r.detail) · \(Int(r.seconds)) s\(r.completed ? "" : " · moved on")") + .font(.subheadline).foregroundStyle(Tokens.muted) + } + .fixedSize(horizontal: false, vertical: true) + } + .accessibilityElement(children: .ignore) + .accessibilityLabel("\(r.title): \(r.completed ? "done" : "moved on before it was done"). \(r.detail). \(Int(r.seconds)) seconds.") + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .card() + .accessibilityIdentifier("report-steps") + } +} diff --git a/apps/ios/Recall/Features/Robots/RobotAccessModel.swift b/apps/ios/Recall/Features/Robots/RobotAccessModel.swift index 0bff152..f1e1a43 100644 --- a/apps/ios/Recall/Features/Robots/RobotAccessModel.swift +++ b/apps/ios/Recall/Features/Robots/RobotAccessModel.swift @@ -1,16 +1,15 @@ import Foundation import Observation import RecallDomain -import RecallUpload -/// Robot Access: the owner's switch on the Mac, and the robot map package to share (a .zip for ROS 2 / Nav2). +/// Robot Access: the owner's switch on the Mac, the room's map preview and recent robot activity. The package +/// itself comes from the Mac (RobotPackageShare). @MainActor @Observable final class RobotAccessModel { private(set) var status: RobotAccessStatus? private(set) var package: RobotMapPackage? /// The newest audit entries about robots (map reads, questions, refusals, the switch). private(set) var activity: [AuditEntry] = [] - private(set) var zip: URL? private(set) var busy = false private(set) var error: String? @@ -27,7 +26,6 @@ final class RobotAccessModel { status = try await sharing.robotAccess() package = try? await sharing.robotMap(home: settings.connection.homeId) await refreshActivity() - zip = nil } } @@ -40,15 +38,6 @@ final class RobotAccessModel { } } - /// Builds the .zip for the share sheet (map.pgm, map.yaml, objects.json, room.json). - func prepareExport() async { - await run { - let p = try await sharing.robotMap(home: settings.connection.homeId) - package = p - zip = try FolderZip.make(name: p.name, files: p.files, in: FileManager.default.temporaryDirectory) - } - } - private func refreshActivity() async { activity = Array(((try? await sharing.activity(limit: 50)) ?? []).filter(\.isAboutRobots).prefix(10)) } diff --git a/apps/ios/Recall/Features/Robots/RobotAccessView.swift b/apps/ios/Recall/Features/Robots/RobotAccessView.swift index 4fbb952..64dda2f 100644 --- a/apps/ios/Recall/Features/Robots/RobotAccessView.swift +++ b/apps/ios/Recall/Features/Robots/RobotAccessView.swift @@ -71,58 +71,16 @@ private struct MapPreview: View { var body: some View { VStack(alignment: .leading, spacing: 10) { Text("\(package.room) map").font(.headline).foregroundStyle(Tokens.ink) - Canvas { ctx, size in - let m = package.map - let k = min(size.width / CGFloat(m.width), size.height / CGFloat(m.height)) - let ox = (size.width - CGFloat(m.width) * k) / 2, oy = (size.height - CGFloat(m.height) * k) / 2 - for r in 0.. SCNNode { + guard !points.isEmpty else { return SCNNode() } + let element = SCNGeometryElement(indices: Array(0..]) { + guard !points.isEmpty else { return } + trail.add(points) + let vertices = trail.points.map { SCNVector3($0.x, $0.y, $0.z) } + let source = SCNGeometrySource(vertices: vertices) + let element = SCNGeometryElement(indices: Array(0..= lastSpokenDirections + 3 { lastSpokenDirections = u.directionsCovered @@ -109,7 +139,8 @@ final class ScanModel { scans.add(scan) phase = .done(scan, m) let grade = m.grade == .good ? "good" : m.grade == .usable ? "usable" : "a redo" - speech.say("Scan saved. Quality: \(grade). \(ScanGrade.reason(for: m))", interrupt: true) + let tip = ScanTips.make(m).first.map { " Next time: \($0)" } ?? "" + speech.say("Scan saved. Quality: \(grade). \(ScanGrade.reason(for: m))\(tip)", interrupt: true) } catch { files.delete(scan: scanId) phase = .failed(error.localizedDescription) diff --git a/apps/ios/Recall/Features/Scan/ScanPlanView.swift b/apps/ios/Recall/Features/Scan/ScanPlanView.swift new file mode 100644 index 0000000..83831b0 --- /dev/null +++ b/apps/ios/Recall/Features/Scan/ScanPlanView.swift @@ -0,0 +1,81 @@ +import RecallDesign +import RecallDomain +import SwiftUI + +/// "How to scan well", before every scan: each step with what to do, why it matters, when it's done, and how high to +/// hold the phone. The same words are spoken during the scan; here they can be read (or heard) first. +struct ScanPlanView: View { + let room: String + let rescan: Bool + let readAloud: () -> Void + let start: () -> Void + let cancel: () -> Void + + private var steps: [ScanGuide.Step] { ScanGuide.Step.allCases.filter { $0 != .recognise || rescan } } + + var body: some View { + NavigationStack { + ScrollView { + VStack(alignment: .leading, spacing: Tokens.stack) { + Text("Scanning \(room) takes about 2 minutes. Recall says each step as you go; here is the whole plan.") + .font(.body).foregroundStyle(Tokens.ink).fixedSize(horizontal: false, vertical: true) + ForEach(Array(steps.enumerated()), id: \.element) { i, step in + PlanStep(number: i + 1, step: step) + } + VStack(alignment: .leading, spacing: 6) { + Text("Good to know").font(.headline).foregroundStyle(Tokens.ink).accessibilityAddTraits(.isHeader) + ForEach(Self.tips, id: \.self) { tip in + Label(tip, systemImage: "checkmark.circle").font(.subheadline).foregroundStyle(Tokens.muted) + .fixedSize(horizontal: false, vertical: true) + } + } + .card() + Button("Read the steps aloud", systemImage: "speaker.wave.2.fill", action: readAloud) + .buttonStyle(SolidPillStyle(fill: Tokens.card, text: Tokens.accent, edge: Tokens.line)) + .accessibilityIdentifier("plan-read") + Button("Start scanning", systemImage: "camera.viewfinder", action: start) + .buttonStyle(SolidPillStyle(fill: Tokens.go, text: Tokens.onGo)) + .accessibilityIdentifier("plan-start") + } + .padding(Tokens.gutter) + } + .recallScreen() + .navigationTitle("How to scan well") + .navigationBarTitleDisplayMode(.inline) + .toolbar { ToolbarItem(placement: .cancellationAction) { Button("Cancel", action: cancel) } } + } + } + + static let tips = [ + "Turn the lights on, and open doors you want in the scan.", + "Walk slowly, as if carrying a full cup.", + "Hold the phone with both hands, screen facing you.", + "Start at the room's door: the Watch starts its directions there.", + ] +} + +private struct PlanStep: View { + let number: Int + let step: ScanGuide.Step + + var body: some View { + HStack(alignment: .top, spacing: 12) { + Text(step == .recognise ? "↺" : "\(number)").font(.headline.monospacedDigit()).foregroundStyle(Tokens.onAccent) + .frame(width: 32, height: 32).background(Tokens.accentFill, in: .circle) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: 4) { + Text(step.title).font(.headline).foregroundStyle(Tokens.ink) + Text(step.instruction).font(.subheadline).foregroundStyle(Tokens.ink) + Text("Why: \(step.why)").font(.subheadline).foregroundStyle(Tokens.muted) + Text(step.doneWhen).font(.subheadline.weight(.semibold)).foregroundStyle(Tokens.goText) + Label("Phone: \(step.heightLabel)", systemImage: "iphone.gen3").font(.footnote).foregroundStyle(Tokens.muted) + } + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + .card() + .accessibilityElement(children: .ignore) + .accessibilityLabel("Step \(number): \(step.title). \(step.instruction) Why: \(step.why) \(step.doneWhen) Phone at \(step.heightLabel).") + .accessibilityIdentifier("plan-step-\(step.rawValue)") + } +} diff --git a/apps/ios/Recall/Features/Scan/ScanView.swift b/apps/ios/Recall/Features/Scan/ScanView.swift index 36989ad..9e8ab4b 100644 --- a/apps/ios/Recall/Features/Scan/ScanView.swift +++ b/apps/ios/Recall/Features/Scan/ScanView.swift @@ -2,6 +2,7 @@ import ARKit import RecallCapture import RecallDesign import RecallDomain +import SceneKit import SwiftUI /// The capture screen. Built for use without looking: everything important is spoken; the screen is for a helper. @@ -15,6 +16,17 @@ struct ScanView: View { } var body: some View { + Group { + if model.showingPlan { + ScanPlanView(room: model.room, rescan: model.isRescan, readAloud: model.readPlan, start: model.begin, + cancel: close) + } else { + phase + } + } + } + + @ViewBuilder private var phase: some View { Group { switch model.phase { case .unsupported: @@ -27,19 +39,35 @@ struct ScanView: View { Failed(message: message, close: close) } } - .onAppear { model.start() } } private var capture: some View { ZStack { - CameraView(session: model.capture.session) + CameraView(session: model.capture.session, scene: model.paint.scene) .ignoresSafeArea() .accessibilityHidden(true) + LiveLook(session: model.capture.session, look: model.look) VStack(spacing: 12) { StepCard(update: model.update, time: time) + HStack(spacing: 8) { + LookChips(look: $model.look) + Button { model.showGrid.toggle() } label: { + Label("Grid", systemImage: "square.grid.3x3").font(.footnote.weight(.semibold)) + .padding(.horizontal, 12).frame(minHeight: 44) + .foregroundStyle(model.showGrid ? Color.black : Color.white) + .background(model.showGrid ? Color(red: 0.49, green: 0.98, blue: 1.0) : Color.black.opacity(0.55), + in: Capsule()) + } + .buttonStyle(.plainTap) + .accessibilityLabel("Zone grid") + .accessibilityValue(model.showGrid ? "On" : "Off") + .accessibilityHint("Neon lines for each metre of floor and each height: knee, waist, chest, head. Zones the scan has seen glow.") + .accessibilityIdentifier("grid-toggle") + } if let cue = model.cue { CueBanner(cue) } Spacer() // the panels keep their own solid fill; only the buttons are glass (see hudRim) + Readouts(update: model.update, points: model.paint.trail.total) HStack(spacing: 16) { CoverageDial(covered: model.update.sectors, count: model.update.directionsCovered) MiniMap(surfaces: model.update.surfaces) @@ -75,42 +103,120 @@ struct ScanView: View { } } -/// The current voice step, the room map's state and the time, for a helper (all of it is also spoken). +/// The current step for a helper and for anyone who wants to read along: which of the 5 it is, what to do, a meter +/// for how far along it is, how high to hold the phone (measured), the distance to the wall on the walls step, and +/// why the step matters. All of it is also spoken. private struct StepCard: View { let update: CaptureUpdate let time: String + @State private var showingWhy = false private var number: Int { min(max(update.step.rawValue + 1, 0), 5) } + private let white = Color.white + private let amber = Color(red: 0.95, green: 0.7, blue: 0.24) + private let green = Color(red: 0.55, green: 0.85, blue: 0.45) var body: some View { - HStack(alignment: .center, spacing: 12) { - VStack(alignment: .leading, spacing: 4) { + VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .center, spacing: 8) { Text(update.step == .finish ? "Last step" : update.step == .recognise ? "Scanning again" : "Step \(number) of 5") - .font(.footnote.weight(.semibold)).foregroundStyle(.white) - Text(update.step.title).font(.title3.weight(.bold)).foregroundStyle(.white) - .fixedSize(horizontal: false, vertical: true) - } - Spacer(minLength: 8) - VStack(alignment: .trailing, spacing: 6) { + .font(.footnote.weight(.semibold)).foregroundStyle(white) + StepDots(current: update.step, done: Set(update.stepsDone.filter(\.completed).map(\.step))) + Spacer(minLength: 4) Label(update.mapStatus == .mapped ? "Map ready" : "Mapping", systemImage: update.mapStatus == .mapped ? "checkmark.circle.fill" : "circle.dashed") - .font(.footnote.weight(.bold)) - .foregroundStyle(update.mapStatus == .mapped ? Tokens.blueDeep : .white) - .padding(.horizontal, 10).padding(.vertical, 6) - .background(update.mapStatus == .mapped ? Tokens.amber : .white.opacity(0.18), in: .capsule) - Text(time).font(.headline).monospacedDigit().foregroundStyle(.white) + .font(.caption.weight(.bold)) + .foregroundStyle(update.mapStatus == .mapped ? Tokens.blueDeep : white) + .padding(.horizontal, 8).padding(.vertical, 4) + .background(update.mapStatus == .mapped ? Tokens.amber : white.opacity(0.18), in: .capsule) + Text(time).font(.subheadline.weight(.semibold)).monospacedDigit().foregroundStyle(white) + } + Text(update.step.title).font(.title3.weight(.bold)).foregroundStyle(white) + .fixedSize(horizontal: false, vertical: true) + Text(update.step.instruction).font(.subheadline).foregroundStyle(white.opacity(0.92)) + .fixedSize(horizontal: false, vertical: true) + if let p = update.stepProgress { + VStack(alignment: .leading, spacing: 4) { + ProgressView(value: p.fraction).tint(green) + Text(p.text).font(.footnote.weight(.semibold)).foregroundStyle(white) + } + } + HStack(spacing: 8) { + chip(heightText, colour: heightColour) + if update.step == .alongWalls, let d = update.centreDistance { chip(wallText(d), colour: wallColour(d)) } + Spacer(minLength: 0) + Button(showingWhy ? "Hide why" : "Why?") { withAnimation(.snappy) { showingWhy.toggle() } } + .font(.footnote.weight(.bold)).foregroundStyle(white) + .padding(.horizontal, 10).frame(minHeight: 44) + .accessibilityHint("Shows why this step matters.") + } + if showingWhy { + Text(update.step.why + " " + update.step.doneWhen).font(.footnote).foregroundStyle(white.opacity(0.9)) + .fixedSize(horizontal: false, vertical: true) } } - .padding(16) + .padding(14) .background(Tokens.blueDeep.opacity(0.92), in: .rect(cornerRadius: 22)) .hudRim(RoundedRectangle(cornerRadius: 22)) - .accessibilityElement(children: .ignore) - .accessibilityLabel(update.step == .recognise ? "Scanning again: \(update.step.title). \(time)." - : "Step \(number): \(update.step.title). \(update.mapStatus.spoken). \(time).") + .accessibilityElement(children: .combine) + .accessibilityLabel(spoken) + .accessibilityAction(named: "Why this step") { UIAccessibility.post(notification: .announcement, argument: update.step.why) } + .accessibilityIdentifier("step-card") + } + + private func chip(_ text: String, colour: Color) -> some View { + Text(text).font(.caption.weight(.bold)).foregroundStyle(.black) + .padding(.horizontal, 8).padding(.vertical, 4).background(colour, in: .capsule) + } + + private var heightText: String { + guard let h = update.phoneHeight else { return "Phone height: measuring…" } + let c = HeightCoach.check(h, for: update.step) + let mark = c == .good ? "✓" : c == .tooLow ? "raise" : "lower" + return String(format: "Phone %.2f m %@", h, mark) + } + + private var heightColour: Color { + guard let h = update.phoneHeight else { return white.opacity(0.7) } + return HeightCoach.check(h, for: update.step) == .good ? green : amber + } + + private func wallText(_ d: Double) -> String { + switch ScanGuide.wallDistance(d) { + case .good: String(format: "%.1f m from the wall ✓", d) + case .tooClose: String(format: "Too close: %.1f m", d) + case .tooFar: String(format: "Too far: %.1f m", d) + } + } + + private func wallColour(_ d: Double) -> Color { ScanGuide.wallDistance(d) == .good ? green : amber } + + private var spoken: String { + let head = update.step == .recognise ? "Scanning again: \(update.step.title)." : "Step \(number) of 5: \(update.step.title)." + let progress = update.stepProgress.map { " \($0.text)." } ?? "" + let height = update.phoneHeight.map { String(format: " Phone at %.1f metres, target %@.", $0, update.step.heightLabel) } ?? "" + return "\(head) \(update.step.instruction)\(progress)\(height) \(update.mapStatus.spoken). \(time)." + } +} + +/// The 5 steps as dots: done (filled with a tick), the current one (ringed), the rest (faint). +private struct StepDots: View { + let current: ScanGuide.Step + let done: Set + + var body: some View { + HStack(spacing: 4) { + ForEach(0..<5, id: \.self) { i in + Circle() + .fill(done.contains(i) ? Color.white : i == current.rawValue ? Color.white.opacity(0.5) : Color.white.opacity(0.2)) + .frame(width: 8, height: 8) + .overlay(Circle().stroke(Color.white, lineWidth: i == current.rawValue ? 1.5 : 0).padding(-2)) + } + } + .accessibilityHidden(true) } } -/// A top-down sketch of the scan: floor the LiDAR saw (white squares), where the person walked (amber line), and /// which sides have had a wall in view (ticks round the edge). private struct MiniMap: View { let surfaces: SurfaceCoverage @@ -155,12 +261,14 @@ private struct MiniMap: View { } } -/// ARKit's camera feed for a sighted helper. It shows our own session; nothing is added to the scene. +/// ARKit's camera feed for a sighted helper, with the captured LiDAR points painted into the scene. private struct CameraView: UIViewRepresentable { let session: ARSession + let scene: SCNScene func makeUIView(context: Context) -> ARSCNView { let v = ARSCNView(frame: .zero) v.session = session + v.scene = scene v.automaticallyUpdatesLighting = false return v } @@ -210,3 +318,46 @@ private struct Failed: View { .background(Tokens.canvas.ignoresSafeArea()) } } + + +/// The live numbers, scanner style: distance ahead, points captured, photos kept and blurry, walking speed (amber +/// when too fast for sharp photos). Read as one sentence by VoiceOver. +private struct Readouts: View { + let update: CaptureUpdate + let points: Int + + var body: some View { + let fast = ScanSpeed.isTooFast(update.speed) + HStack(spacing: 0) { + cell("AHEAD", update.centreDistance.map { String(format: "%.1f m", $0) } ?? "–") + cell("ZONES", "\(update.zones.zonesSeen)/\(update.zones.zonesTotal)") + cell("PHOTOS", "\(update.keyframes)") + cell("BLURRY", "\(update.droppedBlurry)") + cell("SPEED", ScanSpeed.text(update.speed), warn: fast) + } + .padding(.vertical, 8) + .background(Color(red: 0.05, green: 0.08, blue: 0.14).opacity(0.85), in: .rect(cornerRadius: 12)) + .overlay(RoundedRectangle(cornerRadius: 12).stroke(Color.white.opacity(0.25), lineWidth: 1)) + .accessibilityElement(children: .ignore) + .accessibilityLabel(spoken(fast)) + .accessibilityAddTraits(.updatesFrequently) + .accessibilityIdentifier("scan-readouts") + } + + private func cell(_ name: String, _ value: String, warn: Bool = false) -> some View { + VStack(spacing: 2) { + Text(value).font(.system(.subheadline, design: .monospaced).weight(.bold)) + .foregroundStyle(warn ? Color(red: 0.95, green: 0.7, blue: 0.24) : .white) + .minimumScaleFactor(0.7).lineLimit(1) + Text(name).font(.system(.caption2, design: .monospaced)).foregroundStyle(.white.opacity(0.7)) + } + .frame(maxWidth: .infinity) + } + + private func spoken(_ fast: Bool) -> String { + let ahead = update.centreDistance.map { String(format: "%.1f metres ahead", $0) } ?? "nothing in range ahead" + let weak = update.zones.weakestBand.map { ", least seen at \($0) height" } ?? "" + return "\(ahead), zones seen \(update.zones.zonesSeen) of \(update.zones.zonesTotal)\(weak), \(update.keyframes) photos kept, \(update.droppedBlurry) blurry" + + (fast ? ", walking too fast" : "") + } +} diff --git a/apps/ios/Recall/Features/Settings/SettingsView.swift b/apps/ios/Recall/Features/Settings/SettingsView.swift index c6bcf61..7115e23 100644 --- a/apps/ios/Recall/Features/Settings/SettingsView.swift +++ b/apps/ios/Recall/Features/Settings/SettingsView.swift @@ -1,16 +1,19 @@ import RecallDesign import RecallDomain import SwiftUI +import UniformTypeIdentifiers import UIKit /// Where the Mac is and the shared device key (pairing, not a login). The key is kept in the Keychain. /// A native grouped Form: the system draws its rows and bars, and every text scales with Dynamic Type. struct SettingsView: View { @Environment(SettingsModel.self) private var model + @Environment(TrustModel.self) private var trust @State private var testResult: String? @State private var testOK = false @State private var testing = false @State private var showKey = false + @State private var showAddress = false @State private var canPaste = UIPasteboard.general.hasStrings // checking doesn't read it (no paste prompt) @AppStorage(Theme.appearanceKey) private var appearance = Appearance.system @AppStorage(Theme.highContrastKey) private var highContrast = false @@ -41,7 +44,7 @@ struct SettingsView: View { AppearanceView() } label: { HStack { - Label("Appearance", systemImage: "circle.lefthalf.filled").foregroundStyle(Tokens.ink) + Label("Appearance and voice", systemImage: "circle.lefthalf.filled").foregroundStyle(Tokens.ink) Spacer() Text(highContrast ? "\(appearance.title), high contrast" : appearance.title) .foregroundStyle(Tokens.muted) @@ -50,11 +53,40 @@ struct SettingsView: View { .accessibilityIdentifier("appearance") } + TrustSection(trust: trust) + Section { - TextField("Mac address", text: $model.connection.address, - prompt: Text("https://your-mac.ts.net:8445").foregroundStyle(Tokens.muted), axis: .vertical) - .keyboardType(.URL).textInputAutocapitalization(.never).autocorrectionDisabled() - .accessibilityIdentifier("macAddress") + HStack(alignment: .center, spacing: 8) { + if !showAddress && !model.connection.address.isEmpty { + // hidden by default, like the key: enough to recognise it, not to read over a shoulder + Text(model.connection.maskedAddress) + .font(.body.monospaced()).foregroundStyle(Tokens.ink) + .fixedSize(horizontal: false, vertical: true) + .frame(maxWidth: .infinity, minHeight: 44, alignment: .leading) + .accessibilityLabel("Mac address, hidden") + .accessibilityIdentifier("macAddress") + } else { + TextField("Mac address", text: $model.connection.address, + prompt: Text("https://your-mac.ts.net:8445").foregroundStyle(Tokens.muted), axis: .vertical) + .keyboardType(.URL).textInputAutocapitalization(.never).autocorrectionDisabled() + .frame(maxWidth: .infinity, minHeight: 44, alignment: .leading) + .accessibilityIdentifier("macAddress") + .limited($model.connection.address, to: TextInput.address) + } + if !model.connection.address.isEmpty { + Spacer(minLength: 0) + Button { showAddress.toggle() } label: { + Image(systemName: showAddress ? "eye.slash" : "eye") + .font(.body.weight(.semibold)) + .foregroundStyle(Tokens.accent) + .frame(minWidth: 44, minHeight: 44) + .contentShape(Rectangle()) + } + .buttonStyle(.plainTap) + .accessibilityLabel(showAddress ? "Hide the Mac address" : "Show the Mac address") + .accessibilityIdentifier("macAddress.toggle") + } + } } header: { FormHeader("Mac address") } footer: { FormNote("The address the Mac's API prints when it starts, a Tailscale or a Cloudflare Tunnel URL.") } @@ -77,6 +109,7 @@ struct SettingsView: View { } } .textInputAutocapitalization(.never).autocorrectionDisabled() + .limited($model.connection.key, to: TextInput.deviceKey) .accessibilityIdentifier("deviceKey") HStack(spacing: 8) { Button(showKey ? "Hide" : "Show", systemImage: showKey ? "eye.slash" : "eye") { showKey.toggle() } @@ -88,13 +121,18 @@ struct SettingsView: View { Button("Paste", systemImage: "doc.on.clipboard") { guard let text = UIPasteboard.general.string else { return } model.connection.key = text.trimmingCharacters(in: .whitespacesAndNewlines) - model.say("Key pasted. \(model.connection.keySummary)") + model.say(TextInput.keyTooLong(text) + ? "That key is longer than \(TextInput.deviceKey) characters, so it was cut. Copy it again from the Mac." + : "Key pasted. \(model.connection.keySummary)") } .accessibilityLabel("Paste the key") } Button("Copy", systemImage: "doc.on.doc") { - UIPasteboard.general.string = model.connection.cleanKey - model.say("Key copied.") + // this phone only (never Handoff to other devices), and gone after two minutes + UIPasteboard.general.setItems([[UTType.utf8PlainText.identifier: model.connection.cleanKey]], + options: [.localOnly: true, + .expirationDate: Date().addingTimeInterval(120)]) + model.say("Key copied for two minutes.") } .accessibilityLabel("Copy the key") .disabled(model.connection.cleanKey.isEmpty) @@ -112,7 +150,13 @@ struct SettingsView: View { TextField("Home name", text: $model.connection.home, axis: .vertical) .textInputAutocapitalization(.never).autocorrectionDisabled() .accessibilityIdentifier("homeName") - } header: { FormHeader("Home name") } footer: { FormNote("One name per home; every scan joins it.") } + .limited($model.connection.home, to: TextInput.homeId) + } header: { FormHeader("Home name") } footer: { + // the Mac takes lowercase letters, digits and dashes: say what will be sent when it differs + let sent = model.connection.homeId + FormNote(sent == model.connection.home ? "One name per home; every scan joins it." + : "Sent to your Mac as \(sent). One name per home; every scan joins it.") + } Section { Button { Task { await test() } } label: { @@ -141,6 +185,8 @@ struct SettingsView: View { } .listRowBackground(Tokens.card) } + // the Form's own top margin would put the header lower than on the other tabs + .contentMargins(.top, 0, for: .scrollContent) .recallScreen() .toolbar(.hidden, for: .navigationBar) .scrollEdgeEffectStyle(.soft, for: .top) @@ -150,6 +196,7 @@ struct SettingsView: View { .onReceive(NotificationCenter.default.publisher(for: UIApplication.didBecomeActiveNotification)) { _ in canPaste = UIPasteboard.general.hasStrings // copied on the Mac, then back in the app } + .keyboardDismissable() // scroll or Done hides the keyboard; a tap outside does too (KeyboardDismissal) } } @@ -160,10 +207,12 @@ struct SettingsView: View { } } -/// Theme: automatic, light or dark, and high contrast. +/// Theme: automatic, light or dark, high contrast, and the tab tap sound. private struct AppearanceView: View { @AppStorage(Theme.appearanceKey) private var appearance = Appearance.system @AppStorage(Theme.highContrastKey) private var highContrast = false + @AppStorage(TapSound.enabledKey) private var tapSounds = true + @Environment(SettingsModel.self) private var model var body: some View { Form { @@ -182,16 +231,76 @@ private struct AppearanceView: View { .accessibilityAddTraits(appearance == a ? .isSelected : []) } } header: { FormHeader("Theme") } footer: { FormNote("Automatic follows the iPhone's light or dark setting.") } + Section { + ForEach(VoicePreference.allCases, id: \.self) { v in + Button { model.voice = v } label: { + HStack { + Text(v.title).foregroundStyle(Tokens.ink) + Spacer() + if model.voice == v { + Image(systemName: "checkmark").font(.body.weight(.semibold)).foregroundStyle(Tokens.accent) + } + } + } + .accessibilityAddTraits(model.voice == v ? .isSelected : []) + .accessibilityIdentifier("voice-\(v.rawValue)") + } + Button { model.previewVoice() } label: { + HStack { + Text("Speaking in \(model.voiceTitle)").foregroundStyle(Tokens.ink) + .fixedSize(horizontal: false, vertical: true) + Spacer() + Image(systemName: "speaker.wave.2").font(.body.weight(.semibold)).foregroundStyle(Tokens.accent) + } + } + .accessibilityLabel("Hear it. Speaking in \(model.voiceTitle)") + .accessibilityHint("Speaks a sample answer in this voice.") + .accessibilityIdentifier("voice.hear") + } header: { FormHeader("Voice") } footer: { + FormNote(model.hasNaturalVoice + ? "Recall uses the most natural voice on this iPhone. With VoiceOver on, it speaks in VoiceOver's own voice." + : "For a more natural voice, download a Premium or Enhanced voice in the iPhone's Settings, under Accessibility, in the spoken content voices. It's free. With VoiceOver on, Recall speaks in VoiceOver's own voice.") + } Section { Toggle("High contrast", isOn: $highContrast).tint(Tokens.go) } footer: { FormNote("Black, white and yellow, like the website's high contrast. It also turns on with iOS's Increase Contrast.") } + Section { + Toggle("Tap sounds", isOn: $tapSounds).tint(Tokens.go) + } footer: { + FormNote("A soft tap and a light buzz when you switch tabs. The ring/silent switch mutes the sound.") + } } .listRowBackground(Tokens.card) } .recallScreen() - .navigationTitle("Appearance") + .navigationTitle("Appearance and voice") .navigationBarTitleDisplayMode(.inline) } } + +/// How sure Recall must be before it tells you about a thing: the three levels, the chosen one explained under them. +private struct TrustSection: View { + let trust: TrustModel + + var body: some View { + Section { + if let level = trust.level { + Picker("How sure before Recall tells you", selection: Binding(get: { level }, set: { next in + Task { await trust.choose(next) } + })) { + ForEach(TrustLevel.allCases) { Text($0.title).tag($0) } + } + .pickerStyle(.inline).labelsHidden().tint(Tokens.accent) + .accessibilityIdentifier("trust-level") + } else if trust.error == nil { + ProgressView().frame(maxWidth: .infinity) + } + // no Mac yet: the address and key above already say what's missing, so this only explains itself + } header: { FormHeader("How sure before Recall tells you") } footer: { + FormNote(trust.level?.explanation ?? "Set on your Mac, so your voice answers, Find, maps and robots all follow it.") + } + .task { await trust.load() } + } +} diff --git a/apps/ios/Recall/Models/RoomMapsModel.swift b/apps/ios/Recall/Models/RoomMapsModel.swift new file mode 100644 index 0000000..9ce6e71 --- /dev/null +++ b/apps/ios/Recall/Models/RoomMapsModel.swift @@ -0,0 +1,41 @@ +import Foundation +import Observation +import RecallApplication +import RecallDomain + +/// Each room's map for Find, loaded when its card shows and kept until something changes it: a scan finishes +/// processing (a rescan redraws its room), or the user pulls to refresh. A room the Mac has no map for (no processed +/// scan yet, or the Mac unreachable) is just "not ready": Find never shows an error box for a map. +@MainActor @Observable +final class RoomMapsModel { + enum Entry: Equatable { case loading, ready(RobotMapPackage), notReady } + + private(set) var maps: [String: Entry] = [:] + /// Goes up each time the maps are forgotten, so cards already on screen ask again. + private(set) var generation = 0 + + private let source: any RoomMaps + private let settings: SettingsModel + + init(source: any RoomMaps, settings: SettingsModel) { + self.source = source; self.settings = settings + } + + func load(_ room: String) async { + guard maps[room] == nil || maps[room] == .notReady else { return } + maps[room] = .loading + do { + maps[room] = .ready(try await source.roomMap(home: settings.connection.homeId, room: room)) + } catch let e where UserMessage.isCancellation(e) { + maps[room] = nil // the card scrolled away mid-request: ask again when it's back + } catch { + maps[room] = .notReady + } + } + + /// Every map is asked for again: cards on screen at once, the others when they show. + func forget() { + maps = [:] + generation += 1 + } +} diff --git a/apps/ios/Recall/Models/ScansModel.swift b/apps/ios/Recall/Models/ScansModel.swift index 08d701f..65d5954 100644 --- a/apps/ios/Recall/Models/ScansModel.swift +++ b/apps/ios/Recall/Models/ScansModel.swift @@ -7,6 +7,10 @@ import RecallDomain @MainActor @Observable final class ScansModel { private(set) var scans: [Scan] + /// The scan whose "what changed" is being read out (its speaker shows Stop). + private(set) var speakingScan: String? + /// The next poll only updates the cards: the app just opened or came back, so what finished meanwhile isn't spoken. + private var quietPoll = true private let library: any ScanLibrary private let files: any EpisodeFiles @@ -33,11 +37,13 @@ final class ScansModel { func start() async { await uploads.observe { [weak self] id, state in Task { @MainActor in self?.apply(id, state) } } while !Task.isCancelled { - let outcomes = await track(scans, home: settings.connection.homeId) + let outcomes = await track(scans, home: settings.connection.homeId, announce: !quietPoll) + quietPoll = false for (id, outcome) in outcomes { if let i = scans.firstIndex(where: { $0.id == id }) { scans[i].changes = outcome.changes scans[i].fresh = outcome.fresh + scans[i].workerStopped = outcome.workerStopped } apply(id, outcome.state, announce: false) } @@ -108,6 +114,9 @@ final class ScansModel { scans.filter(\.isProcessed).sorted { $0.createdAt > $1.createdAt }.lazy.compactMap { self.files.roomMap(scan: $0.id) }.first } + /// One scan's saved room map (the AR labels are placed in it). + func roomMap(scan id: String) -> Data? { files.roomMap(scan: id) } + /// UI tests only (launch argument `-UITestSampleThings`): a fixed list of scans. Real users never see it. func useForUITests(_ sample: [Scan]) { scans = sample; macReachable = false } @@ -115,10 +124,22 @@ final class ScansModel { func roomMap(scan: Scan) -> Data? { files.roomMap(scan: scan.id) } /// Says a processed rescan's changes again (the "Hear it again" button). - func sayChanges(_ scan: Scan) { - speech.say(ChangeSummary.say(room: scan.room, changes: scan.changes), interrupt: true) + /// The speaker on a scan: reads what changed, or stops if it is already reading. + func toggleChanges(_ scan: Scan) { + if speakingScan != nil || speech.isSpeaking { + speech.stop() + speakingScan = nil + return + } + let line = scan.fresh ? ChangeSummary.sayFresh(room: scan.room) : ChangeSummary.say(room: scan.room, changes: scan.changes) + speakingScan = scan.id + speech.onFinished = { [weak self] in self?.speakingScan = nil } + speech.say(line, interrupt: true) } + /// The app came back to the foreground: the next poll only updates the cards. + func becameActive() { quietPoll = true } + private func apply(_ id: String, _ state: UploadState, announce: Bool = true) { guard let i = scans.firstIndex(where: { $0.id == id }) else { return } let changed = scans[i].apply(state) diff --git a/apps/ios/Recall/Models/SettingsModel.swift b/apps/ios/Recall/Models/SettingsModel.swift index e88fc6f..055aa84 100644 --- a/apps/ios/Recall/Models/SettingsModel.swift +++ b/apps/ios/Recall/Models/SettingsModel.swift @@ -2,6 +2,7 @@ import Foundation import Observation import RecallApplication import RecallDomain +import RecallVoice /// The connection settings being edited, kept saved and handed to the live server on every change. @MainActor @Observable @@ -27,17 +28,34 @@ final class SettingsModel { var issue: SetupIssue? { connection.issue } - /// recall://pair?… from the Mac's QR code. + /// A pairing link waiting for the person to confirm it (shown as an alert). + var pendingPairing: PairingLink? + + /// recall://pair?… from the Mac's QR code. Nothing changes until the person confirms: any app or page can open + /// such a link, and a silent one would hand the device key to someone else's server. func pair(with link: URL) { do { - guard let paired = try pair(link) else { return } - connection = paired - speech.say(paired.issue?.message ?? "Paired with the Mac. Open Settings and test the connection.", interrupt: false) + guard let pairing = try pair.read(link) else { return } + pendingPairing = pairing + speech.say(pairing.summary, interrupt: false) } catch { speech.say(UserMessage.explain(error), interrupt: false) } } + func confirmPairing() { + guard let pairing = pendingPairing else { return } + pendingPairing = nil + connection = pair.apply(pairing) + speech.say(connection.issue?.message ?? "Paired with the Mac. Open Settings and test the connection.", interrupt: false) + } + + func cancelPairing() { + guard pendingPairing != nil else { return } + pendingPairing = nil + speech.say("Not paired. Your Mac's address and key are unchanged.", interrupt: false) + } + /// The result stays on screen (to read or copy); only success is spoken. func testConnection() async -> (message: String, ok: Bool) { if let issue { return (issue.message, false) } @@ -51,4 +69,20 @@ final class SettingsModel { } func say(_ text: String) { speech.say(text, interrupt: false) } + + /// Settings → Voice: automatic, female or male, kept on this iPhone; the Announcer reads it for every line. + var voice: VoicePreference = VoicePreference(rawValue: UserDefaults.standard.string(forKey: Announcer.voiceKey) ?? "") + ?? .automatic { + didSet { UserDefaults.standard.set(voice.rawValue, forKey: Announcer.voiceKey) } + } + + /// The voice that will speak, by name and quality ("Ava (Premium)"). + var voiceTitle: String { Announcer.chosenVoice(voice)?.title ?? "The iPhone's default voice" } + + /// Whether a downloaded voice (enhanced or premium) is speaking; if not, Settings says how to get one. + var hasNaturalVoice: Bool { (Announcer.chosenVoice(voice)?.quality ?? 0) > 0 } + + func previewVoice() { + speech.say("Your keys are on the desk, at waist height. Seen 12 minutes ago.", interrupt: true) + } } diff --git a/apps/ios/Recall/Models/TrustModel.swift b/apps/ios/Recall/Models/TrustModel.swift new file mode 100644 index 0000000..e280c66 --- /dev/null +++ b/apps/ios/Recall/Models/TrustModel.swift @@ -0,0 +1,36 @@ +import Foundation +import Observation +import RecallDomain + +/// How sure Recall must be before it tells you about a thing (Settings). The level lives on the Mac; this shows it +/// and changes it. Until it's loaded, or with no Mac, nothing is shown as chosen. +@MainActor @Observable +final class TrustModel { + private(set) var level: TrustLevel? + private(set) var error: String? + + private let source: any TrustSettings + + init(source: any TrustSettings) { self.source = source } + + func load() async { + do { + level = try await source.trustLevel() + error = nil + } catch { + self.error = UserMessage.explain(error) + } + } + + func choose(_ next: TrustLevel) async { + let was = level + level = next // at once, so the choice doesn't jump back while the Mac answers + do { + level = try await source.setTrustLevel(next) + error = nil + } catch { + level = was + self.error = UserMessage.explain(error) + } + } +} diff --git a/apps/ios/Recall/Resources/Info.plist b/apps/ios/Recall/Resources/Info.plist index 29eed78..c2c7313 100644 --- a/apps/ios/Recall/Resources/Info.plist +++ b/apps/ios/Recall/Resources/Info.plist @@ -31,23 +31,19 @@ CFBundleVersion 1 - LSSupportsOpeningDocumentsInPlace - NSAppTransportSecurity NSAllowsLocalNetworking NSCameraUsageDescription - Recall uses the camera and LiDAR to remember where things are in your home. Nothing is shared except with your own Mac. + Recall uses the camera and LiDAR to remember where things are in your home. Scans go to your own Mac, which sends their photos to Meta's SAM 3.1 to find things. NSLocalNetworkUsageDescription Recall sends your scans to your own Mac on this network. NSMicrophoneUsageDescription - Recall listens after you tap the microphone, so you can ask where something is. The recording goes only to your own Mac. + Recall listens after you tap the microphone, so you can ask where something is. The recording goes to your own Mac, which sends it to Meta's Muse Voice Transcribe and keeps only the words. UIApplicationSupportsIndirectInputEvents - UIFileSharingEnabled - UILaunchScreen UIRequiredDeviceCapabilities diff --git a/apps/ios/Recall/Resources/tap.wav b/apps/ios/Recall/Resources/tap.wav new file mode 100644 index 0000000..19b9621 Binary files /dev/null and b/apps/ios/Recall/Resources/tap.wav differ diff --git a/apps/ios/Recall/Shared/Keyboard.swift b/apps/ios/Recall/Shared/Keyboard.swift new file mode 100644 index 0000000..b82d120 --- /dev/null +++ b/apps/ios/Recall/Shared/Keyboard.swift @@ -0,0 +1,61 @@ +import UIKit + +/// Tapping anywhere outside a text field hides the keyboard, on every screen. While the keyboard is up (and only +/// then), one tap recognizer sits on the key window: it never cancels touches (buttons, rows and chips still get their +/// tap) and ignores taps inside a text field or text view (so tapping into a field still edits it). With no keyboard +/// on screen nothing is attached, so the rest of the app, and its accessibility, are untouched. +@MainActor +final class KeyboardDismissal: NSObject, UIGestureRecognizerDelegate { + static let shared = KeyboardDismissal() + private var tap: UITapGestureRecognizer? + private var observing = false + + /// Starts listening for the keyboard (once). + func install() { + guard !observing else { return } + observing = true + let center = NotificationCenter.default + center.addObserver(self, selector: #selector(keyboardShown), name: UIResponder.keyboardDidShowNotification, object: nil) + center.addObserver(self, selector: #selector(keyboardHidden), name: UIResponder.keyboardWillHideNotification, object: nil) + } + + @objc private func keyboardShown() { + guard tap == nil, let window = keyWindow() else { return } + let recognizer = UITapGestureRecognizer(target: self, action: #selector(tapped(_:))) + recognizer.cancelsTouchesInView = false + recognizer.delegate = self + window.addGestureRecognizer(recognizer) + tap = recognizer + } + + @objc private func keyboardHidden() { + if let tap { tap.view?.removeGestureRecognizer(tap) } + tap = nil + } + + @objc private func tapped(_ tap: UITapGestureRecognizer) { + tap.view?.endEditing(true) + } + + private func keyWindow() -> UIWindow? { + UIApplication.shared.connectedScenes + .compactMap { $0 as? UIWindowScene } + .flatMap(\.windows) + .first(where: \.isKeyWindow) + } + + nonisolated func gestureRecognizer(_ g: UIGestureRecognizer, shouldReceive touch: UITouch) -> Bool { + MainActor.assumeIsolated { + var view = touch.view + while let v = view { // a tap in a field (or its clear button) keeps the keyboard + if v is UITextField || v is UITextView { return false } + view = v.superview + } + return true + } + } + + nonisolated func gestureRecognizer(_ g: UIGestureRecognizer, shouldRecognizeSimultaneouslyWith other: UIGestureRecognizer) -> Bool { + true + } +} diff --git a/apps/ios/Recall/Shared/LiveLook.swift b/apps/ios/Recall/Shared/LiveLook.swift new file mode 100644 index 0000000..7935668 --- /dev/null +++ b/apps/ios/Recall/Shared/LiveLook.swift @@ -0,0 +1,65 @@ +import ARKit +import RecallCapture +import RecallDesign +import SwiftUI + +/// The chosen camera look over an ARKit camera view: nothing, the LiDAR's depth as heat, or edges like a scanner. +/// Redrawn ten times a second from the session's current frame; off with Reduce Motion for the edges (they flicker). +struct LiveLook: View { + let session: ARSession + let look: CameraLook + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @State private var image: CGImage? + private static let filter = ScannerFilter() + + var body: some View { + TimelineView(.periodic(from: .now, by: 0.1)) { context in + Group { + if let image, look != .camera { + Image(decorative: image, scale: 1) + .resizable() + .aspectRatio(contentMode: .fill) + .opacity(look == .lidar ? 0.55 : 0.88) + .blendMode(look == .edges ? .screen : .normal) + } + } + .onChange(of: context.date) { redraw() } + } + .allowsHitTesting(false) + .ignoresSafeArea() + .accessibilityHidden(true) + } + + private func redraw() { + guard look != .camera, let frame = session.currentFrame else { image = nil; return } + switch look { + case .camera: image = nil + case .lidar: image = frame.sceneDepth.flatMap { Self.filter.heat($0.depthMap) } + case .edges: image = reduceMotion ? nil : Self.filter.edges(frame.capturedImage) + } + } +} + +/// Camera · LiDAR · Edges. +struct LookChips: View { + @Binding var look: CameraLook + var choices: [CameraLook] = CameraLook.allCases + + var body: some View { + HStack(spacing: 6) { + ForEach(choices, id: \.self) { l in + Button { look = l } label: { + Text(l.title).font(.footnote.weight(.semibold)) + .padding(.horizontal, 12).frame(minHeight: 44) + .foregroundStyle(look == l ? Color.black : Color.white) + .background(look == l ? Color.white : Color.black.opacity(0.55), in: Capsule()) + } + .buttonStyle(.plainTap) + .accessibilityLabel("\(l.title) view") + .accessibilityHint(l.hint) + .accessibilityAddTraits(look == l ? .isSelected : []) + .accessibilityIdentifier("look-\(l.rawValue)") + } + } + } +} diff --git a/apps/ios/Recall/Shared/RobotPackageShare.swift b/apps/ios/Recall/Shared/RobotPackageShare.swift new file mode 100644 index 0000000..6a752a7 --- /dev/null +++ b/apps/ios/Recall/Shared/RobotPackageShare.swift @@ -0,0 +1,44 @@ +import RecallDesign +import RecallDomain +import SwiftUI +import UniformTypeIdentifiers + +/// "Share for robots": one tap opens the share sheet (AirDrop, Save to Files, Mail…); the Mac makes the room's +/// package only once a place is picked, so nothing is fetched just by showing the button. +struct RobotPackageShare: View { + let room: String? + var title = "Share for robots" + @Environment(\.container) private var container + @Environment(SettingsModel.self) private var settings + + var body: some View { + if let container { + let source = container.robotPackages(), home = settings.connection.homeId, room = room + ShareLink(item: RobotPackageFile(room: room, fetch: { try await source.robotPackage(home: home, room: room) }), + preview: SharePreview("\(room ?? "Room") for robots", image: Image(systemName: "shippingbox"))) { + Label(title, systemImage: "square.and.arrow.up") + } + .buttonStyle(SolidPillStyle(fill: Tokens.card, text: Tokens.accent, edge: Tokens.line, compact: true)) + .accessibilityHint("Sends this room's map, things, outline and 3D points to a robot, with AirDrop, Files or Mail.") + } + } +} + +/// The room's .zip, written to a temporary file when the share sheet asks for it. +struct RobotPackageFile: Transferable { + let room: String? + let fetch: @Sendable () async throws -> Data + + var fileName: String { + let slug = (room ?? "room").lowercased().map { $0.isLetter || $0.isNumber ? $0 : "-" } + return "recall-\(String(slug)).zip" + } + + static var transferRepresentation: some TransferRepresentation { + FileRepresentation(exportedContentType: .zip) { file in + let url = FileManager.default.temporaryDirectory.appending(path: file.fileName) + try await file.fetch().write(to: url, options: .atomic) + return SentTransferredFile(url) + } + } +} diff --git a/apps/ios/Recall/Shared/RoomMapCanvas.swift b/apps/ios/Recall/Shared/RoomMapCanvas.swift new file mode 100644 index 0000000..e0afb8d --- /dev/null +++ b/apps/ios/Recall/Shared/RoomMapCanvas.swift @@ -0,0 +1,70 @@ +import RecallDesign +import RecallDomain +import SwiftUI + +/// A room's map as robots see it: free floor white, furniture and walls dark, unseen parts faint, and the things +/// Recall remembers as blue dots. Used by Find (one card per room) and Robot Access, so both always look the same. +/// Hidden from VoiceOver: the card around it says what it shows. +struct RoomMapCanvas: View { + let package: RobotMapPackage + + var body: some View { + Canvas { ctx, size in + let m = package.map + let k = min(size.width / CGFloat(m.width), size.height / CGFloat(m.height)) + let ox = (size.width - CGFloat(m.width) * k) / 2, oy = (size.height - CGFloat(m.height) * k) / 2 + for r in 0..