From 6d15c36c1369d557558d6654134fc62fa490be8f Mon Sep 17 00:00:00 2001 From: jacob-web10-nyc Date: Wed, 26 Aug 2026 22:35:18 -0400 Subject: [PATCH 1/4] =?UTF-8?q?fix(api):=20/pwa=5Flisting=20500'd=20on=20n?= =?UTF-8?q?on-JSON=20200=20manifest=20bodies=20=E2=80=94=20now=20NO=5FPWA?= =?UTF-8?q?=20(3.16.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A registered file URL (.../index.html) makes the manifest lookup hit /manifest.json, which the SPA fallback answers with HTML + 200. json.loads raised JSONDecodeError (a ValueError, not a RequestException) which the except clause never caught. Now: NO_PWA + [pwa_listing] log, and the Accept header actually goes in headers= (was a query param). --- api/app/endpoints/system.py | 13 ++++++++++++- api/tests/test_v3_endpoints.py | 16 ++++++++++++++++ knowledge/changelogs/CHANGELOG.md | 3 +++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/api/app/endpoints/system.py b/api/app/endpoints/system.py index e77d7bae..35885691 100644 --- a/api/app/endpoints/system.py +++ b/api/app/endpoints/system.py @@ -1,4 +1,5 @@ import json +import logging import requests from fastapi import APIRouter, HTTPException @@ -16,6 +17,7 @@ from app.v3.services import clickhouse as ch router = APIRouter() +log = logging.getLogger(__name__) @router.post("/") @@ -155,7 +157,9 @@ def pwa_listing(url: str): # render. Over the cap → treat as no manifest. _MANIFEST_MAX_BYTES = 256 * 1024 try: - with requests.get(manifest_url, {"Accept": "application/json"}, timeout=1, stream=True) as resp: + with requests.get( + manifest_url, headers={"Accept": "application/json"}, timeout=1, stream=True + ) as resp: resp.raise_for_status() chunks = [] total = 0 @@ -166,6 +170,13 @@ def pwa_listing(url: str): chunks.append(chunk) return json.loads(b"".join(chunks)) except requests.exceptions.RequestException: + log.info("[pwa_listing] fetch failed for %s — NO_PWA", manifest_url) + raise exceptions.NO_PWA + except (json.JSONDecodeError, UnicodeDecodeError): + # A 200 with a non-JSON body (an SPA fallback returning HTML, a text + # file, ...) means there is no manifest at that path — the store + # falls back to the registered name. Not a server error. + log.info("[pwa_listing] non-JSON manifest body at %s — NO_PWA", manifest_url) raise exceptions.NO_PWA diff --git a/api/tests/test_v3_endpoints.py b/api/tests/test_v3_endpoints.py index 1ed8dc85..40adf600 100644 --- a/api/tests/test_v3_endpoints.py +++ b/api/tests/test_v3_endpoints.py @@ -1064,6 +1064,22 @@ def test_no_manifest_returns_no_pwa(self, client): resp = client.get("/pwa_listing", params={"url": "https://host/docs/notes/"}) assert resp.status_code == 401 # NO_PWA — the store falls back to the registered name + def test_non_json_200_body_returns_no_pwa(self, client): + """A 200 with a non-JSON body (SPA fallback HTML) is NO_PWA, not a 500. + + Regression: a registered file URL (.../index.html) makes the manifest + lookup hit /manifest.json, which the static server's SPA + fallback answers with HTML + 200 — json.loads used to 500 on it. + """ + fake = MagicMock() + fake.__enter__ = MagicMock(return_value=fake) + fake.__exit__ = MagicMock(return_value=False) + fake.raise_for_status = MagicMock() + fake.iter_content = MagicMock(return_value=iter([b""])) + with patch("app.endpoints.system.requests.get", return_value=fake): + resp = client.get("/pwa_listing", params={"url": "https://host/docs/media/index.html"}) + assert resp.status_code == 401 # NO_PWA — never a 500 + class TestAppsRegister: def test_register_normalizes_url(self, client, token): diff --git a/knowledge/changelogs/CHANGELOG.md b/knowledge/changelogs/CHANGELOG.md index 29a703ab..9bc8ffba 100644 --- a/knowledge/changelogs/CHANGELOG.md +++ b/knowledge/changelogs/CHANGELOG.md @@ -1,3 +1,6 @@ +3.16.2 || 26.08.2026 +fix(api): `/pwa_listing` no longer 500s when the manifest URL answers 200 with a non-JSON body — it returns NO_PWA (401) and the store falls back to the registered name, per D47. Operator: the dev store 500'd with `JSONDecodeError: Expecting value: line 1 column 1 (char 0)` on `GET /pwa_listing?url=https://dev.web10.app/docs/media/index.html`. Root cause: the registered app URL is a *file* (`.../index.html`), so the manifest lookup hits `.../index.html/manifest.json` — the static server's SPA fallback answers that with HTML + 200, and `json.loads()` raised `JSONDecodeError` (a `ValueError`, not a `RequestException`), which the endpoint's `except requests.exceptions.RequestException` never caught → 500 with the raw error class in the body. The designed behavior (D47, `app-store/overview.md`): no manifest → NO_PWA → the store falls back to the registered name, then the host. Fix: catch `json.JSONDecodeError` + `UnicodeDecodeError` → NO_PWA, with a `[pwa_listing]` log line at each NO_PWA decision point (fetch failure, non-JSON body). Also fixed a latent bug in the same call: `requests.get(url, {"Accept": ...})` passed the header dict as the **params** positional (2nd arg) — the Accept header was going out as a query string (`?Accept=application%2Fjson`); now `headers=`. Regression test: a 200 with an HTML body → 401, never a 500. Note the deeper data issue left alone (a D47 identity call, not a bug): the media demo is registered under its `index.html` file URL on dev, and a file URL can never have a manifest at `{url}/manifest.json` — the store now degrades gracefully instead of 500ing. 715 API tests green. + 3.16.1 || 26.08.2026 docs(kb) + docs(strategy): D50 + D51 — ads are a v3 default service, creator-owned (the rung-0 card is "Partner Links"); ad dissemination is a per-creator setting curated by a shared SDK helper. Operator: direct deals and affiliate links are the same kind of thing — collapse them; "there needs to be a standards ad object that holds ad data that is in the web10 docs, then the social app or any app can pick up the ads per user to display them! clickhouse is great for this"; "ads would be a default service, web10 users manage that they put their ads into! scoped to them!" The decision (D50, `knowledge/strategy/decisions.md`): (1) the rung-0 card "Amazon Associates" + "Direct Deals" collapses into one card — **Partner Links** — because they are the same primitive (a link that pays the creator; `offer.kind` = `affiliate` | `direct` | `own_store`); (2) an ad is a document in the `ads` default service (`collection_name = 'ads'`, `author_key` = the creator) — content (a video, a photo, a post) that carries a monetizable link (the offer), owned by the creator, scoped to their followers group, delivered by architecture (100% of followers) the same way a post is; (3) any app with `ads: [readAll]` picks up ads per viewer with the same multi-group read the feed uses — `w.read('ads', { groups: [...] })`, no new endpoint, rides the existing CRUD + read + media-presign machinery; (4) the Partner Links UI (the Studio card) is the ingest. **D51** — ad dissemination is a per-creator choice (not a platform decision): each creator sets how their own ads rotate to their own audience (`dissemination` = `round_robin` | `greedy` | `pinned` | `frequency_capped` + params, a field on the `settings` doc, picked in the Partner Links card); the feed + ads join is one ClickHouse query (`collection_name IN ('posts','ads')` over the viewer's groups — same table, same delivery), and a post can also `ref` an ad directly; the curation is a shared deterministic SDK helper (`curateAds(creatorAds, creatorSetting)`) rather than SQL, so every app curates a creator's ads identically without stateful ClickHouse logic. New KB doc `knowledge/knowledge-base/web10-v3/social/ads.md` (the standard ad object — creative + offer + stats on the leaf-typing convention, the data-model mapping, the per-user query, the Partner Links ingest, and the two-layer note). The two-layer note keeps this straight: the creator-owned `ads` service is v3 (ships now, zero new tables, thesis-aligned — "the only sponsors you'll ever see are ones the creator chose"); the v4 ad tables (`ad_campaigns`, `ad_targeting`, `ad_partners` with dsp/ssp/exchange, bidding) are the separate exchange layer for when brands buy inventory (M3) — do not build them to serve creator ads. `web10-v3/README.md` updated (ads.md in the structure + the v4 monetization line split into "Ad network" (v4) vs. the creator-owned `ads` service (v3)). New `ads` lane in `parallel-execution.md` (KB bite ticked; follow-up bites: the `curateAds` SDK helper, the Partner Links card collapse + dissemination picker, the `ads` service conformance, the e2e). Docs only — no code. 3.16.0 || 26.08.2026 From be7c244dda23fbe0cb3759a9b9fae0c2b94f296a Mon Sep 17 00:00:00 2001 From: jacob-web10-nyc Date: Wed, 26 Aug 2026 23:42:17 -0400 Subject: [PATCH 2/4] chore(api): ruff format --- api/app/endpoints/system.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/app/endpoints/system.py b/api/app/endpoints/system.py index 35885691..e8649476 100644 --- a/api/app/endpoints/system.py +++ b/api/app/endpoints/system.py @@ -157,9 +157,7 @@ def pwa_listing(url: str): # render. Over the cap → treat as no manifest. _MANIFEST_MAX_BYTES = 256 * 1024 try: - with requests.get( - manifest_url, headers={"Accept": "application/json"}, timeout=1, stream=True - ) as resp: + with requests.get(manifest_url, headers={"Accept": "application/json"}, timeout=1, stream=True) as resp: resp.raise_for_status() chunks = [] total = 0 From c4a6b21d89938a7b37590771da0c66e56c3d3cf6 Mon Sep 17 00:00:00 2001 From: jacob-web10-nyc Date: Thu, 27 Aug 2026 20:07:19 -0400 Subject: [PATCH 3/4] =?UTF-8?q?chore:=20renumber=20pwa-listing=20entry=203?= =?UTF-8?q?.16.2=20=E2=86=92=203.21.2=20after=20dev=20merge=20(version=20c?= =?UTF-8?q?ollision=20with=20the=20discover-group=203.16.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- knowledge/changelogs/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/knowledge/changelogs/CHANGELOG.md b/knowledge/changelogs/CHANGELOG.md index e99c8b80..9a60f7b9 100644 --- a/knowledge/changelogs/CHANGELOG.md +++ b/knowledge/changelogs/CHANGELOG.md @@ -1,7 +1,9 @@ -3.16.2 || 26.08.2026 +3.21.2 || 27.08.2026 fix(api): `/pwa_listing` no longer 500s when the manifest URL answers 200 with a non-JSON body — it returns NO_PWA (401) and the store falls back to the registered name, per D47. Operator: the dev store 500'd with `JSONDecodeError: Expecting value: line 1 column 1 (char 0)` on `GET /pwa_listing?url=https://dev.web10.app/docs/media/index.html`. Root cause: the registered app URL is a *file* (`.../index.html`), so the manifest lookup hits `.../index.html/manifest.json` — the static server's SPA fallback answers that with HTML + 200, and `json.loads()` raised `JSONDecodeError` (a `ValueError`, not a `RequestException`), which the endpoint's `except requests.exceptions.RequestException` never caught → 500 with the raw error class in the body. The designed behavior (D47, `app-store/overview.md`): no manifest → NO_PWA → the store falls back to the registered name, then the host. Fix: catch `json.JSONDecodeError` + `UnicodeDecodeError` → NO_PWA, with a `[pwa_listing]` log line at each NO_PWA decision point (fetch failure, non-JSON body). Also fixed a latent bug in the same call: `requests.get(url, {"Accept": ...})` passed the header dict as the **params** positional (2nd arg) — the Accept header was going out as a query string (`?Accept=application%2Fjson`); now `headers=`. Regression test: a 200 with an HTML body → 401, never a 500. Note the deeper data issue left alone (a D47 identity call, not a bug): the media demo is registered under its `index.html` file URL on dev, and a file URL can never have a manifest at `{url}/manifest.json` — the store now degrades gracefully instead of 500ing. 715 API tests green. + 3.21.1 || 27.08.2026 feature(api) + test(api): the power-mean ranking moves from Python into ClickHouse — the v1 scale-up. Operator, on the branch: "okay, continue! but explain what this even is" → "B! this is clickhouse". **What v0 did (3.18.2):** a sorted `/v3/read` fetched the FULL group membership into Python, ran two ref-count queries, scored every post in-process, sorted, then paged — an O(N) fetch + sort on every read. **What v1 does:** the ranking runs in SQL. `read_documents_in_groups` now dispatches the sort path to a new `_group_docs_ranked_query`: the board base (the same documents ⋈ doc_groups ⋈ group_members + block/share/hidden anti-joins, extracted into a shared `_board_base_sql` fragment the no-sort path also uses) is LEFT JOINed to EXACT engagement counts — one grouped scan of the `reactions` and `comments` collections, no per-row subquery, no maintained counter table — the power-mean score is computed in SQL (`_power_mean_score_sql`, mirroring `_power_mean_score` / `powerMean.ts` exactly: same saturating normalizers, epsilon floor, p=0 geometric mean, recency-only reverse-chron shortcut), and `ORDER BY` + `LIMIT`/`OFFSET` happen in ClickHouse. No full membership fetch into Python, no in-process sort. **The design call (B over A):** `discover.md` / `feed-lens-integration.md` floated a maintained `post_engagement` counter table (option A), but the operator picked the SQL aggregation join (option B) — "this is clickhouse." B is exact (no staleness), race-free (a read-modify-write counter is not atomic in ClickHouse → lost updates), needs no backfill, and touches no write path; it matches the house's own 3.15.0 "metric-as-query, no maintained counters" precedent. The counter table remains a clean v2 trigger if the board ever outgrows the read-time scan. **ClickHouse gotcha:** the board base's `doc_id` collides with `doc_groups.doc_id`, so ClickHouse names the subquery column `p.doc_id` (qualified) and `b.doc_id` won't resolve through the alias — the fix is an explicit `p.doc_id AS doc_id`. **Degenerate all-zero sort:** v0 scored every post 0 and the stable sort preserved created_at DESC; the SQL falls back to a chronological `ORDER BY` to match. **Tests:** the mocked `TestPowerMeanRead` tests are re-aligned to the single-query shape (query shape, params, row mapping — a mock can't verify the SQL math). The ranking math is proven by a real-ClickHouse equivalence check (`.context/verify_power_mean_sql.py`, run against a live CH): the SQL score matches `_power_mean_score` to ≤1e-9 over a 175-input × 6-sort-config grid, and the full ranked query orders a realistic board correctly (most-loved, newest, balanced-vs-Python, paging tiles the full order). 771 API tests green (1 pre-existing `test_change_pass` passlib/bcrypt env failure, red on `origin/dev` too), ruff check + format clean. + 3.21.0 || 27.08.2026 feature(marketing-ui): D53 bite 4 — the group directory UI (the browse surface). Operator: "merged continue!" **The surface:** `/groups` (the directory) + `/groups/:id` (the detail), in `marketing/marketing-ui` alongside the app store (the same public, anon-browsable store pattern). **Directory (`GroupDirectory.tsx`):** fetches `GET /v3/groups/directory` (the minimal list of discoverable groups), renders a responsive grid of `GroupCard`s, with a search box (name/owner/slug) + topic filter chips (the directory's tags, client-side). Empty state when no groups are listed; skeleton while loading. **Detail (`GroupDetail.tsx`):** fetches `GET /v3/groups/detail?group_id=` (the flexible, principal-based read) and shows the group metadata (name, owner, join-policy badge, member count, description, tags) always; posts when the reader is a member (`posts_state: ok`), else a "join to view posts" card (the unlisted-model gated content); a not-found state for a 404. **`GroupCard`** (`components/GroupCard.tsx`): avatar (identity `avatar_ref`, else fallback letter), name, owner, join-policy badge, member count, tags; deep-links to `/groups/:id` with the URL-encoded group id (the `?api=` override survives the navigation, same as the app store). **Wiring:** routes in `App.tsx` + a "Groups" Navbar link. **Tests:** 14 new UI tests (card render/link/skeleton; directory headline/cards/empty/search/tag-filter; detail name/posts/join-to-view/404/skeleton); 228 marketing-ui tests green, build clean. Screenshots captured (desktop + 375px) per the §12 override. **This completes the `groups-directory` lane** — the group directory is built end to end (schema → endpoints → opt-in toggle → browse surface). From ef5a05a1e999f7169cd57ef51ed2173b1710f3c6 Mon Sep 17 00:00:00 2001 From: jacob-web10-nyc Date: Thu, 27 Aug 2026 20:20:17 -0400 Subject: [PATCH 4/4] =?UTF-8?q?chore:=20renumber=20pwa-listing=20entry=203?= =?UTF-8?q?.21.2=20=E2=86=92=203.22.1=20(D54's=203.22.0=20landed=20on=20de?= =?UTF-8?q?v)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- knowledge/changelogs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge/changelogs/CHANGELOG.md b/knowledge/changelogs/CHANGELOG.md index 67c16d52..0e77f772 100644 --- a/knowledge/changelogs/CHANGELOG.md +++ b/knowledge/changelogs/CHANGELOG.md @@ -1,4 +1,4 @@ -3.21.2 || 27.08.2026 +3.22.1 || 27.08.2026 fix(api): `/pwa_listing` no longer 500s when the manifest URL answers 200 with a non-JSON body — it returns NO_PWA (401) and the store falls back to the registered name, per D47. Operator: the dev store 500'd with `JSONDecodeError: Expecting value: line 1 column 1 (char 0)` on `GET /pwa_listing?url=https://dev.web10.app/docs/media/index.html`. Root cause: the registered app URL is a *file* (`.../index.html`), so the manifest lookup hits `.../index.html/manifest.json` — the static server's SPA fallback answers that with HTML + 200, and `json.loads()` raised `JSONDecodeError` (a `ValueError`, not a `RequestException`), which the endpoint's `except requests.exceptions.RequestException` never caught → 500 with the raw error class in the body. The designed behavior (D47, `app-store/overview.md`): no manifest → NO_PWA → the store falls back to the registered name, then the host. Fix: catch `json.JSONDecodeError` + `UnicodeDecodeError` → NO_PWA, with a `[pwa_listing]` log line at each NO_PWA decision point (fetch failure, non-JSON body). Also fixed a latent bug in the same call: `requests.get(url, {"Accept": ...})` passed the header dict as the **params** positional (2nd arg) — the Accept header was going out as a query string (`?Accept=application%2Fjson`); now `headers=`. Regression test: a 200 with an HTML body → 401, never a 500. Note the deeper data issue left alone (a D47 identity call, not a bug): the media demo is registered under its `index.html` file URL on dev, and a file URL can never have a manifest at `{url}/manifest.json` — the store now degrades gracefully instead of 500ing. 715 API tests green. 3.22.0 || 27.08.2026