From a47bab4e2d7ef8388014ee04b5cc4230ff855fa0 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 11 Oct 2026 17:41:23 +1100 Subject: [PATCH 1/6] feat(web-capture): add canonical social-post adapters and portable archives --- packages/connect/src/source-frontmatter.ts | 1 + .../src/source-imports-web-capture.test.ts | 35 ++ .../src/application/source-import-ports.ts | 2 + .../src/application/source-import-values.ts | 1 + packages/core/src/domain/social-post.ts | 27 ++ packages/core/src/domain/source-url.ts | 15 + packages/core/src/index.ts | 1 + packages/renderer-html/src/html-document.ts | 6 + packages/renderer-html/src/html-range.test.ts | 18 + packages/renderer-html/src/html-range.ts | 18 + packages/web-capture/src/index.ts | 2 + packages/web-capture/src/social/api.ts | 172 ++++++++ packages/web-capture/src/social/bluesky.ts | 163 ++++++++ packages/web-capture/src/social/dom.ts | 162 ++++++++ .../web-capture/src/social/fixtures/README.md | 11 + .../src/social/fixtures/bluesky-expanded.html | 17 + .../src/social/fixtures/bluesky.html | 34 ++ .../src/social/fixtures/mastodon.html | 34 ++ .../web-capture/src/social/fixtures/x.html | 51 +++ packages/web-capture/src/social/identity.ts | 53 +++ packages/web-capture/src/social/import.ts | 131 ++++++ packages/web-capture/src/social/index.ts | 42 ++ .../src/social/mastodon-profile.ts | 104 +++++ packages/web-capture/src/social/mastodon.ts | 116 ++++++ packages/web-capture/src/social/registry.ts | 86 ++++ .../web-capture/src/social/social.test.ts | 381 ++++++++++++++++++ packages/web-capture/src/social/thread.ts | 36 ++ packages/web-capture/src/social/x.ts | 61 +++ packages/web-capture/src/web-capture.ts | 28 ++ 29 files changed, 1808 insertions(+) create mode 100644 packages/core/src/domain/social-post.ts create mode 100644 packages/web-capture/src/social/api.ts create mode 100644 packages/web-capture/src/social/bluesky.ts create mode 100644 packages/web-capture/src/social/dom.ts create mode 100644 packages/web-capture/src/social/fixtures/README.md create mode 100644 packages/web-capture/src/social/fixtures/bluesky-expanded.html create mode 100644 packages/web-capture/src/social/fixtures/bluesky.html create mode 100644 packages/web-capture/src/social/fixtures/mastodon.html create mode 100644 packages/web-capture/src/social/fixtures/x.html create mode 100644 packages/web-capture/src/social/identity.ts create mode 100644 packages/web-capture/src/social/import.ts create mode 100644 packages/web-capture/src/social/index.ts create mode 100644 packages/web-capture/src/social/mastodon-profile.ts create mode 100644 packages/web-capture/src/social/mastodon.ts create mode 100644 packages/web-capture/src/social/registry.ts create mode 100644 packages/web-capture/src/social/social.test.ts create mode 100644 packages/web-capture/src/social/thread.ts create mode 100644 packages/web-capture/src/social/x.ts diff --git a/packages/connect/src/source-frontmatter.ts b/packages/connect/src/source-frontmatter.ts index 74380738..39294247 100644 --- a/packages/connect/src/source-frontmatter.ts +++ b/packages/connect/src/source-frontmatter.ts @@ -37,6 +37,7 @@ function metadataFields( ...(metadata.description ? { description: metadata.description } : {}), ...(metadata.language ? { language: metadata.language } : {}), ...(metadata.site ? { site: metadata.site } : {}), + ...(metadata.socialPost ? { social_post: metadata.socialPost } : {}), }; } diff --git a/packages/connect/src/source-imports-web-capture.test.ts b/packages/connect/src/source-imports-web-capture.test.ts index da2eee03..101bfff1 100644 --- a/packages/connect/src/source-imports-web-capture.test.ts +++ b/packages/connect/src/source-imports-web-capture.test.ts @@ -8,6 +8,41 @@ import type { ReaderConnectClient } from "./repository-client.js"; import type { PlannedSourceFileImport } from "@mdbase-reader/core"; describe("ConnectSourceImportRepository web capture provenance", () => { + it("persists post identity, thread parts and media as portable source frontmatter", async () => { + const create = vi.fn(() => Promise.resolve(success(recordDocument()))); + const repository = new ConnectSourceImportRepository( + { create } as unknown as ReaderConnectClient, + { + upload: vi.fn((path: string) => + Promise.resolve(fileDescriptor({ path, mediaType: "text/html" })), + ), + }, + ); + const part = { + platform: "x" as const, + id: "100", + url: "https://x.com/i/status/100", + author: { id: "fixture", handle: "fixture", name: "Fixture Author" }, + text: "Durable post", + media: [ + { kind: "image" as const, url: "https://media.example/image.png", alt: "A blue square" }, + ], + }; + const post = { ...part, threadRoot: part.id, parts: [part], threadPartial: true }; + const base = webCapturePlan(); + await repository.commitFile({ + ...base, + kind: "post", + metadata: { socialPost: post }, + capture: { ...base.capture!, submittedUrl: part.url, canonicalUrl: part.url }, + }); + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + frontmatter: expect.objectContaining({ kind: "post", url: part.url, social_post: post }), + }), + ); + }); + it("preserves raw and readable web representations with capture metadata", async () => { const create = vi.fn(() => Promise.resolve(success(recordDocument()))); const upload = vi.fn((path: string) => diff --git a/packages/core/src/application/source-import-ports.ts b/packages/core/src/application/source-import-ports.ts index 2620e065..d1c281a0 100644 --- a/packages/core/src/application/source-import-ports.ts +++ b/packages/core/src/application/source-import-ports.ts @@ -1,5 +1,6 @@ import type { ReaderRequestOptions } from "./ports.js"; import type { CollectionId, MutationId, SourceId } from "../domain/identity.js"; +import type { SocialPost } from "../domain/social-post.js"; import type { Source, SourceSummary } from "../domain/source.js"; import type { DateTime } from "../domain/time.js"; @@ -63,6 +64,7 @@ export interface SourceCaptureArchive { } export interface SourceImportMetadata { + readonly socialPost?: SocialPost; readonly authors?: readonly string[]; readonly published?: string; readonly description?: string; diff --git a/packages/core/src/application/source-import-values.ts b/packages/core/src/application/source-import-values.ts index f9c51fce..547dd747 100644 --- a/packages/core/src/application/source-import-values.ts +++ b/packages/core/src/application/source-import-values.ts @@ -38,6 +38,7 @@ export function validateMetadata(metadata: SourceImportMetadata): SourceImportMe ...(description ? { description } : {}), ...(language ? { language } : {}), ...(site ? { site } : {}), + ...(metadata.socialPost ? { socialPost: metadata.socialPost } : {}), }; } diff --git a/packages/core/src/domain/social-post.ts b/packages/core/src/domain/social-post.ts new file mode 100644 index 00000000..4f050c7c --- /dev/null +++ b/packages/core/src/domain/social-post.ts @@ -0,0 +1,27 @@ +export type SocialPlatform = "x" | "bluesky" | "mastodon"; + +export interface PostMedia { + readonly url: string; + readonly alt: string; + readonly kind: "image" | "video"; +} + +/** Portable post data; HTML is a reading representation, not the source identity. */ +export interface PostPart { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly author: { readonly id: string; readonly handle: string; readonly name: string }; + readonly published?: string; + readonly text: string; + readonly media: readonly PostMedia[]; + readonly quoted?: PostPart; + readonly inReplyTo?: string; +} + +export interface SocialPost extends PostPart { + readonly threadRoot: string; + readonly parts: readonly PostPart[]; + /** APIs and live pages may omit deleted, private or not-yet-loaded thread parts. */ + readonly threadPartial: boolean; +} diff --git a/packages/core/src/domain/source-url.ts b/packages/core/src/domain/source-url.ts index ea4daa16..4850d6f3 100644 --- a/packages/core/src/domain/source-url.ts +++ b/packages/core/src/domain/source-url.ts @@ -9,6 +9,17 @@ const trackingParameter = export function normalizedSourceUrl(value: string): string { const url = new URL(value); url.hash = ""; + if (/^(?:(?:www|mobile)\.)?(?:x|twitter)\.com$/u.test(url.hostname)) { + const status = /^\/(?:[^/]+\/status|i\/web\/status|i\/status)\/(\d+)(?:\/|$)/u.exec( + url.pathname, + )?.[1]; + if (status) { + return `https://x.com/i/status/${status}`; + } + } + if (url.hostname === "bsky.app" && /^\/profile\/[^/]+\/post\/[^/]+\/?$/u.test(url.pathname)) { + url.search = ""; + } const cached = ampCacheOrigin(url); if (cached) { return normalizedSourceUrl(cached); @@ -43,6 +54,10 @@ export function sameSourceUrl(value: string, normalized: string): boolean { */ export function sourceUrlSearchKey(value: string): string { const url = new URL(normalizedSourceUrl(value)); + // Every X spelling contains /status/ID, even when the stored hostname/author is old. + if (url.hostname === "x.com" && /^\/i\/status\/\d+$/u.test(url.pathname)) { + return url.pathname.replace(/^\/i/u, ""); + } return `${url.hostname}${url.pathname === "/" ? "" : url.pathname}`.toLowerCase(); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index abd82936..e6f4ac8b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -26,6 +26,7 @@ export * from "./domain/revision.js"; export type * from "./domain/search.js"; export * from "./domain/selector.js"; export * from "./domain/source.js"; +export type * from "./domain/social-post.js"; export * from "./domain/source-url.js"; export * from "./domain/text-quote-match.js"; export * from "./domain/time.js"; diff --git a/packages/renderer-html/src/html-document.ts b/packages/renderer-html/src/html-document.ts index 9d3735ee..e83d247d 100644 --- a/packages/renderer-html/src/html-document.ts +++ b/packages/renderer-html/src/html-document.ts @@ -79,6 +79,12 @@ function readerStyle(document: Document): HTMLStyleElement { html { scroll-behavior: smooth; } body { max-width: 46rem; margin: 0 auto; padding: 5rem 3rem 9rem; font: 18px/1.72 Georgia, 'Times New Roman', serif; } main, article { display: block; } + .social-post > header h1 { font-size: 2rem; margin-bottom: .5rem; } + .social-post header, .social-post time, .social-post figcaption, .social-post footer { font: .85em/1.5 ui-sans-serif, system-ui, sans-serif; color: color-mix(in srgb, currentColor 72%, transparent); } + .post-part { margin-block: 2rem; padding-top: 1.5rem; border-top: 1px solid color-mix(in srgb, currentColor 18%, transparent); } + .post-part header p { margin-block: 0 .25rem; } + .social-post figure { margin-inline: 0; } + .quoted-post .post-part { border-top: 0; padding-top: 0; } h1, h2, h3, h4 { color: inherit; font-family: ui-sans-serif, system-ui, sans-serif; line-height: 1.15; letter-spacing: -0.025em; } h1 { margin: 0 0 2.4rem; font-size: clamp(2.15rem, 6vw, 3.7rem); } h2 { margin-top: 2.8rem; font-size: 1.65rem; } diff --git a/packages/renderer-html/src/html-range.test.ts b/packages/renderer-html/src/html-range.test.ts index 81f03611..b1f269ce 100644 --- a/packages/renderer-html/src/html-range.test.ts +++ b/packages/renderer-html/src/html-range.test.ts @@ -4,6 +4,24 @@ import { describe, expect, it } from "vitest"; import { htmlSelectionDraft, locateHtmlTarget } from "./html-range.js"; describe("HTML ranges", () => { + it("anchors identical post passages to their thread part, including quoted context", () => { + document.body.innerHTML = + '

Same passage

Quoted passage

Same passage

Quoted passage

'; + const second = document.querySelector('[data-post-id="101"] > p')!; + const range = document.createRange(); + range.selectNodeContents(second); + const draft = htmlSelectionDraft({ document, range, href: "post.html", progression: 0.5 }); + expect(draft?.target.html?.css).toBe('[data-post-id="101"]'); + expect(locateHtmlTarget(document, draft!.target)?.startContainer.parentElement).toBe(second); + range.selectNodeContents(document.querySelector('[data-post-id="101"] blockquote p')!); + const quoted = htmlSelectionDraft({ document, range, href: "post.html", progression: 0.5 }); + expect(quoted?.target.html?.css).toBe('[data-post-id="101"]'); + expect( + locateHtmlTarget(document, quoted!.target)?.startContainer.parentElement?.closest( + '[data-post-id="101"]', + ), + ).not.toBeNull(); + }); it("captures quotation context and a stable CSS selector", () => { document.body.innerHTML = `

Before selected passage after.

`; const text = document.querySelector("p")?.firstChild; diff --git a/packages/renderer-html/src/html-range.ts b/packages/renderer-html/src/html-range.ts index a6a35e68..b8a8012b 100644 --- a/packages/renderer-html/src/html-range.ts +++ b/packages/renderer-html/src/html-range.ts @@ -127,6 +127,10 @@ function cssSelector(element: Element, body: HTMLElement): string { if (element === body) { return "body"; } + const post = postSelector(element); + if (post) { + return post; + } if (element.id) { return `#${escapeIdentifier(element.id)}`; } @@ -147,6 +151,20 @@ function cssSelector(element: Element, body: HTMLElement): string { return segments[0]?.startsWith("#") ? segments.join(" > ") : `body > ${segments.join(" > ")}`; } +function postSelector(element: Element): string | null { + let part = element.closest("[data-post-id]"); + // Quoted context belongs to the containing post, even when the quote appears twice. + for ( + let parent = part?.parentElement?.closest("[data-post-id]"); + parent; + parent = parent.parentElement?.closest("[data-post-id]") + ) { + part = parent; + } + const id = part?.getAttribute("data-post-id"); + return id ? `[data-post-id=${JSON.stringify(id)}]` : null; +} + function escapeIdentifier(value: string): string { return value.replace( /[^a-zA-Z0-9_-]/gu, diff --git a/packages/web-capture/src/index.ts b/packages/web-capture/src/index.ts index 7a7bcf62..5b88788e 100644 --- a/packages/web-capture/src/index.ts +++ b/packages/web-capture/src/index.ts @@ -1,4 +1,6 @@ export * from "./web-capture.js"; +export * from "./social/index.js"; +export { postHtml, postTitle, postCitation, socialPostImport } from "./social/import.js"; export * from "./scholarly-metadata.js"; export * from "./doi.js"; export * from "./page-citation.js"; diff --git a/packages/web-capture/src/social/api.ts b/packages/web-capture/src/social/api.ts new file mode 100644 index 00000000..8d83c939 --- /dev/null +++ b/packages/web-capture/src/social/api.ts @@ -0,0 +1,172 @@ +import { postAddress, type PostAddress } from "./identity.js"; + +import type { PostMedia, PostPart } from "@mdbase-reader/core"; + +export interface SocialFetchOptions { + readonly fetch?: typeof fetch; + readonly signal?: AbortSignal; +} + +/** Public, credential-free, bounded requests only. No session or undocumented endpoints. */ +export async function publicJson(url: string, options: SocialFetchOptions): Promise { + const response = await (options.fetch ?? fetch)(url, { + credentials: "omit", + redirect: "error", + headers: { accept: "application/json" }, + signal: options.signal ?? AbortSignal.timeout(12_000), + }); + if (!response.ok) { + throw new Error(`Post API returned HTTP ${String(response.status)}.`); + } + if (!response.body) { + throw new Error("Post API returned no content."); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + for (let next = await reader.read(); !next.done; next = await reader.read()) { + length += next.value.length; + if (length > 2 * 1024 * 1024) { + await reader.cancel(); + throw new Error("Post API response is too large."); + } + chunks.push(next.value); + } + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; +} + +export async function blueskyIdentity( + address: PostAddress, + options: SocialFetchOptions, +): Promise { + const actor = address.id.split("/")[2] ?? ""; + if (actor.startsWith("did:")) { + return address; + } + const result = record( + await publicJson( + `https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(actor)}`, + options, + ), + ); + const did = text(result["did"]); + const key = address.id.split("/").at(-1); + const resolved = + did.startsWith("did:") && key ? postAddress(`at://${did}/app.bsky.feed.post/${key}`) : null; + if (!resolved) { + throw new Error("Bluesky could not resolve the author's identity."); + } + return resolved; +} + +export function blueskyPart(value: unknown, depth = 0): PostPart { + const post = record(value); + const address = postAddress(text(post["uri"])); + if (!address?.id.startsWith("at://did:")) { + throw new Error("Bluesky returned an invalid post identity."); + } + const author = record(post["author"]); + const content = record(post["record"]); + const embed = record(post["embed"]); + const images = array(embed["images"] ?? record(embed["media"])["images"]); + const video = text(embed["playlist"] ?? record(embed["media"])["playlist"]); + const media: PostMedia[] = images.flatMap((value) => { + const image = record(value); + const url = https(text(image["fullsize"])); + return url ? [{ kind: "image", url, alt: text(image["alt"]) }] : []; + }); + if (https(video)) { + media.push({ kind: "video", url: video, alt: text(embed["alt"]) }); + } + const quote = record(embed["record"]); + const quoted = quote["uri"] ? quote : record(quote["record"]); + const parent = text(record(record(content["reply"])["parent"])["uri"]); + return { + platform: "bluesky", + id: address.id, + url: address.url, + author: { + id: text(author["did"]), + handle: text(author["handle"]), + name: text(author["displayName"]) || text(author["handle"]), + }, + published: text(content["createdAt"]), + text: text(content["text"]), + media, + ...(depth === 0 && quoted["uri"] + ? { quoted: blueskyPart({ ...quoted, record: quoted["value"] ?? quoted["record"] }, 1) } + : {}), + ...(parent ? { inReplyTo: parent } : {}), + }; +} + +export function mastodonPart(value: unknown, document: Document, depth = 0): PostPart { + const post = record(value); + const uri = https(text(post["uri"])); + if (!uri) { + throw new Error("Mastodon returned an invalid status URI."); + } + const account = record(post["account"]); + const parsed = document.implementation.createHTMLDocument(""); + parsed.body.innerHTML = text(post["content"]); + parsed.querySelectorAll("script,style").forEach((element) => element.remove()); + parsed.querySelectorAll("br").forEach((element) => element.replaceWith("\n")); + parsed.querySelectorAll("p").forEach((element) => element.append("\n")); + const authorUrl = https(text(account["url"])); + const handle = text(account["acct"]); + const parent = text(post["in_reply_to_id"]); + const quote = record(post["quote"])["quoted_status"]; + return { + platform: "mastodon", + id: uri, + url: uri, + author: { + id: authorUrl || text(account["id"]), + handle: handle.includes("@") ? handle : `${handle}@${new URL(uri).hostname}`, + name: text(account["display_name"]) || handle, + }, + published: text(post["created_at"]), + text: [text(post["spoiler_text"]), parsed.body.textContent.trim()].filter(Boolean).join("\n\n"), + media: array(post["media_attachments"]).flatMap((value) => { + const media = record(value); + const url = https(text(media["url"])); + return url + ? [ + { + url, + alt: text(media["description"]), + kind: text(media["type"]) === "image" ? ("image" as const) : ("video" as const), + }, + ] + : []; + }), + ...(parent ? { inReplyTo: parent } : {}), + ...(quote && depth === 0 ? { quoted: mastodonPart(quote, document, 1) } : {}), + }; +} + +export function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} +export function array(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} +export function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} +export function https(value: string): string { + try { + const url = new URL(value); + return url.protocol === "https:" && !url.username && !url.password ? url.href : ""; + } catch { + return ""; + } +} diff --git a/packages/web-capture/src/social/bluesky.ts b/packages/web-capture/src/social/bluesky.ts new file mode 100644 index 00000000..c3d557a2 --- /dev/null +++ b/packages/web-capture/src/social/bluesky.ts @@ -0,0 +1,163 @@ +import { + array, + blueskyIdentity, + blueskyPart, + publicJson, + record, + type SocialFetchOptions, +} from "./api.js"; +import { refFor, type DomSelectors, type PostRef } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { domAdapter, domFallback } from "./registry.js"; +import { unrollThread } from "./thread.js"; + +import type { PostPart } from "@mdbase-reader/core"; + +export const blueskySelectors: DomSelectors = { + post: '[data-testid^="feedItem-by-"], [data-testid="postThreadItem"], [data-testid^="postThreadItem-by-"]', + content: '[data-testid="postText"], div[dir="auto"].r-1xnzce8', + permalink: + 'a[href*="/post/"]:not([data-testid="quotePost"] a):not([href$="/reposted-by"]):not([href$="/quotes"]):not([href$="/liked-by"])', + author: + 'a[href*="/profile/"]:not([href*="/post/"]):has([dir="auto"]), a[data-testid="profileDisplayName"]', + name: '[data-testid="profileDisplayName"], a[href*="/profile/"]:not([href*="/post/"]) [dir="auto"]', + quote: '[data-testid="quotePost"]', + media: '[data-testid="postImages"] img, img[alt]:not([data-testid="userAvatarImage"] img), video', +}; + +export const blueskyAdapter = domAdapter( + "bluesky", + (value) => { + try { + return new URL(value).hostname === "bsky.app"; + } catch { + return false; + } + }, + () => false, + blueskySelectors, + async (ref, document, options = {}) => { + try { + return await blueskyThread(ref.url, options); + } catch (reason) { + options.signal?.throwIfAborted(); + const fallback = domFallback(blueskyAdapter, ref, document, blueskySelectors); + // A handle is not an identity. Even the DOM fallback must resolve it to a DID. + const provisional = postAddress(ref.url); + if (!provisional) { + throw reason; + } + const address = await blueskyIdentity(provisional, options); + const part = { + ...fallback, + id: address.id, + url: address.url, + author: { ...fallback.author, id: address.id.split("/")[2] ?? "" }, + }; + if (!part.text) { + throw reason; + } + return unrollThread(part, [], true); + } + }, + (ref, options = {}) => { + const address = postAddress(ref.url); + if (!address) { + throw new Error("Invalid Bluesky post URL."); + } + return blueskyIdentity(address, options); + }, +); + +function blueskyRef(element: Element): PostRef | null { + const ordinary = refFor(element, "bluesky", blueskySelectors); + if (ordinary) { + return ordinary; + } + // The expanded post links only to its statistics, not back to its own permalink. + if (!element.querySelector('[data-testid="repostCount-expanded"]')) { + return null; + } + const address = postAddress(element.ownerDocument.URL); + if (address?.platform !== "bluesky") { + return null; + } + const rect = element.getBoundingClientRect(); + return { + ...address, + author: element.querySelector(blueskySelectors.name)?.textContent.trim() ?? "", + snippet: + element.querySelector(blueskySelectors.content)?.textContent.trim().slice(0, 180) ?? "", + top: rect.top, + bottom: rect.bottom, + }; +} + +blueskyAdapter.items = (document) => + [...document.querySelectorAll(blueskySelectors.post)].flatMap((element) => { + const ref = blueskyRef(element); + return ref ? [ref] : []; + }); +blueskyAdapter.resolve = (element) => { + const root = element.closest(blueskySelectors.post); + return root ? blueskyRef(root) : null; +}; +blueskyAdapter.element = (ref, document) => + [...document.querySelectorAll(blueskySelectors.post)].find( + (element) => blueskyRef(element)?.id === ref.id, + ) ?? null; + +export async function blueskyThread( + url: string, + options: SocialFetchOptions, + rootFetched = false, +): Promise> { + const address = postAddress(url); + if (address?.platform !== "bluesky") { + throw new Error("Not a Bluesky post URL."); + } + const identity = await blueskyIdentity(address, options); + const result = record( + await publicJson( + `https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?uri=${encodeURIComponent(identity.id)}&depth=100&parentHeight=100`, + options, + ), + ); + const thread = record(result["thread"]); + const selected = blueskyPart(thread["post"]); + const parts: PostPart[] = []; + let partial = false; + const visit = (value: unknown, depth: number): void => { + const node = record(value); + if (!node["post"] || depth > 100) { + partial = true; + return; + } + const part = blueskyPart(node["post"]); + if (parts.some((item) => item.id === part.id)) { + return; + } + parts.push(part); + if (node["parent"]) { + visit(node["parent"], depth + 1); + } + for (const reply of array(node["replies"])) { + const author = record(record(record(reply)["post"])["author"]); + if (author["did"] === selected.author.id) { + visit(reply, depth + 1); + } + } + if (depth === 100) { + partial = true; + } + }; + visit(thread, 0); + const unrolled = unrollThread(selected, parts, partial); + if (unrolled.id !== selected.id && !rootFetched) { + return blueskyThread(unrolled.url, options, true).catch(() => { + options.signal?.throwIfAborted(); + return { ...unrolled, threadPartial: true }; + }); + } + return unrolled; +} diff --git a/packages/web-capture/src/social/dom.ts b/packages/web-capture/src/social/dom.ts new file mode 100644 index 00000000..32a4d24a --- /dev/null +++ b/packages/web-capture/src/social/dom.ts @@ -0,0 +1,162 @@ +import { postAddress, type PostAddress } from "./identity.js"; + +import type { PostMedia, PostPart, SocialPlatform } from "@mdbase-reader/core"; + +export interface PostRef { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly sourceUrl?: string; + readonly author: string; + readonly snippet: string; + readonly published?: string; + readonly top: number; + readonly bottom: number; +} + +export interface DomSelectors { + readonly post: string; + readonly content: string; + readonly permalink: string; + readonly author: string; + readonly name: string; + readonly quote: string; + readonly media: string; +} + +export function refFor( + element: Element, + platform: SocialPlatform, + selectors: DomSelectors, +): PostRef | null { + const address = elementAddress(element, platform, selectors); + if (address?.platform !== platform) { + return null; + } + const rect = element.getBoundingClientRect(); + const published = + element.querySelector("time")?.getAttribute("datetime") ?? + element.getAttribute("data-reader-post-published"); + return { + platform, + id: address.id, + url: address.url, + author: + element.querySelector(selectors.author)?.textContent.trim() ?? + element.getAttribute("data-reader-post-author") ?? + "", + snippet: ( + element.querySelector(selectors.content)?.textContent.trim() ?? + element.getAttribute("data-reader-post-text") ?? + "" + ).slice(0, 180), + ...(published ? { published } : {}), + top: rect.top, + bottom: rect.bottom, + }; +} + +function elementAddress( + element: Element, + platform: SocialPlatform, + selectors: DomSelectors, +): PostAddress | null { + const href = + element.querySelector(selectors.permalink)?.getAttribute("href") ?? + element.getAttribute("data-reader-post-url"); + if (!href) { + return null; + } + const doc = element.ownerDocument; + const fallback = + platform === "bluesky" + ? "https://bsky.app" + : platform === "x" + ? "https://x.com" + : "https://example.invalid"; + const base = doc.URL.startsWith("https://") + ? doc.URL + : (doc.querySelector('meta[property="og:url"]')?.content ?? fallback); + try { + return postAddress(new URL(href, base).href); + } catch { + return null; + } +} + +export function domPart( + element: Element, + ref: PostRef, + selectors: DomSelectors, + depth = 0, +): PostPart { + const quoteElement = element.querySelector(selectors.quote); + const quotedRef = + quoteElement && depth === 0 ? refFor(quoteElement, ref.platform, selectors) : null; + const reply = element.getAttribute("data-in-reply-to"); + return { + platform: ref.platform, + id: ref.id, + url: ref.url, + author: domAuthor(element, ref, selectors), + ...(ref.published ? { published: ref.published } : {}), + text: + element.querySelector(selectors.content)?.textContent.trim() ?? + element.getAttribute("data-reader-post-text") ?? + "", + media: domMedia(element, selectors, quoteElement), + ...(quotedRef && quoteElement + ? { quoted: domPart(quoteElement, quotedRef, selectors, 1) } + : {}), + ...(reply ? { inReplyTo: reply } : {}), + }; +} + +function domAuthor(element: Element, ref: PostRef, selectors: DomSelectors): PostPart["author"] { + const node = element.querySelector(selectors.author); + const link = node?.closest("a") ?? node?.querySelector("a"); + const href = link?.getAttribute("href"); + const handle = + ref.platform === "mastodon" + ? (node?.textContent.trim() ?? ref.author).replace(/^@/u, "") + : (href?.split("/").filter(Boolean).at(-1) ?? ref.author); + return { + id: href ?? handle, + handle, + name: element.querySelector(selectors.name)?.textContent.trim() ?? ref.author, + }; +} + +function domMedia(element: Element, selectors: DomSelectors, quote: Element | null): PostMedia[] { + return [...element.querySelectorAll(selectors.media)] + .filter((node) => !quote?.contains(node)) + .flatMap((node) => { + const source = node.getAttribute("src") ?? node.querySelector("source")?.getAttribute("src"); + const video = node.tagName === "VIDEO"; + const poster = video ? node.getAttribute("poster") : null; + const url = source?.startsWith("https://") ? source : poster; + if (!url?.startsWith("https://")) { + return []; + } + const kind = video && url === source ? ("video" as const) : ("image" as const); + const alt = + node.getAttribute("alt") ?? + node.getAttribute("aria-label") ?? + (poster ? "Video poster" : ""); + return [{ url, alt, kind }]; + }); +} + +export function nearestPost(items: readonly PostRef[], viewportHeight: number): string | null { + const middle = viewportHeight / 2; + return ( + [...items] + .filter((item) => item.bottom > 0 && item.top < viewportHeight) + .sort( + (a, b) => + Math.abs((a.top + a.bottom) / 2 - middle) - Math.abs((b.top + b.bottom) / 2 - middle), + )[0]?.id ?? + items[0]?.id ?? + null + ); +} diff --git a/packages/web-capture/src/social/fixtures/README.md b/packages/web-capture/src/social/fixtures/README.md new file mode 100644 index 00000000..ea00bfaf --- /dev/null +++ b/packages/web-capture/src/social/fixtures/README.md @@ -0,0 +1,11 @@ +# Social DOM fixtures + +Minimal, fictionalised layout fixtures. All authors, IDs, text and media URLs are +synthetic; no account sessions, page scripts or personal data are included. + +Bluesky's profile and expanded-post shapes and Mastodon's status classes were +checked in an unauthenticated LAB Chromium browser on 2026-10-11. The Bluesky +expanded item deliberately has no own permalink and no `postText` test ID. +X's fixture represents its `tweet` / `User-Name` / `tweetText` DOM contract; +there was no logged-in X browser. Its explicit reply attributes model evidence +supplied by a page, not an assumption that every X timeline supplies those edges. diff --git a/packages/web-capture/src/social/fixtures/bluesky-expanded.html b/packages/web-capture/src/social/fixtures/bluesky-expanded.html new file mode 100644 index 00000000..f1bf476a --- /dev/null +++ b/packages/web-capture/src/social/fixtures/bluesky-expanded.html @@ -0,0 +1,17 @@ + + + + Fixture expanded post + + +
+
+
Fixture Author
 +
The first durable post.
+
1 repost
 +
+
+ + diff --git a/packages/web-capture/src/social/fixtures/bluesky.html b/packages/web-capture/src/social/fixtures/bluesky.html new file mode 100644 index 00000000..4b5135dd --- /dev/null +++ b/packages/web-capture/src/social/fixtures/bluesky.html @@ -0,0 +1,34 @@ + + + + Fixture Bluesky profile + + +
+
+
Fixture Author
 +
The first durable post.
+
+ A blue square +
+
+
Context Author
 +
Quoted context.
+
+
+
+
Other Author
 +
Someone else's reply.
+
+
+ + diff --git a/packages/web-capture/src/social/fixtures/mastodon.html b/packages/web-capture/src/social/fixtures/mastodon.html new file mode 100644 index 00000000..ef0af5a0 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/mastodon.html @@ -0,0 +1,34 @@ + + + + + Fixture federated timeline + + +
+
+
+ Fixture Author +

The first durable post.

+
+ A blue square +
+
+
+ Other Author +

Someone else's reply.

+
+
+
+ + diff --git a/packages/web-capture/src/social/fixtures/x.html b/packages/web-capture/src/social/fixtures/x.html new file mode 100644 index 00000000..fa20ee29 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/x.html @@ -0,0 +1,51 @@ + + + + Fixture X timeline + + + +
+ + + + +
+ + diff --git a/packages/web-capture/src/social/identity.ts b/packages/web-capture/src/social/identity.ts new file mode 100644 index 00000000..16df5dd7 --- /dev/null +++ b/packages/web-capture/src/social/identity.ts @@ -0,0 +1,53 @@ +import type { SocialPlatform } from "@mdbase-reader/core"; + +export interface PostAddress { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly host: string; +} + +/** Handle and federated-instance addresses are provisional until resolved by the API. */ +export function postAddress(value: string): PostAddress | null { + try { + if (value.startsWith("at://")) { + return atAddress(value); + } + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password) { + return null; + } + const host = url.hostname.toLowerCase(); + if (/^(?:(?:www|mobile)\.)?(?:x|twitter)\.com$/u.test(host)) { + const id = + /^\/(?:[^/]+\/status|i\/web\/status|i\/status)\/(\d+)(?:\/(?:photo|video)\/\d+)?\/?$/u.exec( + url.pathname, + )?.[1]; + return id ? { platform: "x", id, url: `https://x.com/i/status/${id}`, host: "x.com" } : null; + } + if (host === "bsky.app") { + const match = /^\/profile\/([^/]+)\/post\/([^/]+)\/?$/u.exec(url.pathname); + return match?.[1] && match[2] ? blueskyAddress(decodeURIComponent(match[1]), match[2]) : null; + } + const id = /^\/(?:@[^/]+|users\/[^/]+\/statuses)\/(\d+)\/?$/u.exec(url.pathname)?.[1]; + return id + ? { platform: "mastodon", id, url: `${url.origin}${url.pathname.replace(/\/$/u, "")}`, host } + : null; + } catch { + return null; + } +} + +function atAddress(value: string): PostAddress | null { + const match = /^at:\/\/([^/]+)\/app\.bsky\.feed\.post\/([^/?#]+)$/u.exec(value); + return match?.[1] && match[2] ? blueskyAddress(match[1], match[2]) : null; +} + +function blueskyAddress(actor: string, key: string): PostAddress { + return { + platform: "bluesky", + id: `at://${actor}/app.bsky.feed.post/${key}`, + url: `https://bsky.app/profile/${actor}/post/${key}`, + host: "bsky.app", + }; +} diff --git a/packages/web-capture/src/social/import.ts b/packages/web-capture/src/social/import.ts new file mode 100644 index 00000000..014db6ad --- /dev/null +++ b/packages/web-capture/src/social/import.ts @@ -0,0 +1,131 @@ +import { dateTime, type PostPart, type SocialPost } from "@mdbase-reader/core"; + +import { cslDate, type CitationDraft } from "../csl-values.js"; + +import type { SocialFetchOptions } from "./api.js"; +import type { CapturedWebDocument, WebCaptureImport } from "../web-capture.js"; + +export function postCitation(post: SocialPost): CitationDraft { + return { + type: "post", + title: postTitle(post), + URL: post.url, + author: [{ literal: post.author.name || post.author.handle }], + "container-title": + post.platform === "x" ? "X" : post.platform === "bluesky" ? "Bluesky" : "Mastodon", + ...(post.published ? { issued: cslDate(post.published) ?? { literal: post.published } } : {}), + }; +} + +export function postTitle(post: SocialPost): string { + return `${post.author.name || post.author.handle}: ${post.text.replace(/\s+/gu, " ").slice(0, 120) || "Post"}`; +} + +/** Only the chosen post/thread is archived; never the surrounding feed or account DOM. */ +export function postHtml(post: SocialPost): string { + return `${escape(postTitle(post))}
`; +} + +function partHtml(part: PostPart): string { + return `

${escape(part.author.name)} · @${escape(part.author.handle)}

${part.published ? `` : ""}
${part.text + .split(/\n\n+/u) + .map((paragraph) => `

${escape(paragraph).replaceAll("\n", "
")}

`) + .join( + "", + )}
${part.media.map((media) => `
${media.kind === "image" ? `${escape(media.alt)}` : ""}
${escape(media.alt || (media.kind === "video" ? "Video (original URL retained in source metadata)" : "Image"))}
`).join("")}${part.quoted ? `
${partHtml(part.quoted)}
` : ""}
`; +} + +export async function socialPostImport( + post: SocialPost, + capture: Pick, + options: SocialFetchOptions = {}, +): Promise { + const { prepareHtmlDocument } = await import("@mdbase-reader/renderer-html"); + const archive = postHtml(post); + const parsed = new DOMParser().parseFromString(archive, "text/html"); + let remaining = 20 * 1024 * 1024; + for (const [index, image] of [...parsed.querySelectorAll("img")].entries()) { + try { + if (index >= 20 || remaining <= 0) { + throw new Error("Media budget reached"); + } + const data = await imageData(image.src, Math.min(remaining, 5 * 1024 * 1024), options); + remaining -= data.size; + image.src = data.url; + } catch { + options.signal?.throwIfAborted(); + const caption = image.parentElement?.querySelector("figcaption"); + if (caption) { + caption.append(" — image unavailable offline; original URL retained in source metadata."); + } + image.remove(); + } + } + const stem = `${post.platform}-${ + post.id + .split("/") + .at(-1) + ?.replace(/[^a-zA-Z0-9_-]/gu, "") ?? "post" + }`; + return { + name: `${stem}.readable.html`, + title: postTitle(post), + bytes: new TextEncoder().encode(prepareHtmlDocument(parsed.documentElement.outerHTML)), + archive: { name: `${stem}.archive.html`, bytes: new TextEncoder().encode(archive) }, + capture: { + submittedUrl: post.url, + canonicalUrl: post.url, + retrievedAt: dateTime(capture.retrievedAt), + }, + metadata: { + socialPost: post, + authors: [post.author.name || post.author.handle], + ...(post.published ? { published: post.published } : {}), + site: post.platform, + description: post.text.slice(0, 2_000), + }, + scholarly: { citation: postCitation(post) }, + }; +} + +async function imageData( + url: string, + maximum: number, + options: SocialFetchOptions, +): Promise<{ url: string; size: number }> { + const response = await (options.fetch ?? fetch)(url, { + credentials: "omit", + redirect: "error", + signal: options.signal ?? AbortSignal.timeout(8_000), + }); + const type = response.headers.get("content-type")?.split(";")[0]; + if (!response.ok || !type || !/^image\/(png|jpeg|webp|gif|avif)$/u.test(type) || !response.body) { + throw new Error("Image unavailable"); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (let next = await reader.read(); !next.done; next = await reader.read()) { + size += next.value.length; + if (size > maximum) { + await reader.cancel(); + throw new Error("Image too large"); + } + chunks.push(next.value); + } + let binary = ""; + for (const chunk of chunks) { + for (let offset = 0; offset < chunk.length; offset += 0x8000) { + binary += String.fromCharCode(...chunk.subarray(offset, offset + 0x8000)); + } + } + return { url: `data:${type};base64,${btoa(binary)}`, size }; +} + +function escape(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} diff --git a/packages/web-capture/src/social/index.ts b/packages/web-capture/src/social/index.ts new file mode 100644 index 00000000..b6281250 --- /dev/null +++ b/packages/web-capture/src/social/index.ts @@ -0,0 +1,42 @@ +import { blueskyAdapter, blueskyThread } from "./bluesky.js"; +import { postAddress } from "./identity.js"; +import { mastodonAdapter, mastodonThread } from "./mastodon.js"; +import { xAdapter } from "./x.js"; + +import type { SocialFetchOptions } from "./api.js"; +import type { SiteAdapter } from "./registry.js"; +import type { SocialPost } from "@mdbase-reader/core"; + +export { postAddress } from "./identity.js"; +export { nearestPost } from "./dom.js"; +export { unrollThread } from "./thread.js"; +export type { PostRef } from "./dom.js"; +export type { SiteAdapter, PostDetection } from "./registry.js"; +export type { SocialFetchOptions } from "./api.js"; + +export const siteAdapters = [xAdapter, blueskyAdapter, mastodonAdapter] as const; + +export function adapterFor(document: Document, url = document.URL): SiteAdapter | null { + return siteAdapters.find((adapter) => adapter.detect(document, url) !== null) ?? null; +} + +/** A URL-only Mastodon probe is confirmed by the status API, not assumed from its host. */ +export function postFromUrl( + url: string, + document: Document, + options: SocialFetchOptions = {}, +): Promise { + const address = postAddress(url); + if (!address) { + return Promise.resolve(null); + } + if (address.platform === "x") { + throw new Error( + "Save X posts from their live page using the mdbase Reader extension. Reader cannot retrieve X posts without that page.", + ); + } + if (address.platform === "bluesky") { + return blueskyThread(url, options); + } + return mastodonThread(url, document, options); +} diff --git a/packages/web-capture/src/social/mastodon-profile.ts b/packages/web-capture/src/social/mastodon-profile.ts new file mode 100644 index 00000000..78d0317e --- /dev/null +++ b/packages/web-capture/src/social/mastodon-profile.ts @@ -0,0 +1,104 @@ +import { array, publicJson, record, text } from "./api.js"; + +import type { SocialFetchOptions } from "./api.js"; + +const pending = new WeakMap>(); + +/** Current Mastodon anonymous profiles render text-only placeholders with no permalink. + * Match those loaded placeholders to the official account API, never to unseen API items. */ +export function prepareMastodonProfile( + document: Document, + options: SocialFetchOptions = {}, +): Promise { + const wrappers = [ + ...document.querySelectorAll(".status__wrapper:not([data-reader-post-url])"), + ].filter( + (element) => + element.children.length === 2 && + [...element.children].every((child) => child.tagName === "SPAN"), + ); + const account = /^\/@([^/]+)\/?$/u.exec(document.location.pathname)?.[1]; + if (!wrappers.length || !account) { + return Promise.resolve(); + } + const existing = pending.get(document); + if (existing) { + return existing; + } + const work = hydrate( + document.location.origin, + decodeURIComponent(account), + wrappers, + options, + ).finally(() => pending.delete(document)); + pending.set(document, work); + return work; +} + +async function hydrate( + origin: string, + account: string, + wrappers: readonly Element[], + options: SocialFetchOptions, +): Promise { + const result = record( + await publicJson( + `${origin}/api/v1/accounts/lookup?acct=${encodeURIComponent(account)}`, + options, + ), + ); + const id = text(result["id"]); + if (!/^\d+$/u.test(id)) { + throw new Error("Mastodon could not resolve this profile."); + } + const statuses: Record[] = []; + let maximum = ""; + for (let page = 0; page < 3; page++) { + const batch = array( + await publicJson( + `${origin}/api/v1/accounts/${id}/statuses?limit=40${maximum ? `&max_id=${maximum}` : ""}`, + options, + ), + ).map(record); + statuses.push(...batch); + if (batch.length < 40 || statuses.length >= wrappers.length + 40) { + break; + } + maximum = text(batch.at(-1)?.["id"]); + if (!/^\d+$/u.test(maximum)) { + break; + } + } + for (const wrapper of wrappers) { + const author = wrapper.children[0]?.textContent.trim(); + const content = wrapper.children[1]?.textContent.trim(); + const matches = statuses.filter( + (status) => + text(status["content"]).trim() === content && + text(record(status["account"])["display_name"]).trim() === author, + ); + if (matches.length !== 1) { + continue; + } + const status = matches[0]; + if (!status) { + continue; + } + const statusId = text(status["id"]); + const acct = text(record(status["account"])["acct"]); + if (!/^\d+$/u.test(statusId) || !acct) { + continue; + } + wrapper.setAttribute( + "data-reader-post-url", + `${origin}/@${encodeURIComponent(acct)}/${statusId}`, + ); + wrapper.setAttribute("data-uri", text(status["uri"])); + const parsed = wrapper.ownerDocument.implementation.createHTMLDocument(""); + parsed.body.innerHTML = text(status["content"]); + parsed.querySelectorAll("script,style").forEach((element) => element.remove()); + wrapper.setAttribute("data-reader-post-text", parsed.body.textContent); + wrapper.setAttribute("data-reader-post-author", acct); + wrapper.setAttribute("data-reader-post-published", text(status["created_at"])); + } +} diff --git a/packages/web-capture/src/social/mastodon.ts b/packages/web-capture/src/social/mastodon.ts new file mode 100644 index 00000000..62d17d69 --- /dev/null +++ b/packages/web-capture/src/social/mastodon.ts @@ -0,0 +1,116 @@ +import { array, mastodonPart, publicJson, record, text, type SocialFetchOptions } from "./api.js"; +import { postAddress } from "./identity.js"; +import { prepareMastodonProfile } from "./mastodon-profile.js"; +import { domAdapter, domFallback } from "./registry.js"; +import { unrollThread } from "./thread.js"; + +import type { DomSelectors } from "./dom.js"; + +export const mastodonSelectors: DomSelectors = { + post: ".status, .detailed-status, .entry:has(.status__content), .status__wrapper[data-reader-post-url]", + content: ".status__content, .e-content", + permalink: "a.status__relative-time, a.detailed-status__datetime, a.u-url", + author: ".display-name__account, a.status__display-name", + name: ".display-name__html", + quote: ".quoted-status, .status__quote", + media: ".media-gallery img, .attachment-list img, .video-player video", +}; + +export function mastodonMarker(document: Document): boolean { + return Boolean( + document.querySelector( + 'meta[name="application-name"][content="Mastodon"], #mastodon, .mastodon, .detailed-status, .status__content, script#initial-state', + ), + ); +} + +export const mastodonAdapter = domAdapter( + "mastodon", + () => false, + mastodonMarker, + mastodonSelectors, + async (ref, document, options = {}) => { + try { + return await mastodonThread(ref.url, document, options); + } catch (reason) { + options.signal?.throwIfAborted(); + // Do not guess a remote status's identity from a local instance's numerical ID. + const fallback = domFallback(mastodonAdapter, ref, document, mastodonSelectors); + const element = mastodonAdapter.element(ref, document); + const uri = + element?.getAttribute("data-uri") ?? + element?.querySelector('a[rel="original"]')?.href; + if (!uri?.startsWith("https://")) { + throw reason; + } + const part = { ...fallback, id: uri, url: uri }; + return unrollThread(part, [], true); + } + }, + async (ref, options = {}) => { + const address = postAddress(ref.url); + if (!address) { + throw new Error("Invalid Mastodon post URL."); + } + const status = record( + await publicJson(`https://${address.host}/api/v1/statuses/${address.id}`, options), + ); + const uri = text(status["uri"]); + if (!uri.startsWith("https://")) { + throw new Error("Invalid Mastodon identity."); + } + return { id: uri, url: uri }; + }, +); + +mastodonAdapter.prepare = prepareMastodonProfile; + +export async function mastodonThread( + url: string, + document: Document, + options: SocialFetchOptions, + rootFetched = false, +): Promise> { + const address = postAddress(url); + if (address?.platform !== "mastodon") { + throw new Error("Not a Mastodon status URL."); + } + const origin = new URL(url).origin; + const status = record(await publicJson(`${origin}/api/v1/statuses/${address.id}`, options)); + const selected = mastodonPart(status, document); + const completeness = { partial: false }; + const context = await publicJson( + `${origin}/api/v1/statuses/${address.id}/context`, + options, + ).catch(() => { + options.signal?.throwIfAborted(); + completeness.partial = true; + return {}; + }); + const statuses = [ + status, + ...array(record(context)["ancestors"]), + ...array(record(context)["descendants"]), + ].map(record); + const identities = new Map(statuses.map((item) => [text(item["id"]), text(item["uri"])])); + const parts = statuses.map((item) => { + const part = mastodonPart(item, document); + return { + ...part, + ...(part.inReplyTo ? { inReplyTo: identities.get(part.inReplyTo) ?? part.inReplyTo } : {}), + }; + }); + // Mastodon's context endpoint can be server-capped. Preserve this uncertainty. + const unrolled = unrollThread( + parts.find((part) => part.id === selected.id) ?? selected, + parts, + completeness.partial || statuses.length >= 60, + ); + if (unrolled.id !== selected.id && !rootFetched) { + return mastodonThread(unrolled.url, document, options, true).catch(() => { + options.signal?.throwIfAborted(); + return { ...unrolled, threadPartial: true }; + }); + } + return unrolled; +} diff --git a/packages/web-capture/src/social/registry.ts b/packages/web-capture/src/social/registry.ts new file mode 100644 index 00000000..bbe1e920 --- /dev/null +++ b/packages/web-capture/src/social/registry.ts @@ -0,0 +1,86 @@ +import { domPart, refFor, type DomSelectors, type PostRef } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { unrollThread } from "./thread.js"; + +import type { SocialFetchOptions } from "./api.js"; +import type { SocialPost } from "@mdbase-reader/core"; + +export type PostDetection = "single" | "multiple" | null; +export interface SiteAdapter { + readonly id: PostRef["platform"]; + matches(url: string): boolean; + prepare(document: Document, options?: SocialFetchOptions): Promise; + identity(ref: PostRef, options?: SocialFetchOptions): Promise<{ id: string; url: string }>; + detect(document: Document, url?: string): PostDetection; + items(document: Document): PostRef[]; + resolve(element: Element): PostRef | null; + element(ref: PostRef, document: Document): Element | null; + extract(ref: PostRef, document: Document, options?: SocialFetchOptions): Promise; +} + +export function domAdapter( + id: SiteAdapter["id"], + matches: SiteAdapter["matches"], + marker: (document: Document) => boolean, + selectors: DomSelectors, + extract?: SiteAdapter["extract"], + identity?: SiteAdapter["identity"], +): SiteAdapter { + const adapter: SiteAdapter = { + id, + matches, + prepare: () => Promise.resolve(), + identity: identity ?? ((ref) => Promise.resolve({ id: ref.id, url: ref.url })), + detect: (document, url = document.URL) => { + if (!matches(url) && !marker(document)) { + return null; + } + return postAddress(url)?.platform === id ? "single" : "multiple"; + }, + items: (document) => + [...document.querySelectorAll(selectors.post)] + .flatMap((element) => { + const ref = refFor(element, id, selectors); + return ref ? [ref] : []; + }) + .filter((ref, index, items) => items.findIndex((item) => item.id === ref.id) === index), + resolve: (element) => { + const post = element.closest(selectors.post); + return post ? refFor(post, id, selectors) : null; + }, + element: (ref, document) => + [...document.querySelectorAll(selectors.post)].find( + (element) => refFor(element, id, selectors)?.id === ref.id, + ) ?? null, + extract: + extract ?? + ((ref, document) => { + const element = adapter.element(ref, document); + if (!element) { + throw new Error("This post is no longer loaded. Scroll it into view and try again."); + } + const selected = domPart(element, ref, selectors); + const available = adapter.items(document).flatMap((item) => { + const node = adapter.element(item, document); + return node ? [domPart(node, item, selectors)] : []; + }); + return Promise.resolve(unrollThread(selected, available, true)); + }), + }; + return adapter; +} + +export function domFallback( + adapter: SiteAdapter, + ref: PostRef, + document: Document, + selectors: DomSelectors, +): SocialPost { + const element = adapter.element(ref, document); + if (!element) { + throw new Error( + "Could not retrieve this post. Open its permalink in the extension and try again.", + ); + } + return unrollThread(domPart(element, ref, selectors), [], true); +} diff --git a/packages/web-capture/src/social/social.test.ts b/packages/web-capture/src/social/social.test.ts new file mode 100644 index 00000000..7cca359a --- /dev/null +++ b/packages/web-capture/src/social/social.test.ts @@ -0,0 +1,381 @@ +// @vitest-environment happy-dom +import { readFileSync } from "node:fs"; + +import { normalizedSourceUrl } from "@mdbase-reader/core"; +import { beforeEach, describe, expect, it, vi, type Mock } from "vitest"; + +import { webCaptureImport } from "../web-capture.js"; + +import { blueskyPart, publicJson } from "./api.js"; +import { blueskyAdapter } from "./bluesky.js"; +import { nearestPost } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { socialPostImport } from "./import.js"; +import { mastodonAdapter } from "./mastodon.js"; +import { unrollThread } from "./thread.js"; +import { xAdapter } from "./x.js"; + +import { adapterFor, siteAdapters } from "./index.js"; + +import type { PostPart } from "@mdbase-reader/core"; + +function fixture(name: string, url: string): Document { + window.location.href = url; + const html = readFileSync(`src/social/fixtures/${name}.html`, "utf8"); + document.documentElement.innerHTML = new DOMParser().parseFromString( + html, + "text/html", + ).documentElement.innerHTML; + return document; +} + +function bskyPost( + key: string, + author = "did:plc:fixture", + parent?: string, +): Record & { uri: string } { + return { + uri: `at://${author}/app.bsky.feed.post/${key}`, + author: { did: author, handle: "fixture.test", displayName: "Fixture Author" }, + record: { + text: `Post ${key}`, + createdAt: "2026-01-01T10:00:00Z", + ...(parent ? { reply: { parent: { uri: parent } } } : {}), + }, + }; +} + +function status( + id: string, + uri: string, + parent?: string, + author = "fixture", +): Record { + return { + id, + uri, + url: uri.replace("/users/fixture/statuses/", "/@fixture/"), + account: { + id: author, + url: `https://home.example/@${author}`, + acct: `${author}@home.example`, + display_name: "Fixture Author", + }, + content: `

Post ${id}

`, + created_at: "2026-01-01T10:00:00Z", + in_reply_to_id: parent ?? null, + media_attachments: [], + }; +} + +function json(value: unknown): Response { + return new Response(JSON.stringify(value), { headers: { "content-type": "application/json" } }); +} + +function requestUrl(input: string | URL | Request): string { + return typeof input === "string" ? input : input instanceof URL ? input.href : input.url; +} + +function mockApi(handler: (url: string) => Response): Mock { + return vi.fn((input) => Promise.resolve(handler(requestUrl(input)))); +} + +beforeEach(() => window.getSelection()?.removeAllRanges()); + +describe("post identity", () => { + it.each([ + "https://x.com/fixture/status/100?s=20&t=tracking#photo", + "https://twitter.com/other/status/100", + "https://mobile.twitter.com/fixture/status/100/photo/1", + "https://mobile.x.com/i/web/status/100", + "https://x.com/i/status/100", + ])("normalises X variant %s by status ID, not author or tracking", (url) => { + expect(postAddress(url)).toMatchObject({ id: "100", url: "https://x.com/i/status/100" }); + expect(normalizedSourceUrl(url)).toBe("https://x.com/i/status/100"); + }); + it("keeps an AT URI and DID permalink identical", () => { + expect(postAddress("at://did:plc:fixture/app.bsky.feed.post/first")).toEqual( + postAddress("https://bsky.app/profile/did:plc:fixture/post/first?ref=test"), + ); + }); + it.each([ + "https://x.com/home", + "https://x.com/fixture/status/100/quotes", + "https://bsky.app/profile/fixture.test", + "https://bsky.app/search", + "https://bad.example/@fixture/no-id", + "https://x.com.evil.example/fixture/status/100", + "https://user:secret@x.com/fixture/status/100", + "http://x.com/fixture/status/100", + ])("does not make a post identity from %s", (url) => expect(postAddress(url)).toBeNull()); + it("recognises Mastodon address forms without trusting an instance hostname", () => { + expect(postAddress("https://home.example/users/fixture/statuses/100?ref=test")).toMatchObject({ + platform: "mastodon", + id: "100", + }); + expect(postAddress("https://local.example/@fixture@home.example/555")).toMatchObject({ + platform: "mastodon", + id: "555", + }); + }); +}); + +describe("saved DOM contracts", () => { + it.each([ + ["x", "https://x.com/home", xAdapter, 4], + ["bluesky", "https://bsky.app/profile/fixture.test", blueskyAdapter, 2], + ["mastodon", "https://local.example/public/local", mastodonAdapter, 2], + ] as const)( + "detects, lists and resolves %s loaded posts, not their feed", + (name, url, adapter, count) => { + const doc = fixture(name, url); + expect(adapter.detect(doc)).toBe("multiple"); + expect(adapterFor(doc)).toBe(adapter); + const refs = adapter.items(doc); + expect(refs).toHaveLength(count); + const root = adapter.element(refs[0]!, doc)!; + expect(adapter.resolve(root.querySelector("time")!)).toEqual(refs[0]); + expect(adapter.resolve(doc.body)).toBeNull(); + expect(refs[0]?.snippet).toContain("first durable post"); + window.location.href = refs[0]!.url; + expect(adapter.detect(doc)).toBe("single"); + }, + ); + it("handles Bluesky's expanded item without a self-permalink or postText marker", () => { + const doc = fixture("bluesky-expanded", "https://bsky.app/profile/fixture.test/post/first"); + const refs = blueskyAdapter.items(doc); + expect(refs).toHaveLength(1); + expect(refs[0]).toMatchObject({ + snippet: "The first durable post.", + id: "at://fixture.test/app.bsky.feed.post/first", + }); + expect(blueskyAdapter.resolve(doc.querySelector(".r-1xnzce8")!)).toEqual(refs[0]); + }); + it("falls through for ordinary sites, including arbitrary numerical paths", () => { + window.location.href = "https://ordinary.example/@user/100"; + document.body.innerHTML = "
An ordinary article
"; + document.head.innerHTML = ""; + expect(siteAdapters.every((adapter) => adapter.detect(document) === null)).toBe(true); + }); + it("extracts X content, media, quoted context and explicit self replies from DOM only", async () => { + const doc = fixture("x", "https://x.com/home"); + const post = await xAdapter.extract(xAdapter.items(doc)[1]!, doc); + expect(post.id).toBe("100"); + expect(post.parts.map((part) => part.id)).toEqual(["100", "101"]); + expect(post.quoted?.text).toContain("Quoted context"); + expect(post.media[0]).toMatchObject({ alt: "A blue square" }); + expect(post.author).toMatchObject({ handle: "fixture", name: "Fixture Author" }); + expect(post.threadPartial).toBe(true); + }); +}); + +describe("public API adapters", () => { + it("resolves Bluesky handles to DID identity and unrolls only same-author replies via AppView", async () => { + const doc = fixture("bluesky", "https://bsky.app/profile/fixture.test"); + const root = bskyPost("first"); + const fetcher = mockApi((url) => + url.includes("resolveHandle") + ? json({ did: "did:plc:fixture" }) + : json({ + thread: { + post: root, + replies: [ + { post: bskyPost("second", "did:plc:fixture", root.uri) }, + { post: bskyPost("other", "did:plc:other", root.uri) }, + ], + }, + }), + ); + const post = await blueskyAdapter.extract(blueskyAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe(root.uri); + expect(post.parts.map((part) => part.text)).toEqual(["Post first", "Post second"]); + expect(post.url).toBe("https://bsky.app/profile/did:plc:fixture/post/first"); + expect( + fetcher.mock.calls.every(([url]) => + requestUrl(url).startsWith("https://public.api.bsky.app/xrpc/"), + ), + ).toBe(true); + }); + it("uses a resolved-DID Bluesky DOM fallback when AppView thread retrieval fails", async () => { + const doc = fixture("bluesky", "https://bsky.app/profile/fixture.test"); + const fetcher = mockApi((url) => + url.includes("resolveHandle") + ? json({ did: "did:plc:fixture" }) + : new Response("", { status: 503 }), + ); + const post = await blueskyAdapter.extract(blueskyAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe("at://did:plc:fixture/app.bsky.feed.post/first"); + expect(post.text).toBe("The first durable post."); + expect(post.threadPartial).toBe(true); + }); + it("normalises federated Mastodon local IDs and home URI forms to one record", async () => { + const doc = fixture("mastodon", "https://local.example/public/local"); + const uri = "https://home.example/users/fixture/statuses/100"; + const fetcher = mockApi((url) => + url.endsWith("/context") + ? json({ + ancestors: [], + descendants: [ + status("556", "https://home.example/users/fixture/statuses/101", "555"), + status("557", "https://local.example/users/other/statuses/557", "555", "other"), + ], + }) + : json(status("555", uri)), + ); + const post = await mastodonAdapter.extract(mastodonAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe(uri); + expect(post.url).toBe(uri); + expect(post.parts.map((part) => part.text)).toEqual(["Post 555", "Post 556"]); + expect(post.parts[1]?.inReplyTo).toBe(uri); + expect( + fetcher.mock.calls.some( + ([url]) => requestUrl(url) === "https://local.example/api/v1/statuses/555/context", + ), + ).toBe(true); + }); + it("Mastodon DOM fallback requires a home URI rather than guessing from an instance ID", async () => { + const doc = fixture("mastodon", "https://local.example/public/local"); + const fetcher = mockApi(() => new Response("", { status: 503 })); + const post = await mastodonAdapter.extract(mastodonAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.url).toBe("https://home.example/users/fixture/statuses/100"); + await expect( + mastodonAdapter.extract(mastodonAdapter.items(doc)[1]!, doc, { fetch: fetcher }), + ).rejects.toThrow("HTTP 503"); + }); +}); + +describe("Mastodon anonymous profile placeholders", () => { + it("maps only loaded placeholders to official status identities; unseen API posts are not picker items", async () => { + const doc = fixture("mastodon", "https://local.example/@fixture"); + doc.body.innerHTML = + '
Fixture Author<p>Post 555</p>
'; + const known = status("555", "https://home.example/users/fixture/statuses/100"); + const fetcher = mockApi((url) => + url.includes("accounts/lookup") + ? json({ id: "20" }) + : json([known, status("777", "https://home.example/users/fixture/statuses/777")]), + ); + expect(mastodonAdapter.items(doc)).toHaveLength(0); + await mastodonAdapter.prepare(doc, { fetch: fetcher }); + const items = mastodonAdapter.items(doc); + expect(items).toHaveLength(1); + expect(items[0]).toMatchObject({ + id: "555", + snippet: "Post 555", + author: "fixture@home.example", + }); + expect(mastodonAdapter.resolve(doc.querySelector("span")!)?.id).toBe("555"); + }); + it("does not guess when two API statuses have the same author and text", async () => { + const doc = fixture("mastodon", "https://local.example/@fixture"); + doc.body.innerHTML = + '
Fixture Author<p>Post 555</p>
'; + const first = status("555", "https://home.example/users/fixture/statuses/100"); + const fetcher = mockApi((url) => + url.includes("accounts/lookup") ? json({ id: "20" }) : json([first, { ...first, id: "556" }]), + ); + await mastodonAdapter.prepare(doc, { fetch: fetcher }); + expect(mastodonAdapter.items(doc)).toHaveLength(0); + }); +}); + +describe("threads, picker and post imports", () => { + const part = (id: string, author: string, parent?: string): PostPart => ({ + platform: "x", + id, + url: `https://x.com/i/status/${id}`, + text: `Part ${id}`, + author: { id: author, handle: author, name: author }, + media: [], + ...(parent ? { inReplyTo: parent } : {}), + }); + it("unrolls ancestors and self-reply branches but excludes other authors and unrelated posts", () => { + const root = part("1", "author"); + const middle = part("2", "author", "1"); + expect( + unrollThread( + middle, + [root, middle, part("3", "author", "2"), part("4", "other", "2"), part("5", "author")], + false, + ).parts.map((item) => item.id), + ).toEqual(["1", "2", "3"]); + }); + it("terminates cyclic reply graphs", () => { + expect(unrollThread(part("1", "a", "2"), [part("2", "a", "1")], true).parts).toHaveLength(2); + }); + it("preselects the visible post closest to viewport centre", () => { + const refs = xAdapter.items(fixture("x", "https://x.com/home")); + expect( + nearestPost( + refs.map((ref, index) => ({ ...ref, top: index * 300 - 100, bottom: index * 300 + 100 })), + 800, + ), + ).toBe("101"); + expect(nearestPost([], 800)).toBeNull(); + }); + it("embeds Bluesky quoted-post context from official embed data", () => { + const quoted = { ...bskyPost("quoted", "did:plc:quoted"), value: { text: "Quoted context" } }; + const post = blueskyPart({ ...bskyPost("first"), embed: { record: quoted } }); + expect(post.quoted?.text).toBe("Quoted context"); + }); + it("stores safe readable HTML, a post-only archive and portable metadata with offline images", async () => { + const root = { + ...part("1", "author"), + text: '', + media: [ + { kind: "image" as const, url: "https://media.example/one.png", alt: "A blue square" }, + ], + }; + const prepared = await socialPostImport( + unrollThread(root, [], false), + { retrievedAt: "2026-01-01T10:00:00Z" }, + { + fetch: mockApi( + () => + new Response(new Uint8Array([1, 2, 3]), { headers: { "content-type": "image/png" } }), + ), + }, + ); + const readable = new TextDecoder().decode(prepared.bytes); + expect(readable).toContain("data:image/png;base64,"); + expect(readable).not.toContain("
Ordinary article
', + "text/html", + ); + expect(mastodonAdapter.detect(parsed, "https://ordinary-site.test/@fixture/123")).toBeNull(); + expect(adapterFor(parsed, "https://ordinary-site.test/article")).toBeNull(); + }); + it("retains Mastodon audio attachment URLs/types without pretending to archive playback", async () => { + const root = mastodonPart( + { + uri: "https://any-instance.test/users/fixture/statuses/100", + content: "

Audio note

", + account: { + url: "https://any-instance.test/@fixture", + acct: "fixture", + display_name: "Fixture Author", + }, + media_attachments: [ + { type: "audio", url: "https://media.example/note.mp3", description: "" }, + ], + }, + document, + ); + expect(root.media[0]?.kind).toBe("audio"); + const prepared = await socialPostImport(unrollThread(root, [], false), { + retrievedAt: "2026-01-01T10:00:00Z", + }); + expect(new TextDecoder().decode(prepared.bytes)).toContain( + "Audio (original URL retained in source metadata)", + ); + expect(prepared.metadata.socialPost?.media[0]?.url).toBe("https://media.example/note.mp3"); + }); it("stores safe readable HTML, a post-only archive and portable metadata with offline images", async () => { const root = { ...part("1", "author"), From 5acd667cdb4c66b08dce72ce83f60fa5e52bf5f0 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 11 Oct 2026 20:03:37 +1100 Subject: [PATCH 5/6] fix(social): keep only post identity and thread shape in frontmatter social_post stored every thread part's text and media next to the readable body that already holds them. It now keeps platform, id, url, author, published, thread root, partial flag and each part's id/url (with its quoted post). Highlights already record their post id from both the extension and Reader, so an unmarked quote resolves only on a single-post source instead of by matching text across thread parts. --- apps/extension/src/social-page.test.ts | 24 +++++++++- apps/extension/src/social-quotes.ts | 44 +++++++------------ packages/connect/src/source-frontmatter.ts | 26 ++++++++++- .../src/source-imports-web-capture.test.ts | 16 ++++++- packages/core/src/domain/social-post.ts | 18 ++++++++ 5 files changed, 95 insertions(+), 33 deletions(-) diff --git a/apps/extension/src/social-page.test.ts b/apps/extension/src/social-page.test.ts index d3ba7b23..5f805211 100644 --- a/apps/extension/src/social-page.test.ts +++ b/apps/extension/src/social-page.test.ts @@ -91,7 +91,10 @@ describe("post-by-post live capture and annotation projection", () => { it("scopes annotations on thread parts and redraws after DOM replacement", async () => { const bridge = createSocialPageBridge(document); const capture = await bridge.capture("100"); - const quotes = scopePostQuotes([{ exact: "The second durable post." }], capture?.post); + const quotes = scopePostQuotes( + [{ exact: "The second durable post.", postId: "101" }], + capture?.post, + ); expect(quotes[0]?.postUrl).toBe("https://x.com/i/status/101"); expect(bridge.draw(quotes, window.location.href).report.shown).toBe(1); const root = document.querySelectorAll('article[data-testid="tweet"]')[1]!; @@ -114,6 +117,25 @@ describe("post-by-post live capture and annotation projection", () => { ); }); + it("resolves an unmarked quote only when the source is a single post", () => { + const part = { id: "100", url: "https://x.com/i/status/100" }; + const post = { + platform: "x" as const, + ...part, + author: { id: "fixture", handle: "fixture", name: "Fixture Author" }, + threadRoot: "100", + threadPartial: true, + parts: [{ ...part, quoted: { id: "900", url: "https://x.com/i/status/900" } }], + }; + expect(scopePostQuotes([{ exact: "Anything" }], post)[0]?.postUrl).toBe(part.url); + // A quoted post is drawn inside the post quoting it. + expect(scopePostQuotes([{ exact: "Quoted", postId: "900" }], post)[0]?.postUrl).toBe(part.url); + const thread = { ...post, parts: [part, { id: "101", url: "https://x.com/i/status/101" }] }; + expect(scopePostQuotes([{ exact: "Anything" }], thread)[0]?.postUrl).toContain( + "unresolved-post", + ); + }); + it("preselects the middle post, not the first loaded post", () => { [...document.querySelectorAll('article[data-testid="tweet"]')].forEach((element, index) => { vi.spyOn(element, "getBoundingClientRect").mockReturnValue({ diff --git a/apps/extension/src/social-quotes.ts b/apps/extension/src/social-quotes.ts index 6af67505..ec0aced3 100644 --- a/apps/extension/src/social-quotes.ts +++ b/apps/extension/src/social-quotes.ts @@ -1,43 +1,31 @@ -import { matchTextQuote, type SocialPost } from "@mdbase-reader/core"; - import type { PageQuote } from "./page-annotations.js"; +import type { SocialPostReference } from "@mdbase-reader/core"; + +const unresolvedPost = "https://reader.invalid/unresolved-post"; -/** A quote is searched only inside a matching post, never inside its neighbour's text. */ +/** A quote is drawn only inside the post it was made in, never inside its neighbour's text. */ export function scopePostQuotes( quotes: readonly PageQuote[], - post?: SocialPost, + post?: SocialPostReference, ): readonly PageQuote[] { if (!post) { return quotes; } - return quotes.flatMap((quote) => { - if (quote.postId) { - const explicit = post.parts.find( - (part) => part.id === quote.postId || part.quoted?.id === quote.postId, - ); - if (explicit) { - return [{ ...quote, postUrl: explicit.url }]; - } - } - const parts = post.parts.filter((part) => { - const match = matchTextQuote(part.text, quote); - return match && !match.ambiguous; - }); - // Uncertain matches are not silently projected onto another post. - const [matched] = parts; - return [ - { - ...quote, - postUrl: - parts.length === 1 && matched ? matched.url : "https://reader.invalid/unresolved-post", - }, - ]; + const [only] = post.parts; + return quotes.map((quote) => { + // A quoted post is drawn inside the post quoting it, so it resolves to that part. + const part = quote.postId + ? post.parts.find((item) => item.id === quote.postId || item.quoted?.id === quote.postId) + : undefined; + // Without a recorded post, only a single-post source is certain; a thread is not guessed. + const url = part?.url ?? (!quote.postId && post.parts.length === 1 ? only?.url : undefined); + return { ...quote, postUrl: url ?? unresolvedPost }; }); } export function storedSocialPost( properties?: Readonly>, -): SocialPost | undefined { +): SocialPostReference | undefined { const value = properties?.["social_post"]; if ( !value || @@ -49,5 +37,5 @@ export function storedSocialPost( ) { return undefined; } - return value as SocialPost; + return value as SocialPostReference; } diff --git a/packages/connect/src/source-frontmatter.ts b/packages/connect/src/source-frontmatter.ts index 39294247..e9df00e1 100644 --- a/packages/connect/src/source-frontmatter.ts +++ b/packages/connect/src/source-frontmatter.ts @@ -2,7 +2,12 @@ import { ConnectRepositoryError } from "./repository-client.js"; import { documentEntry } from "./source-files.js"; import type { CollectionFileDescriptor } from "@mdbase-dev/connect"; -import type { PlannedSourceFileImport, PlannedSourceRepresentation } from "@mdbase-reader/core"; +import type { + PlannedSourceFileImport, + PlannedSourceRepresentation, + SocialPost, + SocialPostReference, +} from "@mdbase-reader/core"; type Descriptors = ReadonlyMap; @@ -37,7 +42,24 @@ function metadataFields( ...(metadata.description ? { description: metadata.description } : {}), ...(metadata.language ? { language: metadata.language } : {}), ...(metadata.site ? { site: metadata.site } : {}), - ...(metadata.socialPost ? { social_post: metadata.socialPost } : {}), + ...(metadata.socialPost ? { social_post: socialPostReference(metadata.socialPost) } : {}), + }; +} + +function socialPostReference(post: SocialPost): SocialPostReference { + return { + platform: post.platform, + id: post.id, + url: post.url, + author: post.author, + ...(post.published ? { published: post.published } : {}), + threadRoot: post.threadRoot, + threadPartial: post.threadPartial, + parts: post.parts.map((part) => ({ + id: part.id, + url: part.url, + ...(part.quoted ? { quoted: { id: part.quoted.id, url: part.quoted.url } } : {}), + })), }; } diff --git a/packages/connect/src/source-imports-web-capture.test.ts b/packages/connect/src/source-imports-web-capture.test.ts index 101bfff1..604deb6e 100644 --- a/packages/connect/src/source-imports-web-capture.test.ts +++ b/packages/connect/src/source-imports-web-capture.test.ts @@ -8,7 +8,7 @@ import type { ReaderConnectClient } from "./repository-client.js"; import type { PlannedSourceFileImport } from "@mdbase-reader/core"; describe("ConnectSourceImportRepository web capture provenance", () => { - it("persists post identity, thread parts and media as portable source frontmatter", async () => { + it("persists post identity and thread shape, leaving text and media to the body", async () => { const create = vi.fn(() => Promise.resolve(success(recordDocument()))); const repository = new ConnectSourceImportRepository( { create } as unknown as ReaderConnectClient, @@ -38,7 +38,19 @@ describe("ConnectSourceImportRepository web capture provenance", () => { }); expect(create).toHaveBeenCalledWith( expect.objectContaining({ - frontmatter: expect.objectContaining({ kind: "post", url: part.url, social_post: post }), + frontmatter: expect.objectContaining({ + kind: "post", + url: part.url, + social_post: { + platform: "x", + id: "100", + url: part.url, + author: part.author, + threadRoot: "100", + threadPartial: true, + parts: [{ id: "100", url: part.url }], + }, + }), }), ); }); diff --git a/packages/core/src/domain/social-post.ts b/packages/core/src/domain/social-post.ts index b4fcaab1..a9e74463 100644 --- a/packages/core/src/domain/social-post.ts +++ b/packages/core/src/domain/social-post.ts @@ -25,3 +25,21 @@ export interface SocialPost extends PostPart { /** APIs and live pages may omit deleted, private or not-yet-loaded thread parts. */ readonly threadPartial: boolean; } + +export interface PostPartReference { + readonly id: string; + readonly url: string; + readonly quoted?: { readonly id: string; readonly url: string }; +} + +/** What a source's frontmatter keeps: identity and thread shape. Text and media live in the body. */ +export interface SocialPostReference { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly author: PostPart["author"]; + readonly published?: string; + readonly threadRoot: string; + readonly threadPartial: boolean; + readonly parts: readonly PostPartReference[]; +} From 0888461f1822d8a81b6673164bf865959f2d24bd Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 11 Oct 2026 20:03:37 +1100 Subject: [PATCH 6/6] fix(social): unroll real X threads, scan feeds once and shrink images - X markup has no reply edges, so threads only unrolled for the fixture's invented attribute. On a conversation page the thread is now the author's contiguous run around the focal post; feeds still never join posts. - Adapters expose entries(), pairing each post with its element in one pass. Feed highlight redraw ran element() per post on every mutation (quadratic) and now makes a single pass with quotes grouped by post. - Images are still embedded (the renderer allows only data: images) but are scaled to 1600px and re-encoded as WebP first, within 1.5 MiB each and 6 MiB per post, down from 5 and 20 MiB. Animated GIFs are kept as is. - The archive now links every original video/audio URL, which previously lived only in frontmatter. --- apps/extension/src/social-page.ts | 21 +++-- packages/web-capture/src/social/bluesky.ts | 15 +--- .../src/social/fixtures/x-conversation.html | 45 ++++++++++ packages/web-capture/src/social/import.ts | 83 +++++++++++++++---- packages/web-capture/src/social/registry.ts | 47 +++++++---- .../web-capture/src/social/social.test.ts | 44 +++++++++- packages/web-capture/src/social/x.ts | 67 ++++++++------- 7 files changed, 236 insertions(+), 86 deletions(-) create mode 100644 packages/web-capture/src/social/fixtures/x-conversation.html diff --git a/apps/extension/src/social-page.ts b/apps/extension/src/social-page.ts index 7e23bd9a..576d00f5 100644 --- a/apps/extension/src/social-page.ts +++ b/apps/extension/src/social-page.ts @@ -188,19 +188,22 @@ function drawSocialQuotes( throw new Error("The tab navigated away."); } const adapter = adapterFor(doc); - const listing = socialItems(doc); const outcomes: PageProjection["outcomes"][number][] = quotes.map(() => "missing"); pageText({ action: "draw", quotes: [] }, doc); - if (adapter && listing) { - for (const ref of listing.items) { - const root = adapter.element(ref, doc); - if (!root) { + // Runs on every feed mutation: one pass over the posts, and quotes grouped by post up front. + const byPost = new Map(); + quotes.forEach((quote, index) => { + if (quote.postUrl) { + const url = normalizedSourceUrl(quote.postUrl); + byPost.set(url, [...(byPost.get(url) ?? []), { quote, index }]); + } + }); + if (adapter?.detect(doc) && byPost.size) { + for (const { ref, element: root } of adapter.entries(doc)) { + const matching = byPost.get(normalizedSourceUrl(canonical.get(ref.url) ?? ref.url)) ?? []; + if (!matching.length) { continue; } - const url = normalizedSourceUrl(canonical.get(ref.url) ?? ref.url); - const matching = quotes - .map((quote, index) => ({ quote, index })) - .filter(({ quote }) => quote.postUrl && normalizedSourceUrl(quote.postUrl) === url); const localFocus = matching.findIndex(({ index }) => index === focus); const projection = pageText( { diff --git a/packages/web-capture/src/social/bluesky.ts b/packages/web-capture/src/social/bluesky.ts index c3d557a2..2b0b94c7 100644 --- a/packages/web-capture/src/social/bluesky.ts +++ b/packages/web-capture/src/social/bluesky.ts @@ -67,6 +67,7 @@ export const blueskyAdapter = domAdapter( } return blueskyIdentity(address, options); }, + blueskyRef, ); function blueskyRef(element: Element): PostRef | null { @@ -93,20 +94,6 @@ function blueskyRef(element: Element): PostRef | null { }; } -blueskyAdapter.items = (document) => - [...document.querySelectorAll(blueskySelectors.post)].flatMap((element) => { - const ref = blueskyRef(element); - return ref ? [ref] : []; - }); -blueskyAdapter.resolve = (element) => { - const root = element.closest(blueskySelectors.post); - return root ? blueskyRef(root) : null; -}; -blueskyAdapter.element = (ref, document) => - [...document.querySelectorAll(blueskySelectors.post)].find( - (element) => blueskyRef(element)?.id === ref.id, - ) ?? null; - export async function blueskyThread( url: string, options: SocialFetchOptions, diff --git a/packages/web-capture/src/social/fixtures/x-conversation.html b/packages/web-capture/src/social/fixtures/x-conversation.html new file mode 100644 index 00000000..6da28722 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/x-conversation.html @@ -0,0 +1,45 @@ + + + + Fixture X conversation + + +
+ + + + + +
+ + diff --git a/packages/web-capture/src/social/import.ts b/packages/web-capture/src/social/import.ts index 9bbcf292..ab253788 100644 --- a/packages/web-capture/src/social/import.ts +++ b/packages/web-capture/src/social/import.ts @@ -33,7 +33,15 @@ function partHtml(part: PostPart): string { .map((paragraph) => `

${escape(paragraph).replaceAll("\n", "
")}

`) .join( "", - )}${part.media.map((media) => `
${media.kind === "image" ? `${escape(media.alt)}` : ""}
${escape(mediaCaption(media))}
`).join("")}${part.quoted ? `
${partHtml(part.quoted)}
` : ""}`; + )}${part.media.map(mediaHtml).join("")}${part.quoted ? `
${partHtml(part.quoted)}
` : ""}`; +} + +/** The archive keeps every original media URL; the readable copy drops external links. */ +function mediaHtml(media: PostMedia): string { + const caption = escape(mediaCaption(media)); + return media.kind === "image" + ? `
${escape(media.alt)}
${caption}
` + : `
${caption}
`; } function mediaCaption(media: PostMedia): string { @@ -42,7 +50,7 @@ function mediaCaption(media: PostMedia): string { } return media.kind === "image" ? "Image" - : `${media.kind === "audio" ? "Audio" : "Video"} (original URL retained in source metadata)`; + : `${media.kind === "audio" ? "Audio" : "Video"} (original URL kept in the archived copy)`; } export async function socialPostImport( @@ -53,20 +61,22 @@ export async function socialPostImport( const { prepareHtmlDocument } = await import("@mdbase-reader/renderer-html"); const archive = postHtml(post); const parsed = new DOMParser().parseFromString(archive, "text/html"); - let remaining = 20 * 1024 * 1024; + // Reading copies are self-contained (the renderer allows only data: images), so images + // are embedded, but scaled and re-encoded first to keep the copy light to sync and render. + let remaining = mediaBudget.total; for (const [index, image] of [...parsed.querySelectorAll("img")].entries()) { try { - if (index >= 20 || remaining <= 0) { + if (index >= mediaBudget.images || remaining <= 0) { throw new Error("Media budget reached"); } - const data = await imageData(image.src, Math.min(remaining, 5 * 1024 * 1024), options); + const data = await embeddedImage(image.src, Math.min(remaining, mediaBudget.image), options); remaining -= data.size; image.src = data.url; } catch { options.signal?.throwIfAborted(); const caption = image.parentElement?.querySelector("figcaption"); if (caption) { - caption.append(" — image unavailable offline; original URL retained in source metadata."); + caption.append(" — image unavailable offline; the archived copy keeps its original URL."); } image.remove(); } @@ -98,11 +108,35 @@ export async function socialPostImport( }; } -async function imageData( +const mediaBudget = { + images: 20, + /** Largest original accepted for scaling. */ + download: 10 * 1024 * 1024, + /** Largest embedded image, after scaling. */ + image: 1536 * 1024, + total: 6 * 1024 * 1024, +}; +const maximumDimension = 1600; + +async function embeddedImage( url: string, maximum: number, options: SocialFetchOptions, ): Promise<{ url: string; size: number }> { + const original = await downloadImage(url, options); + const image = await scaledImage(original).catch(() => original); + if (image.size > maximum) { + throw new Error("Image too large"); + } + const bytes = new Uint8Array(await image.arrayBuffer()); + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return { url: `data:${image.type};base64,${btoa(binary)}`, size: bytes.length }; +} + +async function downloadImage(url: string, options: SocialFetchOptions): Promise { const response = await (options.fetch ?? fetch)(url, { credentials: "omit", redirect: "error", @@ -113,23 +147,42 @@ async function imageData( throw new Error("Image unavailable"); } const reader = response.body.getReader(); - const chunks: Uint8Array[] = []; + const chunks: Uint8Array[] = []; let size = 0; for (let next = await reader.read(); !next.done; next = await reader.read()) { size += next.value.length; - if (size > maximum) { + if (size > mediaBudget.download) { await reader.cancel(); throw new Error("Image too large"); } chunks.push(next.value); } - let binary = ""; - for (const chunk of chunks) { - for (let offset = 0; offset < chunk.length; offset += 0x8000) { - binary += String.fromCharCode(...chunk.subarray(offset, offset + 0x8000)); - } + return new Blob(chunks, { type }); +} + +/** Large images are scaled down and re-encoded; animated GIFs are kept so they still play. */ +async function scaledImage(image: Blob): Promise { + if ( + image.type === "image/gif" || + typeof createImageBitmap !== "function" || + typeof OffscreenCanvas !== "function" + ) { + return image; + } + const bitmap = await createImageBitmap(image); + try { + const scale = Math.min(1, maximumDimension / Math.max(bitmap.width, bitmap.height)); + const canvas = new OffscreenCanvas( + Math.max(1, Math.round(bitmap.width * scale)), + Math.max(1, Math.round(bitmap.height * scale)), + ); + canvas.getContext("2d")?.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + const encoded = await canvas.convertToBlob({ type: "image/webp", quality: 0.82 }); + // Browsers without WebP encoding return PNG, which is rarely smaller than the original. + return encoded.type === "image/webp" && encoded.size < image.size ? encoded : image; + } finally { + bitmap.close(); } - return { url: `data:${type};base64,${btoa(binary)}`, size }; } function escape(value: string): string { diff --git a/packages/web-capture/src/social/registry.ts b/packages/web-capture/src/social/registry.ts index bbe1e920..77c88fc7 100644 --- a/packages/web-capture/src/social/registry.ts +++ b/packages/web-capture/src/social/registry.ts @@ -6,12 +6,21 @@ import type { SocialFetchOptions } from "./api.js"; import type { SocialPost } from "@mdbase-reader/core"; export type PostDetection = "single" | "multiple" | null; + +/** A loaded post and its element, found in one pass over the page. */ +export interface PostEntry { + readonly ref: PostRef; + readonly element: Element; +} + export interface SiteAdapter { readonly id: PostRef["platform"]; matches(url: string): boolean; prepare(document: Document, options?: SocialFetchOptions): Promise; identity(ref: PostRef, options?: SocialFetchOptions): Promise<{ id: string; url: string }>; detect(document: Document, url?: string): PostDetection; + /** Every loaded post with its element. Prefer this to `items` plus `element` per post. */ + entries(document: Document): PostEntry[]; items(document: Document): PostRef[]; resolve(element: Element): PostRef | null; element(ref: PostRef, document: Document): Element | null; @@ -25,7 +34,19 @@ export function domAdapter( selectors: DomSelectors, extract?: SiteAdapter["extract"], identity?: SiteAdapter["identity"], + refOf: (element: Element) => PostRef | null = (element) => refFor(element, id, selectors), ): SiteAdapter { + const entries = (document: Document): PostEntry[] => { + const seen = new Set(); + return [...document.querySelectorAll(selectors.post)].flatMap((element) => { + const ref = refOf(element); + if (!ref || seen.has(ref.id)) { + return []; + } + seen.add(ref.id); + return [{ ref, element }]; + }); + }; const adapter: SiteAdapter = { id, matches, @@ -37,34 +58,24 @@ export function domAdapter( } return postAddress(url)?.platform === id ? "single" : "multiple"; }, - items: (document) => - [...document.querySelectorAll(selectors.post)] - .flatMap((element) => { - const ref = refFor(element, id, selectors); - return ref ? [ref] : []; - }) - .filter((ref, index, items) => items.findIndex((item) => item.id === ref.id) === index), + entries, + items: (document) => entries(document).map(({ ref }) => ref), resolve: (element) => { const post = element.closest(selectors.post); - return post ? refFor(post, id, selectors) : null; + return post ? refOf(post) : null; }, element: (ref, document) => - [...document.querySelectorAll(selectors.post)].find( - (element) => refFor(element, id, selectors)?.id === ref.id, - ) ?? null, + entries(document).find((entry) => entry.ref.id === ref.id)?.element ?? null, extract: extract ?? ((ref, document) => { - const element = adapter.element(ref, document); + const loaded = entries(document); + const element = loaded.find((entry) => entry.ref.id === ref.id)?.element; if (!element) { throw new Error("This post is no longer loaded. Scroll it into view and try again."); } - const selected = domPart(element, ref, selectors); - const available = adapter.items(document).flatMap((item) => { - const node = adapter.element(item, document); - return node ? [domPart(node, item, selectors)] : []; - }); - return Promise.resolve(unrollThread(selected, available, true)); + const available = loaded.map((entry) => domPart(entry.element, entry.ref, selectors)); + return Promise.resolve(unrollThread(domPart(element, ref, selectors), available, true)); }), }; return adapter; diff --git a/packages/web-capture/src/social/social.test.ts b/packages/web-capture/src/social/social.test.ts index d1beaedd..3eda052d 100644 --- a/packages/web-capture/src/social/social.test.ts +++ b/packages/web-capture/src/social/social.test.ts @@ -167,6 +167,19 @@ describe("saved DOM contracts", () => { expect(post.author).toMatchObject({ handle: "fixture", name: "Fixture Author" }); expect(post.threadPartial).toBe(true); }); + it("unrolls an X thread from the author's run around the focal post, without reply markup", async () => { + const doc = fixture("x-conversation", "https://x.com/fixture/status/100"); + const focal = xAdapter.items(doc).find((item) => item.id === "100")!; + const post = await xAdapter.extract(focal, doc); + expect(post.threadRoot).toBe("99"); + expect(post.parts.map((part) => part.id)).toEqual(["99", "100", "101"]); + expect(post.threadPartial).toBe(true); + }); + it("never joins X posts into a thread on a feed", async () => { + const doc = fixture("x-conversation", "https://x.com/home"); + const ref = xAdapter.items(doc).find((item) => item.id === "100")!; + expect((await xAdapter.extract(ref, doc)).parts.map((part) => part.id)).toEqual(["100"]); + }); }); describe("public API adapters", () => { @@ -359,9 +372,38 @@ describe("portable post imports and capture safety", () => { retrievedAt: "2026-01-01T10:00:00Z", }); expect(new TextDecoder().decode(prepared.bytes)).toContain( - "Audio (original URL retained in source metadata)", + "Audio (original URL kept in the archived copy)", ); expect(prepared.metadata.socialPost?.media[0]?.url).toBe("https://media.example/note.mp3"); + expect(new TextDecoder().decode(prepared.archive.bytes)).toContain( + 'href="https://media.example/note.mp3"', + ); + }); + it("drops an image that cannot be made small enough to embed, keeping its caption", async () => { + const root = { + ...part("1", "author"), + media: [ + { kind: "image" as const, url: "https://media.example/huge.png", alt: "A huge image" }, + ], + }; + const prepared = await socialPostImport( + unrollThread(root, [], false), + { retrievedAt: "2026-01-01T10:00:00Z" }, + { + fetch: mockApi( + () => + new Response(new Uint8Array(2 * 1024 * 1024), { + headers: { "content-type": "image/png" }, + }), + ), + }, + ); + const readable = new TextDecoder().decode(prepared.bytes); + expect(readable).not.toContain("data:image"); + expect(readable).toContain("A huge image — image unavailable offline"); + expect(new TextDecoder().decode(prepared.archive.bytes)).toContain( + 'src="https://media.example/huge.png"', + ); }); it("stores safe readable HTML, a post-only archive and portable metadata with offline images", async () => { const root = { diff --git a/packages/web-capture/src/social/x.ts b/packages/web-capture/src/social/x.ts index a7d79fd8..17411844 100644 --- a/packages/web-capture/src/social/x.ts +++ b/packages/web-capture/src/social/x.ts @@ -3,6 +3,8 @@ import { postAddress } from "./identity.js"; import { domAdapter } from "./registry.js"; import { unrollThread } from "./thread.js"; +import type { PostPart } from "@mdbase-reader/core"; + /** All X layout assumptions live here. No undocumented or third-party endpoints. */ export const xSelectors: DomSelectors = { post: 'article[data-testid="tweet"]', @@ -26,36 +28,43 @@ export const xAdapter = domAdapter( () => false, xSelectors, (ref, document) => { - const element = xAdapter.element(ref, document); - if (!element) { - throw new Error("X posts must be saved from a loaded post in the Reader extension."); - } - const selected = domPart(element, ref, xSelectors); - const parts = xAdapter.items(document).flatMap((item) => { - const node = xAdapter.element(item, document); - return node ? [domPart(node, item, xSelectors)] : []; - }); - // On a conversation page X omits machine-readable reply edges. Its visible reply label - // is usable only in that conversation, never to join unrelated posts on a feed. + const parts = xAdapter + .entries(document) + .map((entry) => domPart(entry.element, entry.ref, xSelectors)); const pagePost = postAddress(document.URL); - if (pagePost?.platform === "x") { - let previous = parts.find((part) => part.id === pagePost.id); - for (const [index, part] of parts.entries()) { - if (!previous || part.id === previous.id || part.inReplyTo) { - continue; - } - const item = xAdapter.items(document)[index]; - const node = item ? xAdapter.element(item, document) : null; - const handle = previous.author.handle; - const replyLabel = node?.querySelector('[data-testid="replyingTo"]')?.textContent ?? ""; - if (part.author.id === previous.author.id && replyLabel.includes(`@${handle}`)) { - parts[index] = { ...part, inReplyTo: previous.id }; - previous = parts[index]; - } - } + const linked = pagePost?.platform === "x" ? conversationThread(parts, pagePost.id) : parts; + const selected = linked.find((part) => part.id === ref.id); + if (!selected) { + throw new Error("X posts must be saved from a loaded post in the Reader extension."); } - return Promise.resolve( - unrollThread(parts.find((part) => part.id === selected.id) ?? selected, parts, true), - ); + // X never shows every reply, so its threads are always marked partial. + return Promise.resolve(unrollThread(selected, linked, true)); }, ); + +/** + * X's markup has no reply edges. On a conversation page, the author's own thread is the run + * of their posts directly around the focal post: earlier parts above it, later parts below, + * before anyone else's reply. Feeds never get these edges, so unrelated posts never join. + */ +export function conversationThread(parts: readonly PostPart[], focalId: string): PostPart[] { + const focal = parts.findIndex((part) => part.id === focalId); + const author = parts[focal]?.author.id; + if (author === undefined) { + return [...parts]; + } + let first = focal; + while (parts[first - 1]?.author.id === author) { + first -= 1; + } + let last = focal; + while (parts[last + 1]?.author.id === author) { + last += 1; + } + return parts.map((part, index) => { + const previous = parts[index - 1]; + return previous && index > first && index <= last && !part.inReplyTo + ? { ...part, inReplyTo: previous.id } + : part; + }); +}