diff --git a/.prettierignore b/.prettierignore index 843522f6..d165059c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ pnpm-lock.yaml .playwright-mcp/ +.scratch/ dist/ coverage/ apps/capacitor/android/ diff --git a/apps/extension/src/CaptureApp.test.tsx b/apps/extension/src/CaptureApp.test.tsx index 20952ffb..1b325456 100644 --- a/apps/extension/src/CaptureApp.test.tsx +++ b/apps/extension/src/CaptureApp.test.tsx @@ -6,6 +6,44 @@ import { CaptureApp } from "./CaptureApp.js"; import type { ExtensionCaptureController } from "./capture-controller.js"; import type { PanelTab } from "./PanelTabs.js"; +it("shows a feed picker with the viewport post preselected, without offering to save a feed", () => { + const html = markup({ + postPicker: { + open: true, + preselected: "second", + items: [ + { + platform: "x", + id: "first", + url: "https://x.com/i/status/100", + author: "First author", + snippet: "First post", + top: 0, + bottom: 200, + }, + { + platform: "x", + id: "second", + url: "https://x.com/i/status/101", + author: "Second author", + snippet: "Second post", + published: "2026-01-01T10:00:00Z", + top: 300, + bottom: 500, + }, + ], + show: vi.fn(), + close: vi.fn(), + save: vi.fn(), + }, + }); + expect(html).toContain("Choose posts to save"); + expect(html).toContain("Feeds are never saved"); + expect(html).toContain("Second author"); + expect(html).toContain("Save 1 selected"); + expect(html.match(/checked=""/gu)).toHaveLength(1); +}); + function markup(changes: Partial = {}, tab?: PanelTab): string { const controller = { snapshot: { status: "unselected", connections: [] }, diff --git a/apps/extension/src/CaptureApp.tsx b/apps/extension/src/CaptureApp.tsx index 36c2f155..d5214239 100644 --- a/apps/extension/src/CaptureApp.tsx +++ b/apps/extension/src/CaptureApp.tsx @@ -8,6 +8,7 @@ import { ExtensionHeader } from "./ExtensionHeader.js"; import { HighlightFields } from "./HighlightFields.js"; import { NotePanel } from "./NotePanel.js"; import { PanelTabContent, PanelTabs, usePanelTab, type PanelTab } from "./PanelTabs.js"; +import { PostPicker } from "./PostPicker.js"; import { SaveBar, saveReady } from "./SaveBar.js"; import { SavedHighlights } from "./SavedHighlights.js"; import { openSettings, useShortcuts } from "./shortcuts.js"; @@ -78,6 +79,7 @@ export function CaptureApp({ : "Waiting for the active tab"}

+ )} diff --git a/apps/extension/src/PostPicker.test.tsx b/apps/extension/src/PostPicker.test.tsx new file mode 100644 index 00000000..0ccb48c3 --- /dev/null +++ b/apps/extension/src/PostPicker.test.tsx @@ -0,0 +1,80 @@ +// @vitest-environment happy-dom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { PostPicker } from "./PostPicker.js"; + +import type { ExtensionCaptureController } from "./capture-controller.js"; +import type { PostRef } from "@mdbase-reader/web-capture"; + +let host: HTMLDivElement; +let root: Root; +const save = vi.fn(() => Promise.resolve()); +const cancel = vi.fn(); +const items: PostRef[] = ["100", "101", "102"].map((id) => ({ + platform: "x", + id, + url: `https://x.com/i/status/${id}`, + author: "Fixture Author", + snippet: `Durable post ${id}`, + published: "2026-10-01T12:00:00Z", + top: 0, + bottom: 200, +})); + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + save.mockClear(); + host = document.createElement("div"); + document.body.append(host); + root = createRoot(host); +}); +afterEach(() => { + act(() => root.unmount()); + host.remove(); + vi.unstubAllGlobals(); +}); + +function show(loaded = items, busy = false): void { + act(() => + root.render( + , + ), + ); +} + +it("preselects the viewport-centred post, shows author/snippet/time, and supports multi-select", () => { + show(); + const inputs = host.querySelectorAll('input[type="checkbox"]'); + expect([...inputs].map((input) => input.checked)).toEqual([false, true, false]); + expect(host.textContent).toContain("Fixture Author"); + expect(host.textContent).toContain("Durable post 101"); + expect(host.querySelector("time")?.getAttribute("datetime")).toBe(items[0]?.published); + act(() => inputs[0]?.click()); + act(() => host.querySelector("button.primary")?.click()); + expect(save).toHaveBeenCalledWith(["101", "100"]); +}); + +it("excludes a selected post that virtual scrolling has removed rather than saving a stale reference", () => { + show(); + show([items[0]!]); + expect(host.querySelector("input")?.checked).toBe(false); + expect(host.querySelector("button.primary")?.disabled).toBe(true); + expect(save).not.toHaveBeenCalled(); +}); + +it("locks selection and saving while a multi-post save is in progress", () => { + show(items, true); + expect(host.querySelector("fieldset")?.disabled).toBe(true); + expect(host.querySelector("button.primary")?.disabled).toBe(true); +}); diff --git a/apps/extension/src/PostPicker.tsx b/apps/extension/src/PostPicker.tsx new file mode 100644 index 00000000..16e13963 --- /dev/null +++ b/apps/extension/src/PostPicker.tsx @@ -0,0 +1,94 @@ +import { useState } from "react"; + +import type { ExtensionCaptureController } from "./capture-controller.js"; + +type Picker = NonNullable; + +export function PostPicker({ + controller: c, +}: { + readonly controller: ExtensionCaptureController; +}): React.JSX.Element | null { + const picker = c.postPicker; + if (!picker) { + return null; + } + if (!picker.open) { + return ( + + ); + } + return ; +} + +function OpenPostPicker({ + picker, + controller: c, +}: { + readonly picker: Picker; + readonly controller: ExtensionCaptureController; +}): React.JSX.Element { + // Mounted on opening only: scrolling must not discard a deliberate choice. + const [selected, setSelected] = useState( + () => new Set(picker.preselected ? [picker.preselected] : []), + ); + const loadedSelected = [...selected].filter((id) => picker.items.some((item) => item.id === id)); + return ( +
+

Choose posts to save

+

+ Only posts currently loaded on the page are listed. Feeds are never saved. +

+
+ Posts + {picker.items.map((item) => ( + + ))} + {!picker.items.length ?

No posts loaded. Scroll the page to load some posts.

: null} +
+
+ + +
+
+ ); +} diff --git a/apps/extension/src/SaveBar.tsx b/apps/extension/src/SaveBar.tsx index 2c9fb159..0229c101 100644 --- a/apps/extension/src/SaveBar.tsx +++ b/apps/extension/src/SaveBar.tsx @@ -120,5 +120,11 @@ function submitLabel(c: ExtensionCaptureController): string { if (c.busy) { return c.status === "saving" || c.progress ? "Saving…" : "Please wait…"; } - return c.capture?.kind === "pdf" ? "Save PDF" : "Save page"; + return c.capture?.kind === "pdf" + ? "Save PDF" + : c.postPicker + ? "Choose posts to save" + : c.capture?.kind === "html" && c.capture.post + ? "Save post" + : "Save page"; } diff --git a/apps/extension/src/background-page-status.ts b/apps/extension/src/background-page-status.ts index 2e0e8ab5..f706d5cd 100644 --- a/apps/extension/src/background-page-status.ts +++ b/apps/extension/src/background-page-status.ts @@ -1,3 +1,4 @@ +import { updateSocialStatus } from "./background-social-status.js"; import { clearPageMark, markSavedPage } from "./page-badge.js"; import { pageStatusEnabledCached } from "./page-status.js"; import { panelShowsTab } from "./panel-presence.js"; @@ -73,6 +74,9 @@ export async function updatePageStatus( if (!url?.startsWith("https://") || !(await pageStatusEnabledCached())) { return; } + if (await updateSocialStatus(tabId, url, (postUrl) => savedPage(postUrl, connect), current)) { + return; + } const found = await savedPage(url, connect); if (!found || !current()) { return; diff --git a/apps/extension/src/background-social-status.test.ts b/apps/extension/src/background-social-status.test.ts new file mode 100644 index 00000000..9798e83f --- /dev/null +++ b/apps/extension/src/background-social-status.test.ts @@ -0,0 +1,84 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { updateSocialStatus } from "./background-social-status.js"; +import { resetPageMarkCache } from "./page-badge.js"; +import { fakeChrome } from "./testing/fake-chrome.js"; + +import type { PageStatusConnect } from "./page-status-connect.js"; +import type { PostRef } from "@mdbase-reader/web-capture"; + +beforeEach(() => resetPageMarkCache()); +afterEach(() => vi.unstubAllGlobals()); + +function refs(): PostRef[] { + return ["100", "101"].map((id) => ({ + platform: "x", + id, + url: `https://x.com/i/status/${id}`, + sourceUrl: "https://x.com/i/status/100", + author: "Fixture Author", + snippet: "Durable post", + top: 0, + bottom: 200, + })); +} + +function setupPagePosts(posts: readonly PostRef[]): ReturnType { + const fake = fakeChrome(); + vi.stubGlobal("chrome", { + ...fake.chrome, + scripting: { executeScript: vi.fn(() => Promise.resolve([{ result: posts }])) }, + }); + return fake; +} + +it("matches each visible post's source identity and counts/draws only visible part annotations", async () => { + const fake = setupPagePosts(refs()); + const draw = vi.fn(() => Promise.resolve()); + const api = { drawPageQuotes: draw } as unknown as PageStatusConnect; + const find = vi.fn(() => + Promise.resolve({ + api, + quotes: [ + { exact: "First", postUrl: "https://x.com/i/status/100" }, + { exact: "Second", postUrl: "https://x.com/i/status/101" }, + { exact: "Not visible", postUrl: "https://x.com/i/status/999" }, + ], + }), + ); + expect(await updateSocialStatus(1, "https://x.com/home", find, () => true)).toBe(true); + expect(find).toHaveBeenCalledOnce(); + expect(find).toHaveBeenCalledWith("https://x.com/i/status/100"); + expect(draw).toHaveBeenCalledWith( + 1, + [ + expect.objectContaining({ postUrl: "https://x.com/i/status/100" }), + expect.objectContaining({ postUrl: "https://x.com/i/status/101" }), + ], + "https://x.com/home", + ); + expect(fake.chrome.action.setBadgeText).toHaveBeenCalledWith({ tabId: 1, text: "2" }); + expect(fake.chrome.action.setTitle).toHaveBeenLastCalledWith({ + tabId: 1, + title: "2 saved posts in mdbase Reader · 2 highlights on visible posts", + }); +}); + +it("does not look up a feed URL or invent a source when no posts are loaded", async () => { + const fake = setupPagePosts([]); + const find = vi.fn(() => Promise.resolve(null)); + expect(await updateSocialStatus(1, "https://x.com/home", find, () => true)).toBe(true); + expect(find).not.toHaveBeenCalled(); + expect(fake.chrome.action.setBadgeText).not.toHaveBeenCalled(); +}); + +it("does not draw or badge stale results after navigation", async () => { + const fake = setupPagePosts(refs()); + const draw = vi.fn(); + const find = vi.fn(() => + Promise.resolve({ api: { drawPageQuotes: draw } as unknown as PageStatusConnect, quotes: [] }), + ); + await updateSocialStatus(1, "https://x.com/home", find, () => false); + expect(draw).not.toHaveBeenCalled(); + expect(fake.chrome.action.setBadgeText).not.toHaveBeenCalled(); +}); diff --git a/apps/extension/src/background-social-status.ts b/apps/extension/src/background-social-status.ts new file mode 100644 index 00000000..17da92e6 --- /dev/null +++ b/apps/extension/src/background-social-status.ts @@ -0,0 +1,75 @@ +import { normalizedSourceUrl } from "@mdbase-reader/core"; + +import { markSavedPosts } from "./page-badge.js"; + +import type { PageQuote } from "./page-annotations.js"; +import type { PageStatusConnect } from "./page-status-connect.js"; +import type { PostRef } from "@mdbase-reader/web-capture"; + +async function visiblePostRefs(tabId: number): Promise { + try { + await chrome.scripting.executeScript({ target: { tabId }, files: ["social-page.js"] }); + const [execution] = await chrome.scripting.executeScript({ + target: { tabId }, + func: async () => globalThis.mdbaseReaderSocialPage?.identities() ?? null, + }); + return execution?.result ?? null; + } catch { + return null; + } +} + +/** Per-post lookup on continuously loading pages. The bridge owns the throttled observer. */ +export async function updateSocialStatus( + tabId: number, + url: string, + find: (url: string) => Promise<{ api: PageStatusConnect; quotes: readonly PageQuote[] } | null>, + current: () => boolean, +): Promise { + const refs = await visiblePostRefs(tabId); + if (!refs) { + return false; + } + let api: PageStatusConnect | undefined; + let saved = 0; + const quotes = new Map(); + const lookups = new Map>>(); + const visible = new Set(refs.map((ref) => normalizedSourceUrl(ref.url))); + for (const ref of refs) { + const key = ref.sourceUrl ?? ref.url; + if (!lookups.has(key)) { + lookups.set(key, await find(key)); + } + const found = lookups.get(key); + if (!current()) { + return true; + } + if (!found) { + continue; + } + api = found.api; + saved++; + for (const quote of found.quotes) { + const scoped = quote.postUrl ? quote : { ...quote, postUrl: ref.url }; + if (scoped.postUrl && visible.has(normalizedSourceUrl(scoped.postUrl))) { + quotes.set(JSON.stringify(scoped), scoped); + } + } + } + if (current()) { + if (saved) { + await markSavedPosts(tabId, saved, quotes.size); + } + // Clear old highlights too, even when all the visible posts are unsaved. + if (api) { + await api.drawPageQuotes(tabId, [...quotes.values()], url); + } else { + await chrome.scripting.executeScript({ + target: { tabId }, + func: (expectedUrl: string) => globalThis.mdbaseReaderSocialPage?.draw([], expectedUrl), + args: [url], + }); + } + } + return true; +} diff --git a/apps/extension/src/background.ts b/apps/extension/src/background.ts index 97881e8c..8d1d03d8 100644 --- a/apps/extension/src/background.ts +++ b/apps/extension/src/background.ts @@ -64,6 +64,43 @@ chrome.tabs.onUpdated.addListener((tabId, change, tab) => { void updatePageStatus(tabId, tab.url, loadPageStatusConnect); } }); +const postUpdates = new Map(); +chrome.runtime.onMessage.addListener((message: unknown, sender) => { + if (!message || typeof message !== "object" || !("type" in message)) { + return; + } + if ( + message.type !== "mdbase-reader/posts-changed" && + message.type !== "mdbase-reader/posts-refresh" + ) { + return; + } + const tabId = + sender.tab?.id ?? + ("tabId" in message && typeof message.tabId === "number" ? message.tabId : undefined); + if (tabId === undefined) { + return; + } + const pending = postUpdates.get(tabId); + if (pending) { + pending.dirty = true; + return; + } + const state = { dirty: true }; + postUpdates.set(tabId, state); + void refreshPosts(tabId, state) + .catch(() => undefined) + .finally(() => postUpdates.delete(tabId)); +}); + +async function refreshPosts(tabId: number, state: { dirty: boolean }): Promise { + do { + state.dirty = false; + const tab = await chrome.tabs.get(tabId); + await updatePageStatus(tabId, tab.url, loadPageStatusConnect); + } while (postUpdates.get(tabId)?.dirty); +} + chrome.tabs.onRemoved.addListener((tabId) => { // Drafts belong to a tab; free them when it closes. void forgetTab(tabId).catch(() => undefined); diff --git a/apps/extension/src/capture-citation.ts b/apps/extension/src/capture-citation.ts index 1ea5dbb2..24787ea6 100644 --- a/apps/extension/src/capture-citation.ts +++ b/apps/extension/src/capture-citation.ts @@ -6,6 +6,7 @@ import { } from "@mdbase-reader/core"; import { citationForPage, + postCitation, extractScholarlyMetadata, type CitationDraft, type CitationPreview, @@ -21,6 +22,12 @@ export function prepareCitation( capture: PageCapture, signal?: AbortSignal, ): Promise { + if (capture.kind === "html" && capture.post) { + return Promise.resolve({ citation: postCitation(capture.post), origin: "page" }); + } + if (capture.kind === "html" && capture.social && !capture.post) { + return Promise.resolve(null); + } const embedded = capture.kind === "html" ? extractScholarlyMetadata(parsedCaptureHtml(capture.html), capture.canonicalUrl) diff --git a/apps/extension/src/capture-controller.ts b/apps/extension/src/capture-controller.ts index 99ec2bcc..1848024c 100644 --- a/apps/extension/src/capture-controller.ts +++ b/apps/extension/src/capture-controller.ts @@ -13,6 +13,7 @@ import type { SourceImportProgress, SourceSummary, } from "@mdbase-reader/core"; +import type { PostRef } from "@mdbase-reader/web-capture"; import type { Dispatch, SetStateAction } from "react"; export type CaptureStatus = "opening" | "ready" | "saving" | "saved" | "existing"; @@ -24,6 +25,14 @@ export interface PageHighlights { export interface ExtensionCaptureController { readonly snapshot: ReaderConnectSnapshot; readonly capture: PageCapture | null; + readonly postPicker?: { + readonly open: boolean; + readonly items: readonly PostRef[]; + readonly preselected: string | null; + readonly show: () => void; + readonly close: () => void; + readonly save: (ids: readonly string[]) => Promise; + }; readonly draft: CaptureDraft; readonly setDraft: Dispatch>; readonly draftRestored: boolean; diff --git a/apps/extension/src/drafts.ts b/apps/extension/src/drafts.ts index 7d54672a..6878a428 100644 --- a/apps/extension/src/drafts.ts +++ b/apps/extension/src/drafts.ts @@ -10,6 +10,7 @@ import { highlightColors, type CaptureDraft, type HighlightColor } from "./save- export interface StoredDraft { readonly draft: CaptureDraft; readonly selection: QuoteSelector | null; + readonly selectionPostId?: string; } export const emptyDraft: CaptureDraft = { diff --git a/apps/extension/src/extension.css b/apps/extension/src/extension.css index 8e38368b..13dd8d0b 100644 --- a/apps/extension/src/extension.css +++ b/apps/extension/src/extension.css @@ -3,6 +3,36 @@ matches Reader's shell (11px meta, 12px controls, 13px body, 15px headings). Monospace is reserved for keyboard shortcuts and codes. */ +.post-picker fieldset { + border: 0; + padding: 0; + max-height: 50vh; + overflow: auto; +} +.post-picker-item { + display: flex; + gap: 10px; + padding: 12px 0; + border-bottom: 1px solid var(--border); + cursor: pointer; +} +.post-picker-item > span { + min-width: 0; + display: grid; + gap: 4px; +} +.post-picker-item input { + flex: 0 0 auto; + align-self: start; +} +.post-picker-item time { + color: var(--muted); + font-size: 11px; +} +.post-picker .save-bar { + margin-top: 12px; +} + :root { /* Times is the last resort: Linux has Noto or DejaVu Serif, which sit better beside Atkinson. */ --serif: diff --git a/apps/extension/src/messages.ts b/apps/extension/src/messages.ts index b888075c..23dae6e0 100644 --- a/apps/extension/src/messages.ts +++ b/apps/extension/src/messages.ts @@ -10,7 +10,10 @@ export interface InvokeMessage { readonly tabId: number; readonly intent: CaptureIntent; } -export type ExtensionMessage = SelectionMessage | InvokeMessage; +export interface PostsChangedMessage { + readonly type: "mdbase-reader/posts-changed"; +} +export type ExtensionMessage = SelectionMessage | InvokeMessage | PostsChangedMessage; /** The intent a newly opened panel should act on; consumed once. */ export function intentKey(tabId: number): string { diff --git a/apps/extension/src/page-annotations.ts b/apps/extension/src/page-annotations.ts index 98028e5c..ed141ec9 100644 --- a/apps/extension/src/page-annotations.ts +++ b/apps/extension/src/page-annotations.ts @@ -2,6 +2,9 @@ import type { Annotation, QuoteSelector } from "@mdbase-reader/core"; export interface PageQuote extends QuoteSelector { readonly color?: string; + /** Canonical identity of the live post to search, never the surrounding feed. */ + readonly postUrl?: string; + readonly postId?: string; } export interface ProjectionReport { readonly total: number; @@ -21,6 +24,7 @@ export type PageTextRequest = | { readonly action: "selection" } | { readonly action: "draw"; + readonly append?: boolean; readonly quotes: readonly PageQuote[]; readonly expectedUrl?: string; /** Scroll to the quote with this index and mark it briefly. */ @@ -48,11 +52,30 @@ const navigatedMessage = export function annotationQuotes(annotations: readonly Annotation[]): readonly PageQuote[] { return annotations.flatMap((annotation) => annotation.target?.quote - ? [{ ...annotation.target.quote, ...(annotation.color ? { color: annotation.color } : {}) }] + ? [ + { + ...annotation.target.quote, + ...quotePostId(annotation.target.html?.css), + ...(annotation.color ? { color: annotation.color } : {}), + }, + ] : [], ); } +function quotePostId(css?: string): { postId?: string } { + const value = /^\[data-post-id=("(?:[^"\\]|\\.)*")\]/u.exec(css ?? "")?.[1]; + if (!value) { + return {}; + } + try { + const id: unknown = JSON.parse(value); + return typeof id === "string" ? { postId: id } : {}; + } catch { + return {}; + } +} + /** * Draws the quotes on the tab, optionally scrolling to the one at `focus`. The page indexes * its text, anchors the quotes and draws them in one injection, so nothing changes between. @@ -101,6 +124,10 @@ export async function injectPageText( ): Promise { let execution: chrome.scripting.InjectionResult | undefined; try { + const social = await injectSocialQuotes(tabId, request); + if (social) { + return social; + } [execution] = await chrome.scripting.executeScript({ target: { tabId }, func: pageText, @@ -124,6 +151,29 @@ export async function injectPageText( return execution.result; } +async function injectSocialQuotes( + tabId: number, + request: PageTextRequest, +): Promise { + if (request.action !== "draw" && request.action !== "reveal") { + return null; + } + if (!request.quotes.some((quote) => quote.postUrl)) { + return null; + } + const [execution] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (value: Extract) => { + const bridge = globalThis.mdbaseReaderSocialPage; + return bridge?.items() + ? { projection: bridge.draw(value.quotes, value.expectedUrl ?? location.href, value.focus) } + : null; + }, + args: [request], + }); + return execution?.result ?? null; +} + /** * Injected. Indexes the page's visible text, reads the selection, or anchors quotes in that * text and draws them. Anchoring mirrors `textQuoteMatcher` in core, which Reader uses; it @@ -131,7 +181,11 @@ export async function injectPageText( * must be nested. page-annotations.test.ts checks that the two agree. */ // eslint-disable-next-line max-lines-per-function -export function pageText(request: PageTextRequest, doc: Document = document): PageTextResult { +export function pageText( + request: PageTextRequest, + doc: Document = document, + root: Element = doc.body, +): PageTextResult { if ( "expectedUrl" in request && request.expectedUrl && @@ -167,11 +221,11 @@ export function pageText(request: PageTextRequest, doc: Document = document): Pa function indexDocument(): { text: string; nodes: Entry[] } { const nodes: Entry[] = []; let text = ""; - if (doc.body.closest(excluded)) { + if (root.closest(excluded)) { return { text, nodes }; } // Rejecting an excluded element skips its whole subtree, so no text node needs a lookup. - const walker = doc.createTreeWalker(doc.body, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, { + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, { acceptNode(node) { if (node.nodeType === Node.TEXT_NODE) { return NodeFilter.FILTER_ACCEPT; @@ -216,16 +270,24 @@ export function pageText(request: PageTextRequest, doc: Document = document): Pa return range; } + function selectedRange(): Range | null { + const selected = doc.defaultView?.getSelection(); + if (!selected?.rangeCount || selected.isCollapsed) { + return null; + } + const range = selected.getRangeAt(0); + return root.contains(range.startContainer) && root.contains(range.endContainer) ? range : null; + } + /** * Reads the selection from its own text and the context around it, without indexing the * page. Text the index leaves out is left out here too. */ function selection(): QuoteSelector | null { - const selected = doc.defaultView?.getSelection(); - if (!selected?.rangeCount || selected.isCollapsed) { + const range = selectedRange(); + if (!range) { return null; } - const range = selected.getRangeAt(0); const { indexed, step } = textWalker(); const deepestLast = (node: Node): Node => (node.lastChild ? deepestLast(node.lastChild) : node); /** The indexed text at a range boundary, moving inward when the boundary is between nodes. */ @@ -312,8 +374,8 @@ export function pageText(request: PageTextRequest, doc: Document = document): Pa return verdict; }; const indexed = (node: Node): node is Text => - node.nodeType === Node.TEXT_NODE && !isExcluded(node.parentElement); - const walker = doc.createTreeWalker(doc.body, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, { + node.nodeType === Node.TEXT_NODE && root.contains(node) && !isExcluded(node.parentElement); + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, { acceptNode(node) { if (node.nodeType === Node.TEXT_NODE) { return indexed(node) ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT; @@ -349,7 +411,13 @@ export function pageText(request: PageTextRequest, doc: Document = document): Pa } /** Named after the quote itself, so a later reveal can find it without redrawing. */ function highlightName(quote: PageQuote): string { - const key = [quote.color ?? "", quote.prefix ?? "", quote.exact, quote.suffix ?? ""].join("\0"); + const key = [ + quote.postUrl ?? "", + quote.color ?? "", + quote.prefix ?? "", + quote.exact, + quote.suffix ?? "", + ].join("\0"); let hash = 0x811c9dc5; for (let i = 0; i < key.length; i++) { hash = Math.imul(hash ^ key.charCodeAt(i), 0x01000193); @@ -408,12 +476,17 @@ export function pageText(request: PageTextRequest, doc: Document = document): Pa focused = index === focus ? { range, quote } : focused; } } - for (const name of registry.keys()) { - if (name.startsWith("mdbase-reader-")) { - registry.delete(name); + const append = request.action === "draw" && request.append; + if (!append) { + for (const name of registry.keys()) { + if (name.startsWith("mdbase-reader-")) { + registry.delete(name); + } } + doc + .querySelectorAll("style[data-mdbase-reader-highlights]") + .forEach((style) => style.remove()); } - doc.querySelector("style[data-mdbase-reader-highlights]")?.remove(); const style = doc.createElement("style"); style.dataset["mdbaseReaderHighlights"] = "true"; style.textContent = [...decorations] diff --git a/apps/extension/src/page-badge.ts b/apps/extension/src/page-badge.ts index 3f410d15..e080be44 100644 --- a/apps/extension/src/page-badge.ts +++ b/apps/extension/src/page-badge.ts @@ -43,6 +43,18 @@ export async function markSavedPage(tabId: number, highlights: number): Promise< }); } +export async function markSavedPosts( + tabId: number, + posts: number, + highlights: number, +): Promise { + await markSavedPage(tabId, posts); + await chrome.action.setTitle({ + tabId, + title: `${String(posts)} saved post${posts === 1 ? "" : "s"} in mdbase Reader · ${String(highlights)} highlights on visible posts`, + }); +} + async function resetBadge(tabId: number): Promise { await chrome.action.setBadgeText({ tabId, text: "" }); const { action } = chrome.runtime.getManifest() as { diff --git a/apps/extension/src/page-capture.ts b/apps/extension/src/page-capture.ts index b778d754..d0554629 100644 --- a/apps/extension/src/page-capture.ts +++ b/apps/extension/src/page-capture.ts @@ -6,12 +6,17 @@ import { type PageQuote, } from "./page-annotations.js"; +import type { SocialPageItems } from "./social-page.js"; import type { QuoteSelector } from "@mdbase-reader/core"; import type { LiveWebCapture } from "@mdbase-reader/web-capture"; export interface SelectedWebCapture extends LiveWebCapture { readonly kind: "html"; readonly selection: QuoteSelector | null; + readonly selectionPostId?: string; + /** The browsing page may be a feed; provenance always belongs to the post. */ + readonly pageUrl?: string; + readonly social?: SocialPageItems; } /** A PDF open in Chrome's viewer. Its bytes are fetched only when saving. */ @@ -31,7 +36,28 @@ type PageSnapshot = const maximumPdfBytes = 40 * 1024 * 1024; +export async function captureSocialTab( + tabId: number, + id?: string, +): Promise { + try { + await chrome.scripting.executeScript({ target: { tabId }, files: ["social-page.js"] }); + } catch { + return null; + } + const [execution] = await chrome.scripting.executeScript({ + target: { tabId }, + func: async (postId?: string) => globalThis.mdbaseReaderSocialPage?.capture(postId) ?? null, + args: [id], + }); + return execution?.result ?? null; +} + export async function captureTab(tabId: number): Promise { + const social = await captureSocialTab(tabId); + if (social) { + return social; + } let snapshot: PageSnapshot | undefined; try { [{ result: snapshot } = { result: undefined }] = await chrome.scripting.executeScript({ diff --git a/apps/extension/src/page-status-connect.ts b/apps/extension/src/page-status-connect.ts index e411099c..0f10777b 100644 --- a/apps/extension/src/page-status-connect.ts +++ b/apps/extension/src/page-status-connect.ts @@ -5,6 +5,7 @@ import { restoreCollection, } from "./connect-session.js"; import { annotationQuotes, drawPageQuotes, type PageQuote } from "./page-annotations.js"; +import { scopePostQuotes, storedSocialPost } from "./social-quotes.js"; import type { ReaderConnectedCollection, @@ -97,7 +98,7 @@ async function savedQuotes( collection.collectionId, source.id, ); - return annotationQuotes(annotations); + return scopePostQuotes(annotationQuotes(annotations), storedSocialPost(source.properties)); } export const pageStatusConnect: PageStatusConnect = { diff --git a/apps/extension/src/save-capture.ts b/apps/extension/src/save-capture.ts index 2b69ff4c..c00ecd67 100644 --- a/apps/extension/src/save-capture.ts +++ b/apps/extension/src/save-capture.ts @@ -15,6 +15,7 @@ import { type DocumentDescriptor, type MutationId, type QuoteSelector, + type HtmlSelector, type Source, type SourceFileImportRequest, type SourceImportMetadata, @@ -22,7 +23,12 @@ import { type SourceSummary, } from "@mdbase-reader/core"; import { createReaderRuntimeServices, type KeyValueStorage } from "@mdbase-reader/platform"; -import { citationAuthors, citationYear, webCaptureImport } from "@mdbase-reader/web-capture"; +import { + citationAuthors, + citationYear, + postCitation, + webCaptureImport, +} from "@mdbase-reader/web-capture"; import { citekeyNeighbours, @@ -83,6 +89,11 @@ export interface SaveCaptureInput { const savedTextLimit = 4; /** A saved source with the reading copy this save just uploaded, when it created the source. */ +interface SavedText { + readonly text: string; + readonly parts: ReadonlyMap; +} + interface CreatedSource { readonly source: Source; readonly readable: Uint8Array | null; @@ -93,7 +104,7 @@ export class CaptureWriter { readonly #runtime; readonly #annotations = new Map(); /** Keyed by the exact stored revision, so a changed copy is never matched from here. */ - readonly #savedTexts = new Map(); + readonly #savedTexts = new Map(); constructor(private readonly journalStorage: KeyValueStorage) { this.#runtime = createReaderRuntimeServices(journalStorage); @@ -101,6 +112,7 @@ export class CaptureWriter { async save(input: SaveCaptureInput): Promise { const { session, collection, capture, draft, onSource } = input; + assertPostCapture(capture); const outcomes = await session.recoverPendingMutations(); const failure = outcomes.find((outcome) => !outcome.ok); if (failure) { @@ -134,7 +146,13 @@ export class CaptureWriter { void rememberSavedUrls(collection.collectionId, [capture.canonicalUrl, capture.submittedUrl]); const annotation = draft.highlight && capture.kind === "html" && capture.selection - ? await this.#highlight(collection, created ?? source, capture.selection, draft) + ? await this.#highlight( + collection, + created ?? source, + capture.selection, + draft, + capture.selectionPostId, + ) : null; return { source, existing, annotation, notices }; } @@ -144,7 +162,8 @@ export class CaptureWriter { notices: string[], recoverExistingFiles: boolean, ): Promise { - const { collection, capture, draft, citation } = input; + const { collection, capture, draft } = input; + const citation = captureCitation(input); // The citekey lookup does not depend on the new source; fetch it while the page imports. const neighbours = citation ? citekeyNeighbours(collection, citation.citation) : null; neighbours?.catch(() => undefined); @@ -189,6 +208,7 @@ export class CaptureWriter { saved: SourceSummary | CreatedSource, selection: QuoteSelector, draft: CaptureDraft, + postId?: string, ): Promise { // A source this save created is current as returned; others are read for their revision. const created = "readable" in saved ? saved : null; @@ -209,6 +229,7 @@ export class CaptureWriter { comment, draft.color, draft.highlightTags, + ...(postId ? [postId] : []), ]); const recoveryKey = `capture-annotation:${await this.#runtime.hasher.sha256(new TextEncoder().encode(key))}`; const previous = @@ -233,11 +254,12 @@ export class CaptureWriter { } } this.#annotations.set(key, identity); - const { document, quote } = await this.#savedTarget( + const { document, quote, html } = await this.#savedTarget( collection, source, selection, created?.readable ?? null, + postId, ); await this.journalStorage.set(recoveryKey, JSON.stringify(identity)); const result = await createAnnotation( @@ -268,7 +290,7 @@ export class CaptureWriter { .filter(Boolean), ), ], - target: { quote }, + target: { quote, ...(html ? { html } : {}) }, body: highlightBody(quote.exact, comment), }, ); @@ -311,14 +333,21 @@ export class CaptureWriter { source: Source, selection: QuoteSelector, uploaded: Uint8Array | null, - ): Promise<{ document: DocumentDescriptor; quote: QuoteSelector }> { + postId?: string, + ): Promise<{ document: DocumentDescriptor; quote: QuoteSelector; html?: HtmlSelector }> { const document = source.documents.find( (item) => ["primary", "readable"].includes(item.role) && item.mediaType === "text/html", ); if (!document) { throw new Error("This source has no saved HTML reading copy. Open it in Reader to annotate."); } - const text = await this.#savedText(collection, source, document, uploaded); + const savedText = await this.#savedText(collection, source, document, uploaded); + const text = postId ? savedText.parts.get(postId) : savedText.text; + if (text === undefined) { + throw new Error( + "This thread part is absent from the saved copy. Open that copy in Reader; no highlight was created.", + ); + } const match = matchTextQuote(text, selection); if (!match || match.ambiguous) { throw new Error( @@ -326,7 +355,11 @@ export class CaptureWriter { ); } // Stored as the saved copy reads, which is where Reader anchors it. - return { document, quote: textQuoteAt(text, match.start, match.end) }; + return { + document, + quote: textQuoteAt(text, match.start, match.end), + ...(postId ? { html: { css: `[data-post-id=${JSON.stringify(postId)}]` } } : {}), + }; } /** The reading copy's text at exactly `document.revision`, verified against its digest. */ @@ -335,7 +368,7 @@ export class CaptureWriter { source: Source, document: DocumentDescriptor, uploaded: Uint8Array | null, - ): Promise { + ): Promise { const key = JSON.stringify([ collection.collectionId, source.id, @@ -354,7 +387,15 @@ export class CaptureWriter { ? uploaded : await this.#downloadVerified(collection, document); const parsed = new DOMParser().parseFromString(new TextDecoder().decode(bytes), "text/html"); - const text = pageText({ action: "text" }, parsed).text ?? ""; + const text: SavedText = { + text: pageText({ action: "text" }, parsed).text ?? "", + parts: new Map( + [...parsed.querySelectorAll("[data-post-id]")].map((part) => [ + part.getAttribute("data-post-id") ?? "", + pageText({ action: "text" }, parsed, part).text ?? "", + ]), + ), + }; this.#savedTexts.set(key, text); for (const oldest of this.#savedTexts.keys()) { if (this.#savedTexts.size <= savedTextLimit) { @@ -383,6 +424,22 @@ export class CaptureWriter { } } +function captureCitation(input: SaveCaptureInput): CitationPreview | null { + if (input.citation) { + return input.citation; + } + const capture = input.capture; + return capture.kind === "html" && capture.post + ? { citation: postCitation(capture.post), origin: "page" } + : null; +} + +function assertPostCapture(capture: PageCapture): void { + if (capture.kind === "html" && capture.social && !capture.post) { + throw new Error("Choose individual posts; feeds cannot be saved."); + } +} + function importRecoveryKey({ collection, capture }: SaveCaptureInput): string { return `capture-import:${collection.collectionId}:${capture.canonicalUrl}`; } @@ -426,6 +483,7 @@ async function htmlRequest(capture: SelectedWebCapture): Promise { archive: prepared.archive, capture: prepared.capture, metadata: prepared.metadata, + ...(capture.post ? { kind: "post" } : {}), }; } diff --git a/apps/extension/src/save-social-post.test.ts b/apps/extension/src/save-social-post.test.ts new file mode 100644 index 00000000..0bc4e30a --- /dev/null +++ b/apps/extension/src/save-social-post.test.ts @@ -0,0 +1,86 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; + +import { annotationQuotes } from "./page-annotations.js"; +import { scopePostQuotes } from "./social-quotes.js"; +import { capture, fixture } from "./testing/save-capture-fixture.js"; + +import type { SelectedWebCapture } from "./page-capture.js"; + +function postCapture(): SelectedWebCapture { + const first = { + platform: "x" as const, + id: "100", + url: "https://x.com/i/status/100", + author: { id: "fixture", handle: "fixture", name: "Fixture Author" }, + text: "Repeated passage", + media: [], + }; + const second = { ...first, id: "101", url: "https://x.com/i/status/101", inReplyTo: "100" }; + return { + ...capture, + html: "", + canonicalUrl: first.url, + submittedUrl: first.url, + selection: { exact: "Repeated passage" }, + selectionPostId: second.id, + post: { ...first, threadRoot: first.id, threadPartial: true, parts: [first, second] }, + }; +} + +describe("social source/highlight persistence", () => { + it("creates a post source and post CSL citation, with a part-specific HTML selector", async () => { + const f = fixture(); + const page = postCapture(); + const saved = await f.save({}, page); + expect(f.commit.mock.calls[0]?.[0]).toMatchObject({ + kind: "post", + metadata: { socialPost: page.post }, + }); + expect(saved.source.citation?.type).toBe("post"); + expect(saved.annotation?.target).toMatchObject({ + quote: { exact: "Repeated passage" }, + html: { css: '[data-post-id="101"]' }, + }); + const quotes = annotationQuotes([saved.annotation!]); + expect(scopePostQuotes(quotes, page.post)[0]?.postUrl).toBe("https://x.com/i/status/101"); + }); + it("does not mistake identical selections on two thread parts for a highlight retry", async () => { + const f = fixture(); + const page = postCapture(); + const first = await f.save({}, page); + const second = await f.save( + {}, + { ...page, selectionPostId: "100" }, + { known: first.source, lookedUp: true }, + ); + expect(second.source.id).toBe(first.source.id); + expect(second.annotation?.id).not.toBe(first.annotation?.id); + expect(second.annotation?.target?.html?.css).toBe('[data-post-id="100"]'); + expect(f.commit).toHaveBeenCalledOnce(); + expect(f.create).toHaveBeenCalledTimes(2); + }); + it("refuses to anchor a highlight to a part absent from the saved revision", async () => { + const f = fixture(); + await expect(f.save({}, { ...postCapture(), selectionPostId: "999" })).rejects.toThrow( + "part is absent", + ); + expect(f.onSource).toHaveBeenCalledOnce(); + expect(f.create).not.toHaveBeenCalled(); + }); + it("never sends an unselected feed to the source importer", async () => { + const f = fixture(); + await expect( + f.save( + {}, + { + ...capture, + html: "", + selection: null, + social: { mode: "multiple", items: [], nearest: null }, + }, + ), + ).rejects.toThrow("feeds cannot be saved"); + expect(f.commit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/social-page.test.ts b/apps/extension/src/social-page.test.ts new file mode 100644 index 00000000..5f805211 --- /dev/null +++ b/apps/extension/src/social-page.test.ts @@ -0,0 +1,149 @@ +// @vitest-environment happy-dom +import { readFileSync } from "node:fs"; + +import { matchTextQuote } from "@mdbase-reader/core"; +import { siteAdapters } from "@mdbase-reader/web-capture"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { pageText } from "./page-annotations.js"; +import { createSocialPageBridge, selectedPost } from "./social-page.js"; +import { scopePostQuotes } from "./social-quotes.js"; + +const registry = new Map>(); + +beforeEach(() => { + window.location.href = "https://x.com/home"; + document.documentElement.innerHTML = new DOMParser().parseFromString( + readFileSync("../../packages/web-capture/src/social/fixtures/x.html", "utf8"), + "text/html", + ).documentElement.innerHTML; + window.getSelection()?.removeAllRanges(); + registry.clear(); + vi.stubGlobal( + "Highlight", + class extends Set { + constructor(...ranges: Range[]) { + super(ranges); + } + }, + ); + Object.defineProperty(window, "CSS", { value: { highlights: registry }, configurable: true }); +}); +afterEach(() => vi.unstubAllGlobals()); + +function select(start: Node, end: Node = start): void { + const range = document.createRange(); + range.setStart(start, 0); + range.setEnd(end, end.textContent?.length ?? 0); + window.getSelection()!.removeAllRanges(); + window.getSelection()!.addRange(range); +} + +describe("post-by-post live capture and annotation projection", () => { + it("does not snapshot or identify a feed as a source", async () => { + const capture = await createSocialPageBridge(document).capture(); + expect(capture).toMatchObject({ social: { mode: "multiple" }, html: "", selection: null }); + expect(capture?.post).toBeUndefined(); + }); + it("resolves a feed selection to its post and anchors it to the unrolled readable text", async () => { + const texts = document.querySelectorAll('[data-testid="tweetText"]'); + const second = texts[2]!.firstChild!; + select(second); + const bridge = createSocialPageBridge(document); + expect(selectedPost(document, siteAdapters[0])?.id).toBe("101"); + const capture = await bridge.capture(); + expect(capture?.canonicalUrl).toBe("https://x.com/i/status/100"); + expect(capture?.selection?.exact).toBe("The second durable post."); + const parsed = new DOMParser().parseFromString(capture!.html, "text/html"); + expect( + matchTextQuote(pageText({ action: "text" }, parsed).text ?? "", capture!.selection!), + ).toMatchObject({ ambiguous: false }); + expect(capture?.html).not.toContain("Someone else's reply"); + expect(capture?.html).not.toContain("Not a source"); + expect(capture?.pageUrl).toBe("https://x.com/home"); + }); + it("does not resolve selections that cross two posts", () => { + const texts = document.querySelectorAll('[data-testid="tweetText"]'); + select(texts[0]!.firstChild!, texts[2]!.firstChild!); + expect(selectedPost(document, siteAdapters[0])).toBeNull(); + }); + it("saves only the chosen loaded post, with no network call for X", async () => { + const fetcher = vi.fn(() => Promise.reject(new Error("No X network API allowed"))); + vi.stubGlobal("fetch", fetcher); + const capture = await createSocialPageBridge(document).capture("103"); + expect(capture?.post?.parts).toHaveLength(1); + expect(capture?.post?.id).toBe("103"); + expect(fetcher).not.toHaveBeenCalled(); + }); + it("never redraws a saved quote in a neighbouring post with identical text", () => { + const texts = document.querySelectorAll('[data-testid="tweetText"]'); + texts[0]!.textContent = "Same durable quotation."; + texts[2]!.textContent = "Same durable quotation."; + const bridge = createSocialPageBridge(document); + const projection = bridge.draw( + [{ exact: "Same durable quotation.", postUrl: "https://x.com/i/status/101" }], + window.location.href, + ); + expect(projection.report).toMatchObject({ shown: 1, ambiguous: 0 }); + const range = [...registry.values()][0]?.values().next().value; + expect(range?.startContainer.parentElement).toBe(texts[2]); + }); + it("scopes annotations on thread parts and redraws after DOM replacement", async () => { + const bridge = createSocialPageBridge(document); + const capture = await bridge.capture("100"); + const quotes = scopePostQuotes( + [{ exact: "The second durable post.", postId: "101" }], + capture?.post, + ); + expect(quotes[0]?.postUrl).toBe("https://x.com/i/status/101"); + expect(bridge.draw(quotes, window.location.href).report.shown).toBe(1); + const root = document.querySelectorAll('article[data-testid="tweet"]')[1]!; + root.replaceWith(root.cloneNode(true)); + expect(bridge.draw(quotes, window.location.href).report.shown).toBe(1); + expect([...registry.values()][0]?.values().next().value?.startContainer.isConnected).toBe(true); + }); + it("does not guess which thread part owns an identical quote without explicit part evidence", async () => { + const bridge = createSocialPageBridge(document); + const capture = await bridge.capture("100"); + const post = { + ...capture!.post!, + parts: capture!.post!.parts.map((part) => ({ ...part, text: "Identical passage" })), + }; + expect(scopePostQuotes([{ exact: "Identical passage" }], post)[0]?.postUrl).toContain( + "unresolved-post", + ); + expect(scopePostQuotes([{ exact: "Identical passage", postId: "101" }], post)[0]?.postUrl).toBe( + "https://x.com/i/status/101", + ); + }); + + it("resolves an unmarked quote only when the source is a single post", () => { + const part = { id: "100", url: "https://x.com/i/status/100" }; + const post = { + platform: "x" as const, + ...part, + author: { id: "fixture", handle: "fixture", name: "Fixture Author" }, + threadRoot: "100", + threadPartial: true, + parts: [{ ...part, quoted: { id: "900", url: "https://x.com/i/status/900" } }], + }; + expect(scopePostQuotes([{ exact: "Anything" }], post)[0]?.postUrl).toBe(part.url); + // A quoted post is drawn inside the post quoting it. + expect(scopePostQuotes([{ exact: "Quoted", postId: "900" }], post)[0]?.postUrl).toBe(part.url); + const thread = { ...post, parts: [part, { id: "101", url: "https://x.com/i/status/101" }] }; + expect(scopePostQuotes([{ exact: "Anything" }], thread)[0]?.postUrl).toContain( + "unresolved-post", + ); + }); + + it("preselects the middle post, not the first loaded post", () => { + [...document.querySelectorAll('article[data-testid="tweet"]')].forEach((element, index) => { + vi.spyOn(element, "getBoundingClientRect").mockReturnValue({ + top: index * 200, + bottom: index * 200 + 150, + } as DOMRect); + }); + Object.defineProperty(window, "innerHeight", { value: 800, configurable: true }); + expect(createSocialPageBridge(document).items()?.nearest).toBe("102"); + }); +}); diff --git a/apps/extension/src/social-page.ts b/apps/extension/src/social-page.ts new file mode 100644 index 00000000..576d00f5 --- /dev/null +++ b/apps/extension/src/social-page.ts @@ -0,0 +1,327 @@ +import { matchTextQuote, normalizedSourceUrl, textQuoteAt } from "@mdbase-reader/core"; +import { + adapterFor, + nearestPost, + postAddress, + postHtml, + postTitle, +} from "@mdbase-reader/web-capture"; + +import { pageText } from "./page-annotations.js"; + +import type { PageProjection, PageQuote } from "./page-annotations.js"; +import type { SelectedWebCapture } from "./page-capture.js"; +import type { QuoteSelector, SocialPost } from "@mdbase-reader/core"; +import type { PostRef, SiteAdapter } from "@mdbase-reader/web-capture"; + +export interface SocialPageItems { + readonly mode: "single" | "multiple"; + readonly items: readonly PostRef[]; + readonly nearest: string | null; +} +export interface SocialPageBridge { + items(): SocialPageItems | null; + refreshItems(): Promise; + capture(id?: string): Promise; + identities(): Promise; + draw(quotes: readonly PageQuote[], expectedUrl: string, focus?: number): PageProjection; +} + +declare global { + var mdbaseReaderSocialPage: SocialPageBridge | undefined; +} + +function socialItems(doc: Document): SocialPageItems | null { + const adapter = adapterFor(doc); + const mode = adapter?.detect(doc); + if (!adapter || !mode) { + return null; + } + const items = adapter.items(doc); + return { mode, items, nearest: nearestPost(items, doc.defaultView?.innerHeight ?? 800) }; +} + +function boundaryElement(node: Node): Element | null { + return node.nodeType === Node.ELEMENT_NODE ? (node as Element) : node.parentElement; +} + +/** Selections crossing post boundaries do not resolve to either post. */ +export function selectedPost(doc: Document, adapter: SiteAdapter): PostRef | null { + const selected = doc.getSelection(); + if (!selected?.rangeCount || selected.isCollapsed) { + return null; + } + const range = selected.getRangeAt(0); + const start = boundaryElement(range.startContainer); + const end = boundaryElement(range.endContainer); + if (!start || !end) { + return null; + } + const ref = adapter.resolve(start); + return ref && adapter.resolve(end)?.id === ref.id ? ref : null; +} + +function captureRef( + doc: Document, + adapter: SiteAdapter, + listing: SocialPageItems, + id?: string, +): PostRef | null { + if (id) { + return listing.items.find((item) => item.id === id) ?? null; + } + const selection = selectedPost(doc, adapter); + if (selection) { + return selection; + } + const address = postAddress(doc.location.href); + if (listing.mode !== "single" || !address) { + return null; + } + // An explicit permalink can be fetched publicly; X always needs its matching live DOM. + return ( + listing.items.find((item) => item.id === address.id) ?? + (adapter.id === "x" ? null : { ...address, author: "", snippet: "", top: 0, bottom: 0 }) + ); +} + +async function resolvePost( + adapter: SiteAdapter, + ref: PostRef, + doc: Document, +): Promise<{ post: SocialPost; identity: { id: string; url: string } }> { + const post = await adapter.extract(ref, doc); + const identity = await adapter.identity(ref).catch((reason: unknown) => { + if (post.parts.length !== 1) { + throw reason; + } + return { id: post.id, url: post.url }; + }); + return { post, identity }; +} + +function readableSelection( + post: SocialPost, + id: string, + raw: QuoteSelector | null, +): QuoteSelector | null { + if (!raw) { + return null; + } + const parsed = new DOMParser().parseFromString(postHtml(post), "text/html"); + const part = [...parsed.querySelectorAll("[data-post-id]")].find( + (node) => node.getAttribute("data-post-id") === id, + ); + if (!part) { + return null; + } + const partText = pageText({ action: "text" }, parsed, part).text ?? ""; + const match = matchTextQuote(partText, raw); + if (!match || match.ambiguous) { + throw new Error("The selected passage is ambiguous inside this post. Select a longer passage."); + } + // Context is taken from the post's readable representation, not feed chrome or neighbours. + return textQuoteAt(partText, match.start, match.end); +} + +async function captureSocialPage( + doc: Document, + canonical: Map, + id?: string, +): Promise { + const adapter = adapterFor(doc); + await adapter?.prepare(doc); + const listing = socialItems(doc); + if (!adapter || !listing) { + return null; + } + const ref = captureRef(doc, adapter, listing, id); + const pageUrl = doc.location.href; + const retrievedAt = new Date().toISOString(); + if (!ref) { + if (id || listing.mode === "single") { + throw new Error("The post is not loaded. Scroll it into view and try again."); + } + return { + kind: "html", + submittedUrl: pageUrl, + canonicalUrl: pageUrl, + retrievedAt, + pageTitle: "Choose posts to save", + html: "", + selection: null, + pageUrl, + social: listing, + }; + } + const root = adapter.element(ref, doc); + const selected = selectedPost(doc, adapter); + const raw = + root && selected?.id === ref.id + ? (pageText({ action: "selection" }, doc, root).selection ?? null) + : null; + const { post, identity } = await resolvePost(adapter, ref, doc); + canonical.set(ref.url, identity.url); + return { + kind: "html", + submittedUrl: post.url, + canonicalUrl: post.url, + retrievedAt, + pageTitle: postTitle(post), + html: postHtml(post), + selection: readableSelection(post, identity.id, raw), + ...(raw ? { selectionPostId: identity.id } : {}), + post, + pageUrl, + social: listing, + }; +} + +function drawSocialQuotes( + doc: Document, + canonical: ReadonlyMap, + quotes: readonly PageQuote[], + expectedUrl: string, + focus?: number, +): PageProjection { + if (doc.location.href.split("#")[0] !== expectedUrl.split("#")[0]) { + throw new Error("The tab navigated away."); + } + const adapter = adapterFor(doc); + const outcomes: PageProjection["outcomes"][number][] = quotes.map(() => "missing"); + pageText({ action: "draw", quotes: [] }, doc); + // Runs on every feed mutation: one pass over the posts, and quotes grouped by post up front. + const byPost = new Map(); + quotes.forEach((quote, index) => { + if (quote.postUrl) { + const url = normalizedSourceUrl(quote.postUrl); + byPost.set(url, [...(byPost.get(url) ?? []), { quote, index }]); + } + }); + if (adapter?.detect(doc) && byPost.size) { + for (const { ref, element: root } of adapter.entries(doc)) { + const matching = byPost.get(normalizedSourceUrl(canonical.get(ref.url) ?? ref.url)) ?? []; + if (!matching.length) { + continue; + } + const localFocus = matching.findIndex(({ index }) => index === focus); + const projection = pageText( + { + action: "draw", + quotes: matching.map(({ quote }) => quote), + append: true, + ...(localFocus === -1 ? {} : { focus: localFocus }), + }, + doc, + root, + ).projection; + matching.forEach(({ index }, position) => { + outcomes[index] = projection?.outcomes[position] ?? "missing"; + }); + } + } + return { + outcomes, + report: { + total: quotes.length, + shown: outcomes.filter((value) => value === "shown").length, + missing: outcomes.filter((value) => value === "missing").length, + ambiguous: outcomes.filter((value) => value === "ambiguous").length, + }, + }; +} + +export function createSocialPageBridge(doc: Document): SocialPageBridge { + const identities = new Map>(); + const canonical = new Map(); + return { + items: () => socialItems(doc), + refreshItems: async () => { + await adapterFor(doc)?.prepare(doc); + return socialItems(doc); + }, + capture: (id) => captureSocialPage(doc, canonical, id), + identities: async () => { + const adapter = adapterFor(doc); + await adapter?.prepare(doc).catch(() => undefined); + const listing = socialItems(doc); + if (!adapter || !listing) { + return null; + } + const height = doc.defaultView?.innerHeight ?? 800; + const visible = listing.items.filter((ref) => ref.bottom > 0 && ref.top < height); + return Promise.all( + visible.map((ref) => { + let pending = identities.get(ref.id); + if (!pending) { + pending = resolvePost(adapter, ref, doc).then(({ post, identity }) => { + canonical.set(ref.url, identity.url); + return { ...ref, id: identity.id, url: identity.url, sourceUrl: post.url }; + }); + if (identities.size >= 500) { + const oldest = identities.keys().next().value; + if (oldest) { + identities.delete(oldest); + } + } + identities.set(ref.id, pending); + void pending.catch(() => identities.delete(ref.id)); + } + return pending.catch(() => ref); + }), + ); + }, + draw: (quotes, url, focus) => drawSocialQuotes(doc, canonical, quotes, url, focus), + }; +} + +function installSocialPage(): void { + globalThis.mdbaseReaderSocialPage = createSocialPageBridge(document); + let timer: ReturnType | undefined; + const changed = (): void => { + if (timer) { + return; + } + timer = setTimeout(() => { + timer = undefined; + if (globalThis.mdbaseReaderSocialPage?.items()) { + void chrome.runtime.sendMessage({ type: "mdbase-reader/posts-changed" }).catch(() => { + observer.disconnect(); + document.removeEventListener("scroll", changed); + }); + } + }, 600); + }; + const observer = new MutationObserver((mutations) => { + if (mutations.some(postMutation)) { + changed(); + } + }); + observer.observe(document.body, { + childList: true, + subtree: true, + characterData: true, + attributes: true, + attributeFilter: ["href", "datetime"], + }); + document.addEventListener("scroll", changed, { passive: true }); +} + +function postMutation(mutation: MutationRecord): boolean { + const target = boundaryElement(mutation.target); + if (target?.closest("[data-mdbase-reader-highlights],[data-mdbase-reader-ui]")) { + return false; + } + return ( + mutation.type !== "childList" || + [...mutation.addedNodes, ...mutation.removedNodes].some( + (node) => + !(node instanceof Element) || + !node.matches("[data-mdbase-reader-highlights],[data-mdbase-reader-ui]"), + ) + ); +} + +if (typeof chrome !== "undefined" && !globalThis.mdbaseReaderSocialPage) { + installSocialPage(); +} diff --git a/apps/extension/src/social-quotes.ts b/apps/extension/src/social-quotes.ts new file mode 100644 index 00000000..ec0aced3 --- /dev/null +++ b/apps/extension/src/social-quotes.ts @@ -0,0 +1,41 @@ +import type { PageQuote } from "./page-annotations.js"; +import type { SocialPostReference } from "@mdbase-reader/core"; + +const unresolvedPost = "https://reader.invalid/unresolved-post"; + +/** A quote is drawn only inside the post it was made in, never inside its neighbour's text. */ +export function scopePostQuotes( + quotes: readonly PageQuote[], + post?: SocialPostReference, +): readonly PageQuote[] { + if (!post) { + return quotes; + } + const [only] = post.parts; + return quotes.map((quote) => { + // A quoted post is drawn inside the post quoting it, so it resolves to that part. + const part = quote.postId + ? post.parts.find((item) => item.id === quote.postId || item.quoted?.id === quote.postId) + : undefined; + // Without a recorded post, only a single-post source is certain; a thread is not guessed. + const url = part?.url ?? (!quote.postId && post.parts.length === 1 ? only?.url : undefined); + return { ...quote, postUrl: url ?? unresolvedPost }; + }); +} + +export function storedSocialPost( + properties?: Readonly>, +): SocialPostReference | undefined { + const value = properties?.["social_post"]; + if ( + !value || + typeof value !== "object" || + !("parts" in value) || + !Array.isArray(value.parts) || + !("url" in value) || + typeof value.url !== "string" + ) { + return undefined; + } + return value as SocialPostReference; +} diff --git a/apps/extension/src/use-extension-capture.test.tsx b/apps/extension/src/use-extension-capture.test.tsx index 36a5b368..b674901b 100644 --- a/apps/extension/src/use-extension-capture.test.tsx +++ b/apps/extension/src/use-extension-capture.test.tsx @@ -128,6 +128,62 @@ afterEach(async () => { vi.unstubAllGlobals(); }); +it("follows same-document navigation rather than leaving the old post in the panel", async () => { + await mount(); + mocks.captureTab.mockResolvedValueOnce({ + ...page, + canonicalUrl: "https://example.com/new", + submittedUrl: "https://example.com/new", + }); + await act(async () => chromeFake.emit("updated", 1, { url: "https://example.com/new" })); + expect(controller.capture?.canonicalUrl).toBe("https://example.com/new"); + expect(controller.navigated).toBe(false); +}); + +it("never discovers a feed snapshot and ignores a slow, superseded selection capture", async () => { + const feed = { + ...page, + canonicalUrl: "https://x.com/home", + submittedUrl: "https://x.com/home", + selection: null, + social: { mode: "multiple", items: [], nearest: null }, + }; + mocks.captureTab.mockResolvedValue(feed); + await mount(); + expect(mocks.sourceForUrl).not.toHaveBeenCalled(); + const first = { + ...feed, + canonicalUrl: "https://x.com/i/status/100", + selection: { exact: "First passage" }, + selectionPostId: "100", + }; + const latest = { + ...first, + canonicalUrl: "https://x.com/i/status/101", + selection: { exact: "Latest passage" }, + selectionPostId: "101", + }; + let finish!: (value: typeof first) => void; + mocks.captureTab + .mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ) + .mockResolvedValueOnce(latest); + await act(async () => + chromeFake.emit("message", { type: "mdbase-reader/selection" }, { tab: { id: 1 } }), + ); + await act(async () => + chromeFake.emit("message", { type: "mdbase-reader/selection" }, { tab: { id: 1 } }), + ); + expect(controller.capture?.canonicalUrl).toBe(latest.canonicalUrl); + await act(async () => finish(first)); + expect(controller.capture?.canonicalUrl).toBe(latest.canonicalUrl); + expect(controller.capture?.selection?.exact).toBe("Latest passage"); +}); + it("does not auto-save on opening or changing the destination, and remembers the choice", async () => { await mount(); expect(controller.capture?.selection?.exact).toBe("quotation"); diff --git a/apps/extension/src/use-extension-capture.ts b/apps/extension/src/use-extension-capture.ts index f4d8c701..b450695b 100644 --- a/apps/extension/src/use-extension-capture.ts +++ b/apps/extension/src/use-extension-capture.ts @@ -6,8 +6,16 @@ import { connectionUnavailableMessage } from "./connection-status.js"; import { rememberColor } from "./drafts.js"; import { annotationQuotes } from "./page-annotations.js"; import { markSavedPage } from "./page-badge.js"; -import { fetchPdf, renderAnnotations, revealAnnotation, type PageCapture } from "./page-capture.js"; +import { + captureSocialTab, + fetchPdf, + renderAnnotations, + revealAnnotation, + type PageCapture, + type SelectedWebCapture, +} from "./page-capture.js"; import { CaptureWriter, type CaptureDraft } from "./save-capture.js"; +import { scopePostQuotes } from "./social-quotes.js"; import { rememberTags, splitTags } from "./tag-suggestions.js"; import { useActionLock } from "./use-action-lock.js"; import { useKnownTags } from "./use-known-tags.js"; @@ -30,6 +38,7 @@ import type { AnnotationId, SourceImportProgress, SourceSummary, + SocialPost, } from "@mdbase-reader/core"; export type { ExtensionCaptureController } from "./capture-controller.js"; @@ -52,6 +61,8 @@ export function useExtensionCapture( const { connection, lock: connectionLock } = link; const busy = lock.busy || connectionLock.busy; const { capture } = page; + const [pickerOpen, setPickerOpen] = useState(false); + const capturedPost = socialCapture(capture)?.post; const stored = useStoredDraft(tabId, capture, page.setSelection); const [status, setStatus] = useState("opening"); const [source, setSource] = useState(null); @@ -107,10 +118,11 @@ export function useExtensionCapture( capture, ); + const picking = isPostPickerPage(capture); const display = useCallback( (values: readonly Annotation[], url: string, focus?: AnnotationId) => - showSavedHighlights(tabId, values, url, focus), - [tabId], + showSavedHighlights(tabId, values, url, focus, capturedPost), + [tabId, capturedPost], ); useEffect(() => { @@ -121,7 +133,13 @@ export function useExtensionCapture( return; } void (async () => { - const existing = await sourceForUrl(collection, pageUrl, submittedUrl ? [submittedUrl] : []); + const existing = picking + ? null + : await sourceForUrl( + collection, + pageUrl, + !capturedPost && submittedUrl ? [submittedUrl] : [], + ); if (epoch !== discovery.current) { return; } @@ -166,7 +184,18 @@ export function useExtensionCapture( // What the lookup said was about the page or collection being left. setNotice(null); }; - }, [collectionId, display, extension, pageKind, pageUrl, setNotice, submittedUrl, tabId]); + }, [ + collectionId, + display, + extension, + pageKind, + pageUrl, + setNotice, + submittedUrl, + tabId, + capturedPost, + picking, + ]); const { setDraft } = stored; useEffect(() => { @@ -198,8 +227,12 @@ export function useExtensionCapture( const citation = citationFor && citationFor.page === pageUrl ? citationFor.preview : null; const citationPending = Boolean(pageUrl) && citationFor?.page !== pageUrl; - const save = (changes?: Partial): Promise => - lock.run(async () => { + const save = (changes?: Partial): Promise => { + if (socialCapture(capture)?.social?.mode === "multiple" && !changes?.highlight) { + setPickerOpen(true); + return Promise.resolve(); + } + return lock.run(async () => { const collection = extension?.session.connectedCollection(); if (!collection || !writer || !extension) { throw new Error( @@ -260,6 +293,56 @@ export function useExtensionCapture( setStatus((current) => (current === "saving" ? "ready" : current)); } }, "save"); + }; + + const savePosts = (ids: readonly string[]): Promise => + lock.run(async () => { + const collection = extension?.session.connectedCollection(); + if (!collection || !writer || !extension) { + throw new Error(connectionUnavailableMessage(connection.snapshot)); + } + setStatus("saving"); + let count = 0; + try { + for (const id of ids) { + const post = await captureSocialTab(tabId, id); + if (!post?.post) { + throw new Error("The chosen post is not loaded anymore."); + } + const result = await writer.save({ + session: extension.session, + collection, + capture: { ...post, selection: null }, + draft: { + ...stored.draft, + title: post.pageTitle, + highlight: false, + comment: "", + note: "", + }, + citation: await prepareCitation(post), + onProgress: setProgress, + onSource: () => undefined, + }); + count++; + if (result.notices.length) { + setNotice(result.notices.join(" ")); + } + } + setPickerOpen(false); + setNotice(`${String(count)} post${count === 1 ? "" : "s"} saved. Feeds are never saved.`); + await page.readPage(); + void chrome.runtime + .sendMessage({ type: "mdbase-reader/posts-refresh", tabId }) + .catch(() => undefined); + } catch (reason) { + throw new Error( + `${String(count)} posts confirmed before the failure. Retry checks each post for duplicates. ${problemMessage(reason)}`, + ); + } finally { + setStatus("ready"); + } + }, "save"); /** * The list is known when it was loaded for this source, or the source is new; then a new @@ -294,7 +377,7 @@ export function useExtensionCapture( } setAnnotations(values); if (render && capture.kind === "html") { - const shown = await display(values, capture.submittedUrl); + const shown = await display(values, capture.pageUrl ?? capture.submittedUrl); if (epoch === refreshEpoch.current) { setProjection(shown); } @@ -336,9 +419,9 @@ export function useExtensionCapture( } const redrawn = await revealAnnotation( tabId, - annotationQuotes(quoted), + scopePostQuotes(annotationQuotes(quoted), capturedPost), index, - capture.submittedUrl, + capture.pageUrl ?? capture.submittedUrl, ); if (redrawn) { setProjection({ @@ -354,7 +437,7 @@ export function useExtensionCapture( const afterEdit = (next: readonly Annotation[]): void => { setAnnotations(next); if (capture?.kind === "html") { - const url = capture.submittedUrl; + const url = capture.pageUrl ?? capture.submittedUrl; void display(next, url) .then(setProjection) .catch(() => undefined); @@ -459,6 +542,7 @@ export function useExtensionCapture( ...actions, snapshot: connection.snapshot, capture, + ...pickerProps(capture, pickerOpen, setPickerOpen, savePosts), draft: stored.draft, setDraft: stored.setDraft, draftRestored: stored.restored, @@ -466,13 +550,11 @@ export function useExtensionCapture( problem: lock.problem ?? connectionLock.problem, problemKind: lock.problem ? lock.problemKind : connectionLock.problemKind, notice: lock.notice ?? connectionLock.notice, - source, - annotations, + ...visibleSourceState(pageUrl, source, annotations, projection), citation, citationPending, deviceCode: connection.deviceCode, directAccess: connection.directAccess, - projection, progress, refreshing, busy, @@ -497,13 +579,20 @@ async function showSavedHighlights( values: readonly Annotation[], url: string, focus?: AnnotationId, + post?: SocialPost, ): Promise { const quoted = values.filter((annotation) => annotation.target?.quote); - void markSavedPage(tabId, quoted.length).catch(() => undefined); + if (post) { + void chrome.runtime + .sendMessage({ type: "mdbase-reader/posts-refresh", tabId }) + .catch(() => undefined); + } else { + void markSavedPage(tabId, quoted.length).catch(() => undefined); + } const index = quoted.findIndex((annotation) => annotation.id === focus); const { report, outcomes } = await renderAnnotations( tabId, - annotationQuotes(quoted), + scopePostQuotes(annotationQuotes(quoted), post), url, index === -1 ? undefined : index, ); @@ -513,6 +602,52 @@ async function showSavedHighlights( }; } +function isPostPickerPage(capture: PageCapture | null): boolean { + const social = socialCapture(capture); + return social?.social?.mode === "multiple" && !social.post; +} + +function socialCapture(capture: PageCapture | null): SelectedWebCapture | null { + return capture?.kind === "html" ? capture : null; +} + +function capturableUrl(capture: PageCapture | null): string | undefined { + const html = socialCapture(capture); + return html?.social && !html.post ? undefined : capture?.canonicalUrl; +} + +function pickerProps( + capture: PageCapture | null, + open: boolean, + setOpen: (value: boolean) => void, + save: (ids: readonly string[]) => Promise, +): Pick { + const listing = socialCapture(capture)?.social; + return listing?.mode === "multiple" + ? { + postPicker: { + open, + items: listing.items, + preselected: listing.nearest, + show: () => setOpen(true), + close: () => setOpen(false), + save, + }, + } + : {}; +} + +function visibleSourceState( + url: string | undefined, + source: SourceSummary | null, + annotations: readonly Annotation[], + projection: PageHighlights | null, +): Pick { + return url + ? { source, annotations, projection } + : { source: null, annotations: [], projection: null }; +} + /** What identifies the panel's current page and collection, for discarding stale work. */ function pageKeys( snapshot: ExtensionCaptureController["snapshot"], @@ -528,11 +663,12 @@ function pageKeys( readonly attemptKey: string; } { const collectionId = snapshot.status === "ready" ? snapshot.collectionId : null; - const pageUrl = capture?.canonicalUrl; + const pageUrl = capturableUrl(capture); return { collectionId, pageUrl, - submittedUrl: capture?.submittedUrl, + submittedUrl: + capture?.kind === "html" ? (capture.pageUrl ?? capture.submittedUrl) : capture?.submittedUrl, pageKind: capture?.kind, discoveryKey: collectionId && pageUrl ? `${collectionId} ${pageUrl}` : null, attemptKey: `${collectionId ?? ""} ${pageUrl ?? ""}`, diff --git a/apps/extension/src/use-loaded-posts.ts b/apps/extension/src/use-loaded-posts.ts new file mode 100644 index 00000000..696095af --- /dev/null +++ b/apps/extension/src/use-loaded-posts.ts @@ -0,0 +1,41 @@ +import { useEffect, type Dispatch, type SetStateAction } from "react"; + +import type { PageCapture } from "./page-capture.js"; + +/** Keep the picker current as virtualised feeds load/replace posts, without re-extracting. */ +export function useLoadedPosts( + tabId: number, + setCapture: Dispatch>, +): void { + useEffect(() => { + const changed = (message: unknown, sender: chrome.runtime.MessageSender): void => { + if ( + !message || + typeof message !== "object" || + !("type" in message) || + message.type !== "mdbase-reader/posts-changed" || + sender.tab?.id !== tabId + ) { + return; + } + void chrome.scripting + .executeScript({ + target: { tabId }, + func: async () => globalThis.mdbaseReaderSocialPage?.refreshItems() ?? null, + }) + .then(([result]) => { + const listing = result?.result; + if (listing) { + setCapture((current) => + current?.kind === "html" && current.social + ? { ...current, social: listing } + : current, + ); + } + }) + .catch(() => undefined); + }; + chrome.runtime.onMessage.addListener(changed); + return () => chrome.runtime.onMessage.removeListener(changed); + }, [tabId, setCapture]); +} diff --git a/apps/extension/src/use-page-capture.ts b/apps/extension/src/use-page-capture.ts index 9e02c4ad..1e6b2f9a 100644 --- a/apps/extension/src/use-page-capture.ts +++ b/apps/extension/src/use-page-capture.ts @@ -1,6 +1,6 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { intentKey, isExtensionMessage, type CaptureIntent } from "./messages.js"; +import { intentKey, type CaptureIntent } from "./messages.js"; import { captureTab, readSelection, @@ -9,6 +9,8 @@ import { type PageCapture, } from "./page-capture.js"; import { pageStatusEnabled, pageStatusOrigins } from "./page-status.js"; +import { useLoadedPosts } from "./use-loaded-posts.js"; +import { usePageEvents } from "./use-page-events.js"; import type { QuoteSelector } from "@mdbase-reader/core"; @@ -28,7 +30,7 @@ export interface PageLink { * from a click: Chrome asks only during a user gesture. */ readonly allowAllSites: () => Promise; - readonly setSelection: (selection: QuoteSelector | null) => void; + readonly setSelection: (selection: QuoteSelector | null, postId?: string) => void; } /** @@ -39,36 +41,37 @@ export interface PageLink { */ export function usePageCapture(tabId: number, onProblem: (message: string) => void): PageLink { const [capture, setCapture] = useState(null); + useLoadedPosts(tabId, setCapture); const [navigated, setNavigated] = useState(false); const [following, setFollowing] = useState(false); - // The page reports selections only while a panel shows its tab. - useEffect(() => () => unwatchTabSelection(tabId), [tabId]); - const [siteAccess, setSiteAccess] = useState(null); - useEffect(() => { - const check = (): void => { - pageStatusEnabled() - .then(setSiteAccess) - .catch(() => setSiteAccess(false)); - }; - check(); - chrome.permissions.onAdded.addListener(check); - chrome.permissions.onRemoved.addListener(check); - return () => { - chrome.permissions.onAdded.removeListener(check); - chrome.permissions.onRemoved.removeListener(check); - }; - }, []); + const readingEpoch = useRef(0); + // Invalidate pending public-API captures when leaving the tab/panel. + useEffect( + () => () => { + ++readingEpoch.current; + unwatchTabSelection(tabId); + }, + [tabId], + ); const [invocation, setInvocation] = useState(null); const problem = useRef(onProblem); useEffect(() => { problem.current = onProblem; }, [onProblem]); - const setSelection = useCallback((selection: QuoteSelector | null) => { - setCapture((current) => (current?.kind === "html" ? { ...current, selection } : current)); + const setSelection = useCallback((selection: QuoteSelector | null, postId?: string) => { + setCapture((current) => + current?.kind === "html" + ? { ...current, selection, ...(postId ? { selectionPostId: postId } : {}) } + : current, + ); }, []); const readPage = useCallback(async (): Promise => { + const epoch = ++readingEpoch.current; const value = await captureTab(tabId); + if (epoch !== readingEpoch.current) { + return value; + } setCapture(value); setNavigated(false); setFollowing(false); @@ -77,58 +80,85 @@ export function usePageCapture(tabId: number, onProblem: (message: string) => vo } return value; }, [tabId]); + const { siteAccess, allowAllSites } = useSiteAccess(readPage); const followSelection = useCallback(async () => { + const epoch = ++readingEpoch.current; + if (capture?.kind === "html" && capture.social) { + const next = await captureTab(tabId); + if (epoch === readingEpoch.current) { + setCapture(next); + } + return; + } const selection = await readSelection(tabId); - if (selection) { + if (selection && epoch === readingEpoch.current) { setSelection(selection); } - }, [setSelection, tabId]); + }, [capture, setSelection, tabId]); + + usePageEvents({ + tabId, + capture, + navigated, + following, + readPage, + followSelection, + setNavigated, + setFollowing, + setInvocation, + readingEpoch, + problem, + }); + + useInitialIntent(tabId, setInvocation); + + return { + capture, + navigated, + following, + invocation, + readPage, + siteAccess, + allowAllSites, + setSelection, + }; +} +function useSiteAccess( + readPage: PageLink["readPage"], +): Pick { + const [siteAccess, setSiteAccess] = useState(null); useEffect(() => { - const onMessage = (message: unknown, sender: chrome.runtime.MessageSender): void => { - if (!isExtensionMessage(message)) { - return; - } - if (message.type === "mdbase-reader/selection" && sender.tab?.id === tabId) { - followSelection().catch(() => undefined); - } - if (message.type === "mdbase-reader/invoke" && message.tabId === tabId) { - const { intent } = message; - void chrome.storage.session.remove(intentKey(tabId)).catch(() => undefined); - // A new invocation grants activeTab, so a page not read yet can be read now. - // The intent counts once the selection it refers to has been read. - (navigated || !capture ? readPage() : followSelection()) - .then(() => setInvocation({ intent, at: Date.now() })) - .catch((reason: unknown) => - problem.current(reason instanceof Error ? reason.message : String(reason)), - ); - } - }; - const onUpdated = (updatedTab: number, change: { readonly status?: string }): void => { - if (updatedTab !== tabId) { - return; - } - if (change.status === "loading") { - setNavigated(true); - setFollowing(true); - } - if (change.status === "complete" && navigated && following) { - // A page Reader cannot read leaves the panel waiting for the toolbar button. - readPage().catch(() => setFollowing(false)); - } + const check = (): void => { + void pageStatusEnabled() + .then(setSiteAccess) + .catch(() => setSiteAccess(false)); }; - chrome.runtime.onMessage.addListener(onMessage); - chrome.tabs.onUpdated.addListener(onUpdated); + check(); + chrome.permissions.onAdded.addListener(check); + chrome.permissions.onRemoved.addListener(check); return () => { - chrome.runtime.onMessage.removeListener(onMessage); - chrome.tabs.onUpdated.removeListener(onUpdated); + chrome.permissions.onAdded.removeListener(check); + chrome.permissions.onRemoved.removeListener(check); }; - }, [capture, followSelection, following, navigated, readPage, tabId]); + }, []); + const allowAllSites = useCallback(async (): Promise => { + const granted = await chrome.permissions.request({ origins: pageStatusOrigins }); + if (granted) { + await readPage(); + } + return granted; + }, [readPage]); + return { siteAccess, allowAllSites }; +} +function useInitialIntent( + tabId: number, + setInvocation: (value: PageLink["invocation"]) => void, +): void { useEffect(() => { - // The intent that opened this panel (e.g. "Highlight with a comment" from the context menu). const key = intentKey(tabId); - chrome.storage.session + void chrome.storage.session .get(key) .then(async ({ [key]: intent }) => { if (intent === "highlight" || intent === "note") { @@ -137,25 +167,5 @@ export function usePageCapture(tabId: number, onProblem: (message: string) => vo await chrome.storage.session.remove(key); }) .catch(() => undefined); - }, [tabId]); - - const allowAllSites = useCallback(async (): Promise => { - // The request comes first, while the click still counts as a user gesture. - const granted = await chrome.permissions.request({ origins: pageStatusOrigins }); - if (granted) { - await readPage(); - } - return granted; - }, [readPage]); - - return { - capture, - navigated, - following, - invocation, - readPage, - siteAccess, - allowAllSites, - setSelection, - }; + }, [setInvocation, tabId]); } diff --git a/apps/extension/src/use-page-events.ts b/apps/extension/src/use-page-events.ts new file mode 100644 index 00000000..c79f44dc --- /dev/null +++ b/apps/extension/src/use-page-events.ts @@ -0,0 +1,112 @@ +import { useEffect, type RefObject } from "react"; + +import { intentKey, isExtensionMessage, type CaptureIntent } from "./messages.js"; + +import type { PageCapture } from "./page-capture.js"; + +interface PageEvents { + readonly tabId: number; + readonly capture: PageCapture | null; + readonly navigated: boolean; + readonly following: boolean; + readonly readPage: () => Promise; + readonly followSelection: () => Promise; + readonly setNavigated: (value: boolean) => void; + readonly setFollowing: (value: boolean) => void; + readonly setInvocation: (value: { intent: CaptureIntent; at: number }) => void; + readonly readingEpoch: RefObject; + readonly problem: RefObject<(message: string) => void>; +} + +/** Follows full and same-document navigation; stale captures cannot trigger a quick save. */ +export function usePageEvents({ + tabId, + capture, + navigated, + following, + readPage, + followSelection, + setNavigated, + setFollowing, + setInvocation, + readingEpoch, + problem, +}: PageEvents): void { + useEffect(() => { + const onMessage = (message: unknown, sender: chrome.runtime.MessageSender): void => { + if (!isExtensionMessage(message)) { + return; + } + if ( + message.type === "mdbase-reader/posts-changed" && + sender.tab?.id === tabId && + navigated && + !following + ) { + setFollowing(true); + void readPage().catch(() => setFollowing(false)); + } + if (message.type === "mdbase-reader/selection" && sender.tab?.id === tabId) { + void followSelection().catch(() => undefined); + } + if (message.type === "mdbase-reader/invoke" && message.tabId === tabId) { + const { intent } = message; + void chrome.storage.session.remove(intentKey(tabId)).catch(() => undefined); + const reading = navigated || !capture ? readPage() : followSelection(); + const epoch = readingEpoch.current; + void reading + .then(() => { + if (epoch === readingEpoch.current) { + setInvocation({ intent, at: Date.now() }); + } + }) + .catch((reason: unknown) => + problem.current(reason instanceof Error ? reason.message : String(reason)), + ); + } + }; + const onUpdated = ( + updatedTab: number, + change: { readonly status?: string; readonly url?: string }, + ): void => { + if (updatedTab !== tabId) { + return; + } + const capturedUrl = + capture?.kind === "html" + ? (capture.pageUrl ?? capture.submittedUrl) + : capture?.submittedUrl; + if (change.url && change.status !== "loading" && change.url !== capturedUrl) { + setNavigated(true); + setFollowing(true); + void readPage().catch(() => setFollowing(false)); + } + if (change.status === "loading") { + ++readingEpoch.current; + setNavigated(true); + setFollowing(true); + } + if (change.status === "complete" && navigated && following) { + void readPage().catch(() => setFollowing(false)); + } + }; + chrome.runtime.onMessage.addListener(onMessage); + chrome.tabs.onUpdated.addListener(onUpdated); + return () => { + chrome.runtime.onMessage.removeListener(onMessage); + chrome.tabs.onUpdated.removeListener(onUpdated); + }; + }, [ + capture, + followSelection, + following, + navigated, + readPage, + tabId, + problem, + readingEpoch, + setFollowing, + setInvocation, + setNavigated, + ]); +} diff --git a/apps/extension/src/use-stored-draft.ts b/apps/extension/src/use-stored-draft.ts index e1f7d7fb..7eb3d6fa 100644 --- a/apps/extension/src/use-stored-draft.ts +++ b/apps/extension/src/use-stored-draft.ts @@ -17,13 +17,14 @@ export interface StoredDraftState { export function useStoredDraft( tabId: number, capture: PageCapture | null, - setSelection: (selection: QuoteSelector | null) => void, + setSelection: (selection: QuoteSelector | null, postId?: string) => void, ): StoredDraftState { const [draft, setDraft] = useState(emptyDraft); const [loadedFor, setLoadedFor] = useState(null); const [restored, setRestored] = useState(false); const page = capture?.canonicalUrl ?? null; const selection = capture?.kind === "html" ? capture.selection : null; + const selectionPostId = capture?.kind === "html" ? capture.selectionPostId : undefined; // A new passage is what the reader wants to highlight; a cleared one is done. const [seenSelection, setSeenSelection] = useState(selection); @@ -46,7 +47,7 @@ export function useStoredDraft( if (stored) { setDraft({ ...stored.draft, title: stored.draft.title || capture.pageTitle }); if (!selection && stored.selection) { - setSelection(stored.selection); + setSelection(stored.selection, stored.selectionPostId); } setRestored(Boolean(stored.draft.note.trim() || stored.draft.comment.trim())); } else { @@ -69,9 +70,17 @@ export function useStoredDraft( if (!page || loadedFor !== page) { return; } - const timer = setTimeout(() => void saveDraft(tabId, page, { draft, selection }), 250); + const timer = setTimeout( + () => + void saveDraft(tabId, page, { + draft, + selection, + ...(selectionPostId ? { selectionPostId } : {}), + }), + 250, + ); return () => clearTimeout(timer); - }, [draft, loadedFor, page, selection, tabId]); + }, [draft, loadedFor, page, selection, selectionPostId, tabId]); return { draft, setDraft, restored }; } diff --git a/apps/extension/vite.config.ts b/apps/extension/vite.config.ts index 12b274b5..06572deb 100644 --- a/apps/extension/vite.config.ts +++ b/apps/extension/vite.config.ts @@ -30,6 +30,7 @@ function serviceWorker(): Plugin { const scripts = { background: "src/background.ts", "page-status": "src/page-status-worker.ts", + "social-page": "src/social-page.ts", }; return { name: "mdbase-reader-extension-service-worker", diff --git a/apps/reader/src/connect-workspace.test.ts b/apps/reader/src/connect-workspace.test.ts index 6efec719..dd65d105 100644 --- a/apps/reader/src/connect-workspace.test.ts +++ b/apps/reader/src/connect-workspace.test.ts @@ -44,6 +44,43 @@ const annotation: Annotation = { }; describe("ConnectWorkspaceGateway", () => { + it("reuses a saved post by canonical identity even when its capture bytes or title change", async () => { + const findByUrl = vi.fn().mockResolvedValue(source); + const imports = emptyImports(); + const gateway = new ConnectWorkspaceGateway( + { findByUrl, get: vi.fn().mockResolvedValue(source) } as unknown as SourceRepository, + {} as AnnotationRepository, + { store: vi.fn() }, + imports, + source.collectionId, + "[test] Posts", + createReaderRuntimeServices(new MemoryStorage()), + ); + const part = { + platform: "bluesky" as const, + id: "at://did:plc:fixture/app.bsky.feed.post/first", + url: "https://bsky.app/profile/did:plc:fixture/post/first", + author: { id: "did:plc:fixture", handle: "fixture.test", name: "Fixture Author" }, + text: "Durable post", + media: [], + }; + const request = { + name: "post.html", + bytes: new Uint8Array(), + title: "New title", + metadata: { + socialPost: { ...part, threadRoot: part.id, parts: [part], threadPartial: false }, + }, + }; + expect(await gateway.importSourceFile(request)).toBe(source); + expect(await gateway.importSourceFile({ ...request, bytes: new Uint8Array([1, 2, 3]) })).toBe( + source, + ); + expect(findByUrl).toHaveBeenCalledWith(source.collectionId, part.url, {}); + expect(imports.commitFile).not.toHaveBeenCalled(); + expect(source.body).toBe("Original"); + }); + it("loads summaries before source bodies and saves notes revision-safely", async () => { const controller = new AbortController(); const updateBody = vi.fn().mockResolvedValue({ ...source, body: "Updated" }); diff --git a/apps/reader/src/connect-workspace.ts b/apps/reader/src/connect-workspace.ts index d01b3863..6d4736ca 100644 --- a/apps/reader/src/connect-workspace.ts +++ b/apps/reader/src/connect-workspace.ts @@ -5,6 +5,8 @@ import { citekeyForCitation, createSourceRecord, importSourceFile, + normalizedSourceUrl, + sameSourceUrl, planAnnotationDeletion, saveSourceCitation, transcludeAnnotation, @@ -457,6 +459,20 @@ export class ConnectWorkspaceGateway implements ReaderWorkspaceGateway { request: Omit, options: SourceImportOptions = {}, ): Promise { + const postUrl = request.metadata?.socialPost?.url; + if (postUrl) { + const found = this.sources.findByUrl + ? await this.sources.findByUrl(this.collectionId, postUrl, options) + : (await this.library(options.signal ? { signal: options.signal } : {})).sources.find( + (source) => source.url && sameSourceUrl(source.url, normalizedSourceUrl(postUrl)), + ); + if (found) { + const existing = await this.source(found.id, options); + if (existing) { + return existing; + } + } + } const imported = await importSourceFile( { imports: this.sourceImports, diff --git a/apps/reader/src/source-addition-flow.test.ts b/apps/reader/src/source-addition-flow.test.ts index d969e3c0..ddd8762a 100644 --- a/apps/reader/src/source-addition-flow.test.ts +++ b/apps/reader/src/source-addition-flow.test.ts @@ -160,6 +160,50 @@ describe("adding a source from pasted text", () => { }); }); +describe("adding social sources", () => { + it.each([ + "https://x.com/home", + "https://x.com/fixture", + "https://bsky.app/search", + "https://bsky.app/profile/fixture.test", + ])("rejects feed/profile URLs without offering citation-only capture: %s", async (url) => { + const deps = services(); + await expect(addSourceFromInput(url, deps)).rejects.toThrow("cannot be saved"); + expect(deps.capture).not.toHaveBeenCalled(); + expect(deps.lookUp).not.toHaveBeenCalled(); + }); + it("explains X's live-page requirement instead of requesting undocumented APIs", async () => { + const deps = services(); + await expect( + addSourceFromInput("https://twitter.com/fixture/status/100", deps), + ).rejects.toThrow("extension"); + expect(deps.capture).not.toHaveBeenCalled(); + }); + it("does not offer a citation-only Mastodon profile after marker-based feed rejection", async () => { + const deps = services({ + capture: vi.fn(() => + Promise.reject( + new Error( + "Feeds cannot be saved as sources. Choose individual posts in the Reader extension.", + ), + ), + ), + }); + await expect(addSourceFromInput("https://any-instance.test/@fixture", deps)).rejects.toThrow( + "Feeds cannot", + ); + expect(deps.lookUp).not.toHaveBeenCalled(); + expect(deps.workspace.createSource).not.toHaveBeenCalled(); + }); + it("does not offer a citation-only post after an API failure", async () => { + const deps = services(); + await expect( + addSourceFromInput("https://bsky.app/profile/fixture.test/post/first", deps), + ).rejects.toThrow("blocked"); + expect(deps.lookUp).not.toHaveBeenCalled(); + }); +}); + function source(): Source { return { collectionId: collectionId("reading"), diff --git a/apps/reader/src/source-addition-flow.ts b/apps/reader/src/source-addition-flow.ts index c5099926..2d82d0cf 100644 --- a/apps/reader/src/source-addition-flow.ts +++ b/apps/reader/src/source-addition-flow.ts @@ -5,6 +5,8 @@ import { citationYear, identifierLookup, parseSourceInput, + postAddress, + siteAdapters, type CitationDraft, type SourceIdentifier, } from "@mdbase-reader/web-capture"; @@ -67,12 +69,16 @@ async function addFromUrl( url: string, services: SourceAdditionServices, ): Promise { + const socialAddress = assertSocialInput(url); services.onStatus("Fetching the page…"); let captured: CaptureResult; try { captured = await services.capture(url, { allowPdf: true, ...signalOf(services) }); } catch (reason) { services.signal?.throwIfAborted(); + if (socialAddress || messageOf(reason).includes("Feeds cannot be saved as sources")) { + throw reason; + } services.onStatus("Looking up citation details instead…"); const candidate = await services.lookUp({ kind: "url", value: url }).catch(() => null); if (candidate) { @@ -102,13 +108,33 @@ async function addFromUrl( capture: page.capture, archive: page.archive, metadata: { ...fields?.metadata, ...page.metadata, ...authorsOf(fields) }, - ...(fields?.kind && fields.kind !== "webpage" ? { kind: fields.kind } : {}), + ...capturedKind(page.metadata, fields?.kind), }, services.importOptions, ); return finish(source, preview?.citation, services); } +function assertSocialInput(url: string): ReturnType { + const address = postAddress(url); + if (siteAdapters.some((adapter) => adapter.matches(url)) && !address) { + throw new Error( + "Feeds, profiles and search pages cannot be saved. Choose individual posts in the Reader extension.", + ); + } + if (address?.platform === "x") { + throw new Error("Save X posts from their live page using the mdbase Reader extension."); + } + return address; +} + +function capturedKind(metadata: SourceImportMetadata, kind?: string): { kind?: string } { + if (metadata.socialPost) { + return { kind: "post" }; + } + return kind && kind !== "webpage" ? { kind } : {}; +} + async function addFromIdentifier( identifier: Exclude, services: SourceAdditionServices, @@ -191,7 +217,7 @@ async function finish( return { kind: "not-added" }; } const save = services.workspace.saveNewSourceCitation; - if (!citation || !save) { + if (!citation || !save || source.citation) { return { kind: "added", source, notices: [] }; } // A candidate's placeholder citekey is replaced with one unique in the collection. diff --git a/apps/reader/src/source-addition-services.test.ts b/apps/reader/src/source-addition-services.test.ts new file mode 100644 index 00000000..1a6b1219 --- /dev/null +++ b/apps/reader/src/source-addition-services.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment happy-dom +import { afterEach, expect, it, vi } from "vitest"; + +import { readerSourceAdditionServices } from "./source-addition-services.js"; + +import type { CaptureResult } from "./web-capture-client.js"; +import type { Source } from "@mdbase-reader/core"; +import type * as WebCapture from "@mdbase-reader/web-capture"; + +const mocks = vi.hoisted(() => ({ postFromUrl: vi.fn(), fetchCapture: vi.fn() })); +vi.mock("@mdbase-reader/web-capture", async (original) => ({ + ...(await original()), + postFromUrl: mocks.postFromUrl, +})); +vi.mock("./web-capture-client.js", () => ({ fetchCapture: mocks.fetchCapture })); +const workspace = { + importSourceFile: () => Promise.resolve(null), + createSource: () => Promise.resolve(null), + saveNewSourceCitation: (source: Source) => Promise.resolve(source), +}; +afterEach(() => vi.clearAllMocks()); + +it("falls through to marker-aware HTML capture when a post-shaped path is not a Mastodon API", async () => { + const result = { kind: "html" } as CaptureResult; + mocks.postFromUrl.mockRejectedValue(new Error("Not Mastodon")); + mocks.fetchCapture.mockResolvedValue(result); + const services = readerSourceAdditionServices({ workspace, onStatus: vi.fn() }); + const url = "https://ordinary-site.test/@fixture/123"; + expect(await services.capture(url, { allowPdf: true })).toBe(result); + expect(mocks.fetchCapture).toHaveBeenCalledWith(url, { allowPdf: true }); +}); + +it("does not turn a known Bluesky API failure into an ordinary article capture", async () => { + mocks.postFromUrl.mockRejectedValue(new Error("AppView unavailable")); + const services = readerSourceAdditionServices({ workspace, onStatus: vi.fn() }); + await expect( + services.capture("https://bsky.app/profile/did:plc:fixture/post/first", { allowPdf: true }), + ).rejects.toThrow("AppView unavailable"); + expect(mocks.fetchCapture).not.toHaveBeenCalled(); +}); + +it("preserves cancellation rather than starting an HTML fallback", async () => { + const controller = new AbortController(); + controller.abort(new Error("Cancelled")); + mocks.postFromUrl.mockRejectedValue(new Error("API request aborted")); + const services = readerSourceAdditionServices({ + workspace, + onStatus: vi.fn(), + signal: controller.signal, + }); + await expect( + services.capture("https://ordinary-site.test/@fixture/123", { + allowPdf: true, + signal: controller.signal, + }), + ).rejects.toThrow("Cancelled"); + expect(mocks.fetchCapture).not.toHaveBeenCalled(); +}); diff --git a/apps/reader/src/source-addition-services.ts b/apps/reader/src/source-addition-services.ts index 5cfe8a97..2ccf01f6 100644 --- a/apps/reader/src/source-addition-services.ts +++ b/apps/reader/src/source-addition-services.ts @@ -1,3 +1,4 @@ +import { postAddress, postFromUrl, socialPostImport } from "@mdbase-reader/web-capture"; import { lookUpCitation } from "@mdbase-reader/web-capture/citation-lookup"; import { readDocumentFileDetails } from "./document-details.js"; @@ -12,7 +13,33 @@ export function readerSourceAdditionServices( const signal = input.signal ? { signal: input.signal } : {}; return { ...input, - capture: (url, options) => fetchCapture(url, options), + capture: async (url, options) => { + const activeSignal = options.signal ?? input.signal; + const fetchOptions = activeSignal ? { signal: activeSignal } : {}; + const address = postAddress(url); + if (address) { + const document = new DOMParser().parseFromString("", "text/html"); + const post = await postFromUrl(url, document, fetchOptions).catch((reason: unknown) => { + activeSignal?.throwIfAborted(); + if (address.platform !== "mastodon") { + throw reason; + } + // A /@user/ID path alone is not proof of Mastodon. The HTML capture checks markers. + return null; + }); + if (post) { + return { + kind: "html", + page: await socialPostImport( + post, + { retrievedAt: new Date().toISOString() }, + fetchOptions, + ), + }; + } + } + return fetchCapture(url, options); + }, lookUp: (request) => lookUpCitation(request, { clientName: `mdbase-reader (${globalThis.location.origin})`, diff --git a/packages/connect/src/source-frontmatter.ts b/packages/connect/src/source-frontmatter.ts index 74380738..e9df00e1 100644 --- a/packages/connect/src/source-frontmatter.ts +++ b/packages/connect/src/source-frontmatter.ts @@ -2,7 +2,12 @@ import { ConnectRepositoryError } from "./repository-client.js"; import { documentEntry } from "./source-files.js"; import type { CollectionFileDescriptor } from "@mdbase-dev/connect"; -import type { PlannedSourceFileImport, PlannedSourceRepresentation } from "@mdbase-reader/core"; +import type { + PlannedSourceFileImport, + PlannedSourceRepresentation, + SocialPost, + SocialPostReference, +} from "@mdbase-reader/core"; type Descriptors = ReadonlyMap; @@ -37,6 +42,24 @@ function metadataFields( ...(metadata.description ? { description: metadata.description } : {}), ...(metadata.language ? { language: metadata.language } : {}), ...(metadata.site ? { site: metadata.site } : {}), + ...(metadata.socialPost ? { social_post: socialPostReference(metadata.socialPost) } : {}), + }; +} + +function socialPostReference(post: SocialPost): SocialPostReference { + return { + platform: post.platform, + id: post.id, + url: post.url, + author: post.author, + ...(post.published ? { published: post.published } : {}), + threadRoot: post.threadRoot, + threadPartial: post.threadPartial, + parts: post.parts.map((part) => ({ + id: part.id, + url: part.url, + ...(part.quoted ? { quoted: { id: part.quoted.id, url: part.quoted.url } } : {}), + })), }; } diff --git a/packages/connect/src/source-imports-web-capture.test.ts b/packages/connect/src/source-imports-web-capture.test.ts index da2eee03..604deb6e 100644 --- a/packages/connect/src/source-imports-web-capture.test.ts +++ b/packages/connect/src/source-imports-web-capture.test.ts @@ -8,6 +8,53 @@ import type { ReaderConnectClient } from "./repository-client.js"; import type { PlannedSourceFileImport } from "@mdbase-reader/core"; describe("ConnectSourceImportRepository web capture provenance", () => { + it("persists post identity and thread shape, leaving text and media to the body", async () => { + const create = vi.fn(() => Promise.resolve(success(recordDocument()))); + const repository = new ConnectSourceImportRepository( + { create } as unknown as ReaderConnectClient, + { + upload: vi.fn((path: string) => + Promise.resolve(fileDescriptor({ path, mediaType: "text/html" })), + ), + }, + ); + const part = { + platform: "x" as const, + id: "100", + url: "https://x.com/i/status/100", + author: { id: "fixture", handle: "fixture", name: "Fixture Author" }, + text: "Durable post", + media: [ + { kind: "image" as const, url: "https://media.example/image.png", alt: "A blue square" }, + ], + }; + const post = { ...part, threadRoot: part.id, parts: [part], threadPartial: true }; + const base = webCapturePlan(); + await repository.commitFile({ + ...base, + kind: "post", + metadata: { socialPost: post }, + capture: { ...base.capture!, submittedUrl: part.url, canonicalUrl: part.url }, + }); + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + frontmatter: expect.objectContaining({ + kind: "post", + url: part.url, + social_post: { + platform: "x", + id: "100", + url: part.url, + author: part.author, + threadRoot: "100", + threadPartial: true, + parts: [{ id: "100", url: part.url }], + }, + }), + }), + ); + }); + it("preserves raw and readable web representations with capture metadata", async () => { const create = vi.fn(() => Promise.resolve(success(recordDocument()))); const upload = vi.fn((path: string) => diff --git a/packages/core/src/application/source-import-ports.ts b/packages/core/src/application/source-import-ports.ts index 2620e065..d1c281a0 100644 --- a/packages/core/src/application/source-import-ports.ts +++ b/packages/core/src/application/source-import-ports.ts @@ -1,5 +1,6 @@ import type { ReaderRequestOptions } from "./ports.js"; import type { CollectionId, MutationId, SourceId } from "../domain/identity.js"; +import type { SocialPost } from "../domain/social-post.js"; import type { Source, SourceSummary } from "../domain/source.js"; import type { DateTime } from "../domain/time.js"; @@ -63,6 +64,7 @@ export interface SourceCaptureArchive { } export interface SourceImportMetadata { + readonly socialPost?: SocialPost; readonly authors?: readonly string[]; readonly published?: string; readonly description?: string; diff --git a/packages/core/src/application/source-import-values.ts b/packages/core/src/application/source-import-values.ts index f9c51fce..547dd747 100644 --- a/packages/core/src/application/source-import-values.ts +++ b/packages/core/src/application/source-import-values.ts @@ -38,6 +38,7 @@ export function validateMetadata(metadata: SourceImportMetadata): SourceImportMe ...(description ? { description } : {}), ...(language ? { language } : {}), ...(site ? { site } : {}), + ...(metadata.socialPost ? { socialPost: metadata.socialPost } : {}), }; } diff --git a/packages/core/src/domain/social-post.ts b/packages/core/src/domain/social-post.ts new file mode 100644 index 00000000..a9e74463 --- /dev/null +++ b/packages/core/src/domain/social-post.ts @@ -0,0 +1,45 @@ +export type SocialPlatform = "x" | "bluesky" | "mastodon"; + +export interface PostMedia { + readonly url: string; + readonly alt: string; + readonly kind: "image" | "video" | "audio"; +} + +/** Portable post data; HTML is a reading representation, not the source identity. */ +export interface PostPart { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly author: { readonly id: string; readonly handle: string; readonly name: string }; + readonly published?: string; + readonly text: string; + readonly media: readonly PostMedia[]; + readonly quoted?: PostPart; + readonly inReplyTo?: string; +} + +export interface SocialPost extends PostPart { + readonly threadRoot: string; + readonly parts: readonly PostPart[]; + /** APIs and live pages may omit deleted, private or not-yet-loaded thread parts. */ + readonly threadPartial: boolean; +} + +export interface PostPartReference { + readonly id: string; + readonly url: string; + readonly quoted?: { readonly id: string; readonly url: string }; +} + +/** What a source's frontmatter keeps: identity and thread shape. Text and media live in the body. */ +export interface SocialPostReference { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly author: PostPart["author"]; + readonly published?: string; + readonly threadRoot: string; + readonly threadPartial: boolean; + readonly parts: readonly PostPartReference[]; +} diff --git a/packages/core/src/domain/source-url.ts b/packages/core/src/domain/source-url.ts index ea4daa16..4850d6f3 100644 --- a/packages/core/src/domain/source-url.ts +++ b/packages/core/src/domain/source-url.ts @@ -9,6 +9,17 @@ const trackingParameter = export function normalizedSourceUrl(value: string): string { const url = new URL(value); url.hash = ""; + if (/^(?:(?:www|mobile)\.)?(?:x|twitter)\.com$/u.test(url.hostname)) { + const status = /^\/(?:[^/]+\/status|i\/web\/status|i\/status)\/(\d+)(?:\/|$)/u.exec( + url.pathname, + )?.[1]; + if (status) { + return `https://x.com/i/status/${status}`; + } + } + if (url.hostname === "bsky.app" && /^\/profile\/[^/]+\/post\/[^/]+\/?$/u.test(url.pathname)) { + url.search = ""; + } const cached = ampCacheOrigin(url); if (cached) { return normalizedSourceUrl(cached); @@ -43,6 +54,10 @@ export function sameSourceUrl(value: string, normalized: string): boolean { */ export function sourceUrlSearchKey(value: string): string { const url = new URL(normalizedSourceUrl(value)); + // Every X spelling contains /status/ID, even when the stored hostname/author is old. + if (url.hostname === "x.com" && /^\/i\/status\/\d+$/u.test(url.pathname)) { + return url.pathname.replace(/^\/i/u, ""); + } return `${url.hostname}${url.pathname === "/" ? "" : url.pathname}`.toLowerCase(); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index abd82936..e6f4ac8b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -26,6 +26,7 @@ export * from "./domain/revision.js"; export type * from "./domain/search.js"; export * from "./domain/selector.js"; export * from "./domain/source.js"; +export type * from "./domain/social-post.js"; export * from "./domain/source-url.js"; export * from "./domain/text-quote-match.js"; export * from "./domain/time.js"; diff --git a/packages/renderer-html/src/html-document.ts b/packages/renderer-html/src/html-document.ts index 9d3735ee..e83d247d 100644 --- a/packages/renderer-html/src/html-document.ts +++ b/packages/renderer-html/src/html-document.ts @@ -79,6 +79,12 @@ function readerStyle(document: Document): HTMLStyleElement { html { scroll-behavior: smooth; } body { max-width: 46rem; margin: 0 auto; padding: 5rem 3rem 9rem; font: 18px/1.72 Georgia, 'Times New Roman', serif; } main, article { display: block; } + .social-post > header h1 { font-size: 2rem; margin-bottom: .5rem; } + .social-post header, .social-post time, .social-post figcaption, .social-post footer { font: .85em/1.5 ui-sans-serif, system-ui, sans-serif; color: color-mix(in srgb, currentColor 72%, transparent); } + .post-part { margin-block: 2rem; padding-top: 1.5rem; border-top: 1px solid color-mix(in srgb, currentColor 18%, transparent); } + .post-part header p { margin-block: 0 .25rem; } + .social-post figure { margin-inline: 0; } + .quoted-post .post-part { border-top: 0; padding-top: 0; } h1, h2, h3, h4 { color: inherit; font-family: ui-sans-serif, system-ui, sans-serif; line-height: 1.15; letter-spacing: -0.025em; } h1 { margin: 0 0 2.4rem; font-size: clamp(2.15rem, 6vw, 3.7rem); } h2 { margin-top: 2.8rem; font-size: 1.65rem; } diff --git a/packages/renderer-html/src/html-range.test.ts b/packages/renderer-html/src/html-range.test.ts index 81f03611..b1f269ce 100644 --- a/packages/renderer-html/src/html-range.test.ts +++ b/packages/renderer-html/src/html-range.test.ts @@ -4,6 +4,24 @@ import { describe, expect, it } from "vitest"; import { htmlSelectionDraft, locateHtmlTarget } from "./html-range.js"; describe("HTML ranges", () => { + it("anchors identical post passages to their thread part, including quoted context", () => { + document.body.innerHTML = + '

Same passage

Quoted passage

Same passage

Quoted passage

'; + const second = document.querySelector('[data-post-id="101"] > p')!; + const range = document.createRange(); + range.selectNodeContents(second); + const draft = htmlSelectionDraft({ document, range, href: "post.html", progression: 0.5 }); + expect(draft?.target.html?.css).toBe('[data-post-id="101"]'); + expect(locateHtmlTarget(document, draft!.target)?.startContainer.parentElement).toBe(second); + range.selectNodeContents(document.querySelector('[data-post-id="101"] blockquote p')!); + const quoted = htmlSelectionDraft({ document, range, href: "post.html", progression: 0.5 }); + expect(quoted?.target.html?.css).toBe('[data-post-id="101"]'); + expect( + locateHtmlTarget(document, quoted!.target)?.startContainer.parentElement?.closest( + '[data-post-id="101"]', + ), + ).not.toBeNull(); + }); it("captures quotation context and a stable CSS selector", () => { document.body.innerHTML = `

Before selected passage after.

`; const text = document.querySelector("p")?.firstChild; diff --git a/packages/renderer-html/src/html-range.ts b/packages/renderer-html/src/html-range.ts index a6a35e68..b8a8012b 100644 --- a/packages/renderer-html/src/html-range.ts +++ b/packages/renderer-html/src/html-range.ts @@ -127,6 +127,10 @@ function cssSelector(element: Element, body: HTMLElement): string { if (element === body) { return "body"; } + const post = postSelector(element); + if (post) { + return post; + } if (element.id) { return `#${escapeIdentifier(element.id)}`; } @@ -147,6 +151,20 @@ function cssSelector(element: Element, body: HTMLElement): string { return segments[0]?.startsWith("#") ? segments.join(" > ") : `body > ${segments.join(" > ")}`; } +function postSelector(element: Element): string | null { + let part = element.closest("[data-post-id]"); + // Quoted context belongs to the containing post, even when the quote appears twice. + for ( + let parent = part?.parentElement?.closest("[data-post-id]"); + parent; + parent = parent.parentElement?.closest("[data-post-id]") + ) { + part = parent; + } + const id = part?.getAttribute("data-post-id"); + return id ? `[data-post-id=${JSON.stringify(id)}]` : null; +} + function escapeIdentifier(value: string): string { return value.replace( /[^a-zA-Z0-9_-]/gu, diff --git a/packages/web-capture/src/index.ts b/packages/web-capture/src/index.ts index 7a7bcf62..5b88788e 100644 --- a/packages/web-capture/src/index.ts +++ b/packages/web-capture/src/index.ts @@ -1,4 +1,6 @@ export * from "./web-capture.js"; +export * from "./social/index.js"; +export { postHtml, postTitle, postCitation, socialPostImport } from "./social/import.js"; export * from "./scholarly-metadata.js"; export * from "./doi.js"; export * from "./page-citation.js"; diff --git a/packages/web-capture/src/social/api.test.ts b/packages/web-capture/src/social/api.test.ts new file mode 100644 index 00000000..ac7d367a --- /dev/null +++ b/packages/web-capture/src/social/api.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from "vitest"; + +import { boundedSignal } from "./api.js"; + +it("keeps a request deadline even when the caller supplies a cancellation signal", async () => { + const caller = new AbortController(); + const signal = boundedSignal(caller.signal, 5); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(signal.aborted).toBe(true); + expect(caller.signal.aborted).toBe(false); +}); + +it("preserves caller cancellation without waiting for the request deadline", () => { + const caller = new AbortController(); + const signal = boundedSignal(caller.signal, 12_000); + const reason = new Error("Cancelled"); + caller.abort(reason); + expect(signal.aborted).toBe(true); + expect(signal.reason).toBe(reason); +}); diff --git a/packages/web-capture/src/social/api.ts b/packages/web-capture/src/social/api.ts new file mode 100644 index 00000000..77690c03 --- /dev/null +++ b/packages/web-capture/src/social/api.ts @@ -0,0 +1,181 @@ +import { postAddress, type PostAddress } from "./identity.js"; + +import type { PostMedia, PostPart } from "@mdbase-reader/core"; + +export interface SocialFetchOptions { + readonly fetch?: typeof fetch; + readonly signal?: AbortSignal; +} + +/** Public, credential-free, bounded requests only. No session or undocumented endpoints. */ +export async function publicJson(url: string, options: SocialFetchOptions): Promise { + const response = await (options.fetch ?? fetch)(url, { + credentials: "omit", + redirect: "error", + headers: { accept: "application/json" }, + signal: boundedSignal(options.signal, 12_000), + }); + if (!response.ok) { + throw new Error(`Post API returned HTTP ${String(response.status)}.`); + } + if (!response.body) { + throw new Error("Post API returned no content."); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + for (let next = await reader.read(); !next.done; next = await reader.read()) { + length += next.value.length; + if (length > 2 * 1024 * 1024) { + await reader.cancel(); + throw new Error("Post API response is too large."); + } + chunks.push(next.value); + } + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; +} + +export function boundedSignal(signal: AbortSignal | undefined, milliseconds: number): AbortSignal { + const deadline = AbortSignal.timeout(milliseconds); + return signal ? AbortSignal.any([signal, deadline]) : deadline; +} + +export async function blueskyIdentity( + address: PostAddress, + options: SocialFetchOptions, +): Promise { + const actor = address.id.split("/")[2] ?? ""; + if (actor.startsWith("did:")) { + return address; + } + const result = record( + await publicJson( + `https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(actor)}`, + options, + ), + ); + const did = text(result["did"]); + const key = address.id.split("/").at(-1); + const resolved = + did.startsWith("did:") && key ? postAddress(`at://${did}/app.bsky.feed.post/${key}`) : null; + if (!resolved) { + throw new Error("Bluesky could not resolve the author's identity."); + } + return resolved; +} + +export function blueskyPart(value: unknown, depth = 0): PostPart { + const post = record(value); + const address = postAddress(text(post["uri"])); + if (!address?.id.startsWith("at://did:")) { + throw new Error("Bluesky returned an invalid post identity."); + } + const author = record(post["author"]); + const content = record(post["record"]); + const embed = record(post["embed"]); + const images = array(embed["images"] ?? record(embed["media"])["images"]); + const video = text(embed["playlist"] ?? record(embed["media"])["playlist"]); + const media: PostMedia[] = images.flatMap((value) => { + const image = record(value); + const url = https(text(image["fullsize"])); + return url ? [{ kind: "image", url, alt: text(image["alt"]) }] : []; + }); + if (https(video)) { + media.push({ kind: "video", url: video, alt: text(embed["alt"]) }); + } + const quote = record(embed["record"]); + const quoted = quote["uri"] ? quote : record(quote["record"]); + const parent = text(record(record(content["reply"])["parent"])["uri"]); + return { + platform: "bluesky", + id: address.id, + url: address.url, + author: { + id: text(author["did"]), + handle: text(author["handle"]), + name: text(author["displayName"]) || text(author["handle"]), + }, + published: text(content["createdAt"]), + text: text(content["text"]), + media, + ...(depth === 0 && quoted["uri"] + ? { quoted: blueskyPart({ ...quoted, record: quoted["value"] ?? quoted["record"] }, 1) } + : {}), + ...(parent ? { inReplyTo: parent } : {}), + }; +} + +export function mastodonPart(value: unknown, document: Document, depth = 0): PostPart { + const post = record(value); + const uri = https(text(post["uri"])); + if (!uri) { + throw new Error("Mastodon returned an invalid status URI."); + } + const account = record(post["account"]); + const parsed = document.implementation.createHTMLDocument(""); + parsed.body.innerHTML = text(post["content"]); + parsed.querySelectorAll("script,style").forEach((element) => element.remove()); + parsed.querySelectorAll("br").forEach((element) => element.replaceWith("\n")); + parsed.querySelectorAll("p").forEach((element) => element.append("\n")); + const authorUrl = https(text(account["url"])); + const handle = text(account["acct"]); + const parent = text(post["in_reply_to_id"]); + const quote = record(post["quote"])["quoted_status"]; + return { + platform: "mastodon", + id: uri, + url: uri, + author: { + id: authorUrl || text(account["id"]), + handle: handle.includes("@") ? handle : `${handle}@${new URL(uri).hostname}`, + name: text(account["display_name"]) || handle, + }, + published: text(post["created_at"]), + text: [text(post["spoiler_text"]), parsed.body.textContent.trim()].filter(Boolean).join("\n\n"), + media: array(post["media_attachments"]).flatMap((value) => { + const media = record(value); + const url = https(text(media["url"])); + return url + ? [ + { + url, + alt: text(media["description"]), + kind: attachmentKind(text(media["type"])), + }, + ] + : []; + }), + ...(parent ? { inReplyTo: parent } : {}), + ...(quote && depth === 0 ? { quoted: mastodonPart(quote, document, 1) } : {}), + }; +} + +function attachmentKind(type: string): PostMedia["kind"] { + return type === "image" || type === "audio" ? type : "video"; +} + +export function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} +export function array(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} +export function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} +export function https(value: string): string { + try { + const url = new URL(value); + return url.protocol === "https:" && !url.username && !url.password ? url.href : ""; + } catch { + return ""; + } +} diff --git a/packages/web-capture/src/social/bluesky.ts b/packages/web-capture/src/social/bluesky.ts new file mode 100644 index 00000000..2b0b94c7 --- /dev/null +++ b/packages/web-capture/src/social/bluesky.ts @@ -0,0 +1,150 @@ +import { + array, + blueskyIdentity, + blueskyPart, + publicJson, + record, + type SocialFetchOptions, +} from "./api.js"; +import { refFor, type DomSelectors, type PostRef } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { domAdapter, domFallback } from "./registry.js"; +import { unrollThread } from "./thread.js"; + +import type { PostPart } from "@mdbase-reader/core"; + +export const blueskySelectors: DomSelectors = { + post: '[data-testid^="feedItem-by-"], [data-testid="postThreadItem"], [data-testid^="postThreadItem-by-"]', + content: '[data-testid="postText"], div[dir="auto"].r-1xnzce8', + permalink: + 'a[href*="/post/"]:not([data-testid="quotePost"] a):not([href$="/reposted-by"]):not([href$="/quotes"]):not([href$="/liked-by"])', + author: + 'a[href*="/profile/"]:not([href*="/post/"]):has([dir="auto"]), a[data-testid="profileDisplayName"]', + name: '[data-testid="profileDisplayName"], a[href*="/profile/"]:not([href*="/post/"]) [dir="auto"]', + quote: '[data-testid="quotePost"]', + media: '[data-testid="postImages"] img, img[alt]:not([data-testid="userAvatarImage"] img), video', +}; + +export const blueskyAdapter = domAdapter( + "bluesky", + (value) => { + try { + return new URL(value).hostname === "bsky.app"; + } catch { + return false; + } + }, + () => false, + blueskySelectors, + async (ref, document, options = {}) => { + try { + return await blueskyThread(ref.url, options); + } catch (reason) { + options.signal?.throwIfAborted(); + const fallback = domFallback(blueskyAdapter, ref, document, blueskySelectors); + // A handle is not an identity. Even the DOM fallback must resolve it to a DID. + const provisional = postAddress(ref.url); + if (!provisional) { + throw reason; + } + const address = await blueskyIdentity(provisional, options); + const part = { + ...fallback, + id: address.id, + url: address.url, + author: { ...fallback.author, id: address.id.split("/")[2] ?? "" }, + }; + if (!part.text) { + throw reason; + } + return unrollThread(part, [], true); + } + }, + (ref, options = {}) => { + const address = postAddress(ref.url); + if (!address) { + throw new Error("Invalid Bluesky post URL."); + } + return blueskyIdentity(address, options); + }, + blueskyRef, +); + +function blueskyRef(element: Element): PostRef | null { + const ordinary = refFor(element, "bluesky", blueskySelectors); + if (ordinary) { + return ordinary; + } + // The expanded post links only to its statistics, not back to its own permalink. + if (!element.querySelector('[data-testid="repostCount-expanded"]')) { + return null; + } + const address = postAddress(element.ownerDocument.URL); + if (address?.platform !== "bluesky") { + return null; + } + const rect = element.getBoundingClientRect(); + return { + ...address, + author: element.querySelector(blueskySelectors.name)?.textContent.trim() ?? "", + snippet: + element.querySelector(blueskySelectors.content)?.textContent.trim().slice(0, 180) ?? "", + top: rect.top, + bottom: rect.bottom, + }; +} + +export async function blueskyThread( + url: string, + options: SocialFetchOptions, + rootFetched = false, +): Promise> { + const address = postAddress(url); + if (address?.platform !== "bluesky") { + throw new Error("Not a Bluesky post URL."); + } + const identity = await blueskyIdentity(address, options); + const result = record( + await publicJson( + `https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?uri=${encodeURIComponent(identity.id)}&depth=100&parentHeight=100`, + options, + ), + ); + const thread = record(result["thread"]); + const selected = blueskyPart(thread["post"]); + const parts: PostPart[] = []; + let partial = false; + const visit = (value: unknown, depth: number): void => { + const node = record(value); + if (!node["post"] || depth > 100) { + partial = true; + return; + } + const part = blueskyPart(node["post"]); + if (parts.some((item) => item.id === part.id)) { + return; + } + parts.push(part); + if (node["parent"]) { + visit(node["parent"], depth + 1); + } + for (const reply of array(node["replies"])) { + const author = record(record(record(reply)["post"])["author"]); + if (author["did"] === selected.author.id) { + visit(reply, depth + 1); + } + } + if (depth === 100) { + partial = true; + } + }; + visit(thread, 0); + const unrolled = unrollThread(selected, parts, partial); + if (unrolled.id !== selected.id && !rootFetched) { + return blueskyThread(unrolled.url, options, true).catch(() => { + options.signal?.throwIfAborted(); + return { ...unrolled, threadPartial: true }; + }); + } + return unrolled; +} diff --git a/packages/web-capture/src/social/dom.ts b/packages/web-capture/src/social/dom.ts new file mode 100644 index 00000000..32a4d24a --- /dev/null +++ b/packages/web-capture/src/social/dom.ts @@ -0,0 +1,162 @@ +import { postAddress, type PostAddress } from "./identity.js"; + +import type { PostMedia, PostPart, SocialPlatform } from "@mdbase-reader/core"; + +export interface PostRef { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly sourceUrl?: string; + readonly author: string; + readonly snippet: string; + readonly published?: string; + readonly top: number; + readonly bottom: number; +} + +export interface DomSelectors { + readonly post: string; + readonly content: string; + readonly permalink: string; + readonly author: string; + readonly name: string; + readonly quote: string; + readonly media: string; +} + +export function refFor( + element: Element, + platform: SocialPlatform, + selectors: DomSelectors, +): PostRef | null { + const address = elementAddress(element, platform, selectors); + if (address?.platform !== platform) { + return null; + } + const rect = element.getBoundingClientRect(); + const published = + element.querySelector("time")?.getAttribute("datetime") ?? + element.getAttribute("data-reader-post-published"); + return { + platform, + id: address.id, + url: address.url, + author: + element.querySelector(selectors.author)?.textContent.trim() ?? + element.getAttribute("data-reader-post-author") ?? + "", + snippet: ( + element.querySelector(selectors.content)?.textContent.trim() ?? + element.getAttribute("data-reader-post-text") ?? + "" + ).slice(0, 180), + ...(published ? { published } : {}), + top: rect.top, + bottom: rect.bottom, + }; +} + +function elementAddress( + element: Element, + platform: SocialPlatform, + selectors: DomSelectors, +): PostAddress | null { + const href = + element.querySelector(selectors.permalink)?.getAttribute("href") ?? + element.getAttribute("data-reader-post-url"); + if (!href) { + return null; + } + const doc = element.ownerDocument; + const fallback = + platform === "bluesky" + ? "https://bsky.app" + : platform === "x" + ? "https://x.com" + : "https://example.invalid"; + const base = doc.URL.startsWith("https://") + ? doc.URL + : (doc.querySelector('meta[property="og:url"]')?.content ?? fallback); + try { + return postAddress(new URL(href, base).href); + } catch { + return null; + } +} + +export function domPart( + element: Element, + ref: PostRef, + selectors: DomSelectors, + depth = 0, +): PostPart { + const quoteElement = element.querySelector(selectors.quote); + const quotedRef = + quoteElement && depth === 0 ? refFor(quoteElement, ref.platform, selectors) : null; + const reply = element.getAttribute("data-in-reply-to"); + return { + platform: ref.platform, + id: ref.id, + url: ref.url, + author: domAuthor(element, ref, selectors), + ...(ref.published ? { published: ref.published } : {}), + text: + element.querySelector(selectors.content)?.textContent.trim() ?? + element.getAttribute("data-reader-post-text") ?? + "", + media: domMedia(element, selectors, quoteElement), + ...(quotedRef && quoteElement + ? { quoted: domPart(quoteElement, quotedRef, selectors, 1) } + : {}), + ...(reply ? { inReplyTo: reply } : {}), + }; +} + +function domAuthor(element: Element, ref: PostRef, selectors: DomSelectors): PostPart["author"] { + const node = element.querySelector(selectors.author); + const link = node?.closest("a") ?? node?.querySelector("a"); + const href = link?.getAttribute("href"); + const handle = + ref.platform === "mastodon" + ? (node?.textContent.trim() ?? ref.author).replace(/^@/u, "") + : (href?.split("/").filter(Boolean).at(-1) ?? ref.author); + return { + id: href ?? handle, + handle, + name: element.querySelector(selectors.name)?.textContent.trim() ?? ref.author, + }; +} + +function domMedia(element: Element, selectors: DomSelectors, quote: Element | null): PostMedia[] { + return [...element.querySelectorAll(selectors.media)] + .filter((node) => !quote?.contains(node)) + .flatMap((node) => { + const source = node.getAttribute("src") ?? node.querySelector("source")?.getAttribute("src"); + const video = node.tagName === "VIDEO"; + const poster = video ? node.getAttribute("poster") : null; + const url = source?.startsWith("https://") ? source : poster; + if (!url?.startsWith("https://")) { + return []; + } + const kind = video && url === source ? ("video" as const) : ("image" as const); + const alt = + node.getAttribute("alt") ?? + node.getAttribute("aria-label") ?? + (poster ? "Video poster" : ""); + return [{ url, alt, kind }]; + }); +} + +export function nearestPost(items: readonly PostRef[], viewportHeight: number): string | null { + const middle = viewportHeight / 2; + return ( + [...items] + .filter((item) => item.bottom > 0 && item.top < viewportHeight) + .sort( + (a, b) => + Math.abs((a.top + a.bottom) / 2 - middle) - Math.abs((b.top + b.bottom) / 2 - middle), + )[0]?.id ?? + items[0]?.id ?? + null + ); +} diff --git a/packages/web-capture/src/social/fixtures/README.md b/packages/web-capture/src/social/fixtures/README.md new file mode 100644 index 00000000..ea00bfaf --- /dev/null +++ b/packages/web-capture/src/social/fixtures/README.md @@ -0,0 +1,11 @@ +# Social DOM fixtures + +Minimal, fictionalised layout fixtures. All authors, IDs, text and media URLs are +synthetic; no account sessions, page scripts or personal data are included. + +Bluesky's profile and expanded-post shapes and Mastodon's status classes were +checked in an unauthenticated LAB Chromium browser on 2026-10-11. The Bluesky +expanded item deliberately has no own permalink and no `postText` test ID. +X's fixture represents its `tweet` / `User-Name` / `tweetText` DOM contract; +there was no logged-in X browser. Its explicit reply attributes model evidence +supplied by a page, not an assumption that every X timeline supplies those edges. diff --git a/packages/web-capture/src/social/fixtures/bluesky-expanded.html b/packages/web-capture/src/social/fixtures/bluesky-expanded.html new file mode 100644 index 00000000..f1bf476a --- /dev/null +++ b/packages/web-capture/src/social/fixtures/bluesky-expanded.html @@ -0,0 +1,17 @@ + + + + Fixture expanded post + + +
+
+
Fixture Author
 +
The first durable post.
+
1 repost
 +
+
+ + diff --git a/packages/web-capture/src/social/fixtures/bluesky.html b/packages/web-capture/src/social/fixtures/bluesky.html new file mode 100644 index 00000000..4b5135dd --- /dev/null +++ b/packages/web-capture/src/social/fixtures/bluesky.html @@ -0,0 +1,34 @@ + + + + Fixture Bluesky profile + + +
+
+
Fixture Author
 +
The first durable post.
+
+ A blue square +
+
+
Context Author
 +
Quoted context.
+
+
+
+
Other Author
 +
Someone else's reply.
+
+
+ + diff --git a/packages/web-capture/src/social/fixtures/mastodon.html b/packages/web-capture/src/social/fixtures/mastodon.html new file mode 100644 index 00000000..ef0af5a0 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/mastodon.html @@ -0,0 +1,34 @@ + + + + + Fixture federated timeline + + +
+
+
+ Fixture Author +

The first durable post.

+
+ A blue square +
+
+
+ Other Author +

Someone else's reply.

+
+
+
+ + diff --git a/packages/web-capture/src/social/fixtures/x-conversation.html b/packages/web-capture/src/social/fixtures/x-conversation.html new file mode 100644 index 00000000..6da28722 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/x-conversation.html @@ -0,0 +1,45 @@ + + + + Fixture X conversation + + +
+ + + + + +
+ + diff --git a/packages/web-capture/src/social/fixtures/x.html b/packages/web-capture/src/social/fixtures/x.html new file mode 100644 index 00000000..fa20ee29 --- /dev/null +++ b/packages/web-capture/src/social/fixtures/x.html @@ -0,0 +1,51 @@ + + + + Fixture X timeline + + + +
+ + + + +
+ + diff --git a/packages/web-capture/src/social/identity.ts b/packages/web-capture/src/social/identity.ts new file mode 100644 index 00000000..16df5dd7 --- /dev/null +++ b/packages/web-capture/src/social/identity.ts @@ -0,0 +1,53 @@ +import type { SocialPlatform } from "@mdbase-reader/core"; + +export interface PostAddress { + readonly platform: SocialPlatform; + readonly id: string; + readonly url: string; + readonly host: string; +} + +/** Handle and federated-instance addresses are provisional until resolved by the API. */ +export function postAddress(value: string): PostAddress | null { + try { + if (value.startsWith("at://")) { + return atAddress(value); + } + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password) { + return null; + } + const host = url.hostname.toLowerCase(); + if (/^(?:(?:www|mobile)\.)?(?:x|twitter)\.com$/u.test(host)) { + const id = + /^\/(?:[^/]+\/status|i\/web\/status|i\/status)\/(\d+)(?:\/(?:photo|video)\/\d+)?\/?$/u.exec( + url.pathname, + )?.[1]; + return id ? { platform: "x", id, url: `https://x.com/i/status/${id}`, host: "x.com" } : null; + } + if (host === "bsky.app") { + const match = /^\/profile\/([^/]+)\/post\/([^/]+)\/?$/u.exec(url.pathname); + return match?.[1] && match[2] ? blueskyAddress(decodeURIComponent(match[1]), match[2]) : null; + } + const id = /^\/(?:@[^/]+|users\/[^/]+\/statuses)\/(\d+)\/?$/u.exec(url.pathname)?.[1]; + return id + ? { platform: "mastodon", id, url: `${url.origin}${url.pathname.replace(/\/$/u, "")}`, host } + : null; + } catch { + return null; + } +} + +function atAddress(value: string): PostAddress | null { + const match = /^at:\/\/([^/]+)\/app\.bsky\.feed\.post\/([^/?#]+)$/u.exec(value); + return match?.[1] && match[2] ? blueskyAddress(match[1], match[2]) : null; +} + +function blueskyAddress(actor: string, key: string): PostAddress { + return { + platform: "bluesky", + id: `at://${actor}/app.bsky.feed.post/${key}`, + url: `https://bsky.app/profile/${actor}/post/${key}`, + host: "bsky.app", + }; +} diff --git a/packages/web-capture/src/social/import.ts b/packages/web-capture/src/social/import.ts new file mode 100644 index 00000000..ab253788 --- /dev/null +++ b/packages/web-capture/src/social/import.ts @@ -0,0 +1,194 @@ +import { dateTime, type PostMedia, type PostPart, type SocialPost } from "@mdbase-reader/core"; + +import { cslDate, type CitationDraft } from "../csl-values.js"; + +import { boundedSignal, type SocialFetchOptions } from "./api.js"; + +import type { CapturedWebDocument, WebCaptureImport } from "../web-capture.js"; + +export function postCitation(post: SocialPost): CitationDraft { + return { + type: "post", + title: postTitle(post), + URL: post.url, + author: [{ literal: post.author.name || post.author.handle }], + "container-title": + post.platform === "x" ? "X" : post.platform === "bluesky" ? "Bluesky" : "Mastodon", + ...(post.published ? { issued: cslDate(post.published) ?? { literal: post.published } } : {}), + }; +} + +export function postTitle(post: SocialPost): string { + return `${post.author.name || post.author.handle}: ${post.text.replace(/\s+/gu, " ").slice(0, 120) || "Post"}`; +} + +/** Only the chosen post/thread is archived; never the surrounding feed or account DOM. */ +export function postHtml(post: SocialPost): string { + return `${escape(postTitle(post))}
`; +} + +function partHtml(part: PostPart): string { + return `

${escape(part.author.name)} · @${escape(part.author.handle)}

${part.published ? `` : ""}
${part.text + .split(/\n\n+/u) + .map((paragraph) => `

${escape(paragraph).replaceAll("\n", "
")}

`) + .join( + "", + )}
${part.media.map(mediaHtml).join("")}${part.quoted ? `
${partHtml(part.quoted)}
` : ""}
`; +} + +/** The archive keeps every original media URL; the readable copy drops external links. */ +function mediaHtml(media: PostMedia): string { + const caption = escape(mediaCaption(media)); + return media.kind === "image" + ? `
${escape(media.alt)}
${caption}
` + : `
${caption}
`; +} + +function mediaCaption(media: PostMedia): string { + if (media.alt) { + return media.alt; + } + return media.kind === "image" + ? "Image" + : `${media.kind === "audio" ? "Audio" : "Video"} (original URL kept in the archived copy)`; +} + +export async function socialPostImport( + post: SocialPost, + capture: Pick, + options: SocialFetchOptions = {}, +): Promise { + const { prepareHtmlDocument } = await import("@mdbase-reader/renderer-html"); + const archive = postHtml(post); + const parsed = new DOMParser().parseFromString(archive, "text/html"); + // Reading copies are self-contained (the renderer allows only data: images), so images + // are embedded, but scaled and re-encoded first to keep the copy light to sync and render. + let remaining = mediaBudget.total; + for (const [index, image] of [...parsed.querySelectorAll("img")].entries()) { + try { + if (index >= mediaBudget.images || remaining <= 0) { + throw new Error("Media budget reached"); + } + const data = await embeddedImage(image.src, Math.min(remaining, mediaBudget.image), options); + remaining -= data.size; + image.src = data.url; + } catch { + options.signal?.throwIfAborted(); + const caption = image.parentElement?.querySelector("figcaption"); + if (caption) { + caption.append(" — image unavailable offline; the archived copy keeps its original URL."); + } + image.remove(); + } + } + const stem = `${post.platform}-${ + post.id + .split("/") + .at(-1) + ?.replace(/[^a-zA-Z0-9_-]/gu, "") ?? "post" + }`; + return { + name: `${stem}.readable.html`, + title: postTitle(post), + bytes: new TextEncoder().encode(prepareHtmlDocument(parsed.documentElement.outerHTML)), + archive: { name: `${stem}.archive.html`, bytes: new TextEncoder().encode(archive) }, + capture: { + submittedUrl: post.url, + canonicalUrl: post.url, + retrievedAt: dateTime(capture.retrievedAt), + }, + metadata: { + socialPost: post, + authors: [post.author.name || post.author.handle], + ...(post.published ? { published: post.published } : {}), + site: post.platform, + description: post.text.slice(0, 2_000), + }, + scholarly: { citation: postCitation(post) }, + }; +} + +const mediaBudget = { + images: 20, + /** Largest original accepted for scaling. */ + download: 10 * 1024 * 1024, + /** Largest embedded image, after scaling. */ + image: 1536 * 1024, + total: 6 * 1024 * 1024, +}; +const maximumDimension = 1600; + +async function embeddedImage( + url: string, + maximum: number, + options: SocialFetchOptions, +): Promise<{ url: string; size: number }> { + const original = await downloadImage(url, options); + const image = await scaledImage(original).catch(() => original); + if (image.size > maximum) { + throw new Error("Image too large"); + } + const bytes = new Uint8Array(await image.arrayBuffer()); + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return { url: `data:${image.type};base64,${btoa(binary)}`, size: bytes.length }; +} + +async function downloadImage(url: string, options: SocialFetchOptions): Promise { + const response = await (options.fetch ?? fetch)(url, { + credentials: "omit", + redirect: "error", + signal: boundedSignal(options.signal, 8_000), + }); + const type = response.headers.get("content-type")?.split(";")[0]; + if (!response.ok || !type || !/^image\/(png|jpeg|webp|gif|avif)$/u.test(type) || !response.body) { + throw new Error("Image unavailable"); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (let next = await reader.read(); !next.done; next = await reader.read()) { + size += next.value.length; + if (size > mediaBudget.download) { + await reader.cancel(); + throw new Error("Image too large"); + } + chunks.push(next.value); + } + return new Blob(chunks, { type }); +} + +/** Large images are scaled down and re-encoded; animated GIFs are kept so they still play. */ +async function scaledImage(image: Blob): Promise { + if ( + image.type === "image/gif" || + typeof createImageBitmap !== "function" || + typeof OffscreenCanvas !== "function" + ) { + return image; + } + const bitmap = await createImageBitmap(image); + try { + const scale = Math.min(1, maximumDimension / Math.max(bitmap.width, bitmap.height)); + const canvas = new OffscreenCanvas( + Math.max(1, Math.round(bitmap.width * scale)), + Math.max(1, Math.round(bitmap.height * scale)), + ); + canvas.getContext("2d")?.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + const encoded = await canvas.convertToBlob({ type: "image/webp", quality: 0.82 }); + // Browsers without WebP encoding return PNG, which is rarely smaller than the original. + return encoded.type === "image/webp" && encoded.size < image.size ? encoded : image; + } finally { + bitmap.close(); + } +} + +function escape(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} diff --git a/packages/web-capture/src/social/index.ts b/packages/web-capture/src/social/index.ts new file mode 100644 index 00000000..b6281250 --- /dev/null +++ b/packages/web-capture/src/social/index.ts @@ -0,0 +1,42 @@ +import { blueskyAdapter, blueskyThread } from "./bluesky.js"; +import { postAddress } from "./identity.js"; +import { mastodonAdapter, mastodonThread } from "./mastodon.js"; +import { xAdapter } from "./x.js"; + +import type { SocialFetchOptions } from "./api.js"; +import type { SiteAdapter } from "./registry.js"; +import type { SocialPost } from "@mdbase-reader/core"; + +export { postAddress } from "./identity.js"; +export { nearestPost } from "./dom.js"; +export { unrollThread } from "./thread.js"; +export type { PostRef } from "./dom.js"; +export type { SiteAdapter, PostDetection } from "./registry.js"; +export type { SocialFetchOptions } from "./api.js"; + +export const siteAdapters = [xAdapter, blueskyAdapter, mastodonAdapter] as const; + +export function adapterFor(document: Document, url = document.URL): SiteAdapter | null { + return siteAdapters.find((adapter) => adapter.detect(document, url) !== null) ?? null; +} + +/** A URL-only Mastodon probe is confirmed by the status API, not assumed from its host. */ +export function postFromUrl( + url: string, + document: Document, + options: SocialFetchOptions = {}, +): Promise { + const address = postAddress(url); + if (!address) { + return Promise.resolve(null); + } + if (address.platform === "x") { + throw new Error( + "Save X posts from their live page using the mdbase Reader extension. Reader cannot retrieve X posts without that page.", + ); + } + if (address.platform === "bluesky") { + return blueskyThread(url, options); + } + return mastodonThread(url, document, options); +} diff --git a/packages/web-capture/src/social/mastodon-profile.ts b/packages/web-capture/src/social/mastodon-profile.ts new file mode 100644 index 00000000..78d0317e --- /dev/null +++ b/packages/web-capture/src/social/mastodon-profile.ts @@ -0,0 +1,104 @@ +import { array, publicJson, record, text } from "./api.js"; + +import type { SocialFetchOptions } from "./api.js"; + +const pending = new WeakMap>(); + +/** Current Mastodon anonymous profiles render text-only placeholders with no permalink. + * Match those loaded placeholders to the official account API, never to unseen API items. */ +export function prepareMastodonProfile( + document: Document, + options: SocialFetchOptions = {}, +): Promise { + const wrappers = [ + ...document.querySelectorAll(".status__wrapper:not([data-reader-post-url])"), + ].filter( + (element) => + element.children.length === 2 && + [...element.children].every((child) => child.tagName === "SPAN"), + ); + const account = /^\/@([^/]+)\/?$/u.exec(document.location.pathname)?.[1]; + if (!wrappers.length || !account) { + return Promise.resolve(); + } + const existing = pending.get(document); + if (existing) { + return existing; + } + const work = hydrate( + document.location.origin, + decodeURIComponent(account), + wrappers, + options, + ).finally(() => pending.delete(document)); + pending.set(document, work); + return work; +} + +async function hydrate( + origin: string, + account: string, + wrappers: readonly Element[], + options: SocialFetchOptions, +): Promise { + const result = record( + await publicJson( + `${origin}/api/v1/accounts/lookup?acct=${encodeURIComponent(account)}`, + options, + ), + ); + const id = text(result["id"]); + if (!/^\d+$/u.test(id)) { + throw new Error("Mastodon could not resolve this profile."); + } + const statuses: Record[] = []; + let maximum = ""; + for (let page = 0; page < 3; page++) { + const batch = array( + await publicJson( + `${origin}/api/v1/accounts/${id}/statuses?limit=40${maximum ? `&max_id=${maximum}` : ""}`, + options, + ), + ).map(record); + statuses.push(...batch); + if (batch.length < 40 || statuses.length >= wrappers.length + 40) { + break; + } + maximum = text(batch.at(-1)?.["id"]); + if (!/^\d+$/u.test(maximum)) { + break; + } + } + for (const wrapper of wrappers) { + const author = wrapper.children[0]?.textContent.trim(); + const content = wrapper.children[1]?.textContent.trim(); + const matches = statuses.filter( + (status) => + text(status["content"]).trim() === content && + text(record(status["account"])["display_name"]).trim() === author, + ); + if (matches.length !== 1) { + continue; + } + const status = matches[0]; + if (!status) { + continue; + } + const statusId = text(status["id"]); + const acct = text(record(status["account"])["acct"]); + if (!/^\d+$/u.test(statusId) || !acct) { + continue; + } + wrapper.setAttribute( + "data-reader-post-url", + `${origin}/@${encodeURIComponent(acct)}/${statusId}`, + ); + wrapper.setAttribute("data-uri", text(status["uri"])); + const parsed = wrapper.ownerDocument.implementation.createHTMLDocument(""); + parsed.body.innerHTML = text(status["content"]); + parsed.querySelectorAll("script,style").forEach((element) => element.remove()); + wrapper.setAttribute("data-reader-post-text", parsed.body.textContent); + wrapper.setAttribute("data-reader-post-author", acct); + wrapper.setAttribute("data-reader-post-published", text(status["created_at"])); + } +} diff --git a/packages/web-capture/src/social/mastodon.ts b/packages/web-capture/src/social/mastodon.ts new file mode 100644 index 00000000..a6e763d6 --- /dev/null +++ b/packages/web-capture/src/social/mastodon.ts @@ -0,0 +1,116 @@ +import { array, mastodonPart, publicJson, record, text, type SocialFetchOptions } from "./api.js"; +import { postAddress } from "./identity.js"; +import { prepareMastodonProfile } from "./mastodon-profile.js"; +import { domAdapter, domFallback } from "./registry.js"; +import { unrollThread } from "./thread.js"; + +import type { DomSelectors } from "./dom.js"; + +export const mastodonSelectors: DomSelectors = { + post: ".status, .detailed-status, .entry:has(.status__content), .status__wrapper[data-reader-post-url]", + content: ".status__content, .e-content", + permalink: "a.status__relative-time, a.detailed-status__datetime, a.u-url", + author: ".display-name__account, a.status__display-name", + name: ".display-name__html", + quote: ".quoted-status, .status__quote", + media: ".media-gallery img, .attachment-list img, .video-player video", +}; + +export function mastodonMarker(document: Document): boolean { + return Boolean( + document.querySelector( + 'meta[name="application-name"][content="Mastodon"], #mastodon, .mastodon, .detailed-status, .status__content', + ), + ); +} + +export const mastodonAdapter = domAdapter( + "mastodon", + () => false, + mastodonMarker, + mastodonSelectors, + async (ref, document, options = {}) => { + try { + return await mastodonThread(ref.url, document, options); + } catch (reason) { + options.signal?.throwIfAborted(); + // Do not guess a remote status's identity from a local instance's numerical ID. + const fallback = domFallback(mastodonAdapter, ref, document, mastodonSelectors); + const element = mastodonAdapter.element(ref, document); + const uri = + element?.getAttribute("data-uri") ?? + element?.querySelector('a[rel="original"]')?.href; + if (!uri?.startsWith("https://")) { + throw reason; + } + const part = { ...fallback, id: uri, url: uri }; + return unrollThread(part, [], true); + } + }, + async (ref, options = {}) => { + const address = postAddress(ref.url); + if (!address) { + throw new Error("Invalid Mastodon post URL."); + } + const status = record( + await publicJson(`https://${address.host}/api/v1/statuses/${address.id}`, options), + ); + const uri = text(status["uri"]); + if (!uri.startsWith("https://")) { + throw new Error("Invalid Mastodon identity."); + } + return { id: uri, url: uri }; + }, +); + +mastodonAdapter.prepare = prepareMastodonProfile; + +export async function mastodonThread( + url: string, + document: Document, + options: SocialFetchOptions, + rootFetched = false, +): Promise> { + const address = postAddress(url); + if (address?.platform !== "mastodon") { + throw new Error("Not a Mastodon status URL."); + } + const origin = new URL(url).origin; + const status = record(await publicJson(`${origin}/api/v1/statuses/${address.id}`, options)); + const selected = mastodonPart(status, document); + const completeness = { partial: false }; + const context = await publicJson( + `${origin}/api/v1/statuses/${address.id}/context`, + options, + ).catch(() => { + options.signal?.throwIfAborted(); + completeness.partial = true; + return {}; + }); + const statuses = [ + status, + ...array(record(context)["ancestors"]), + ...array(record(context)["descendants"]), + ].map(record); + const identities = new Map(statuses.map((item) => [text(item["id"]), text(item["uri"])])); + const parts = statuses.map((item) => { + const part = mastodonPart(item, document); + return { + ...part, + ...(part.inReplyTo ? { inReplyTo: identities.get(part.inReplyTo) ?? part.inReplyTo } : {}), + }; + }); + // Mastodon's context endpoint can be server-capped. Preserve this uncertainty. + const unrolled = unrollThread( + parts.find((part) => part.id === selected.id) ?? selected, + parts, + completeness.partial || statuses.length >= 60, + ); + if (unrolled.id !== selected.id && !rootFetched) { + return mastodonThread(unrolled.url, document, options, true).catch(() => { + options.signal?.throwIfAborted(); + return { ...unrolled, threadPartial: true }; + }); + } + return unrolled; +} diff --git a/packages/web-capture/src/social/registry.ts b/packages/web-capture/src/social/registry.ts new file mode 100644 index 00000000..77c88fc7 --- /dev/null +++ b/packages/web-capture/src/social/registry.ts @@ -0,0 +1,97 @@ +import { domPart, refFor, type DomSelectors, type PostRef } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { unrollThread } from "./thread.js"; + +import type { SocialFetchOptions } from "./api.js"; +import type { SocialPost } from "@mdbase-reader/core"; + +export type PostDetection = "single" | "multiple" | null; + +/** A loaded post and its element, found in one pass over the page. */ +export interface PostEntry { + readonly ref: PostRef; + readonly element: Element; +} + +export interface SiteAdapter { + readonly id: PostRef["platform"]; + matches(url: string): boolean; + prepare(document: Document, options?: SocialFetchOptions): Promise; + identity(ref: PostRef, options?: SocialFetchOptions): Promise<{ id: string; url: string }>; + detect(document: Document, url?: string): PostDetection; + /** Every loaded post with its element. Prefer this to `items` plus `element` per post. */ + entries(document: Document): PostEntry[]; + items(document: Document): PostRef[]; + resolve(element: Element): PostRef | null; + element(ref: PostRef, document: Document): Element | null; + extract(ref: PostRef, document: Document, options?: SocialFetchOptions): Promise; +} + +export function domAdapter( + id: SiteAdapter["id"], + matches: SiteAdapter["matches"], + marker: (document: Document) => boolean, + selectors: DomSelectors, + extract?: SiteAdapter["extract"], + identity?: SiteAdapter["identity"], + refOf: (element: Element) => PostRef | null = (element) => refFor(element, id, selectors), +): SiteAdapter { + const entries = (document: Document): PostEntry[] => { + const seen = new Set(); + return [...document.querySelectorAll(selectors.post)].flatMap((element) => { + const ref = refOf(element); + if (!ref || seen.has(ref.id)) { + return []; + } + seen.add(ref.id); + return [{ ref, element }]; + }); + }; + const adapter: SiteAdapter = { + id, + matches, + prepare: () => Promise.resolve(), + identity: identity ?? ((ref) => Promise.resolve({ id: ref.id, url: ref.url })), + detect: (document, url = document.URL) => { + if (!matches(url) && !marker(document)) { + return null; + } + return postAddress(url)?.platform === id ? "single" : "multiple"; + }, + entries, + items: (document) => entries(document).map(({ ref }) => ref), + resolve: (element) => { + const post = element.closest(selectors.post); + return post ? refOf(post) : null; + }, + element: (ref, document) => + entries(document).find((entry) => entry.ref.id === ref.id)?.element ?? null, + extract: + extract ?? + ((ref, document) => { + const loaded = entries(document); + const element = loaded.find((entry) => entry.ref.id === ref.id)?.element; + if (!element) { + throw new Error("This post is no longer loaded. Scroll it into view and try again."); + } + const available = loaded.map((entry) => domPart(entry.element, entry.ref, selectors)); + return Promise.resolve(unrollThread(domPart(element, ref, selectors), available, true)); + }), + }; + return adapter; +} + +export function domFallback( + adapter: SiteAdapter, + ref: PostRef, + document: Document, + selectors: DomSelectors, +): SocialPost { + const element = adapter.element(ref, document); + if (!element) { + throw new Error( + "Could not retrieve this post. Open its permalink in the extension and try again.", + ); + } + return unrollThread(domPart(element, ref, selectors), [], true); +} diff --git a/packages/web-capture/src/social/social.test.ts b/packages/web-capture/src/social/social.test.ts new file mode 100644 index 00000000..3eda052d --- /dev/null +++ b/packages/web-capture/src/social/social.test.ts @@ -0,0 +1,460 @@ +// @vitest-environment happy-dom +import { readFileSync } from "node:fs"; + +import { normalizedSourceUrl } from "@mdbase-reader/core"; +import { beforeEach, describe, expect, it, vi, type Mock } from "vitest"; + +import { webCaptureImport } from "../web-capture.js"; + +import { blueskyPart, mastodonPart, publicJson } from "./api.js"; +import { blueskyAdapter } from "./bluesky.js"; +import { nearestPost } from "./dom.js"; +import { postAddress } from "./identity.js"; +import { socialPostImport } from "./import.js"; +import { mastodonAdapter } from "./mastodon.js"; +import { unrollThread } from "./thread.js"; +import { xAdapter } from "./x.js"; + +import { adapterFor, siteAdapters } from "./index.js"; + +import type { PostPart } from "@mdbase-reader/core"; + +function fixture(name: string, url: string): Document { + window.location.href = url; + const html = readFileSync(`src/social/fixtures/${name}.html`, "utf8"); + document.documentElement.innerHTML = new DOMParser().parseFromString( + html, + "text/html", + ).documentElement.innerHTML; + return document; +} + +function bskyPost( + key: string, + author = "did:plc:fixture", + parent?: string, +): Record & { uri: string } { + return { + uri: `at://${author}/app.bsky.feed.post/${key}`, + author: { did: author, handle: "fixture.test", displayName: "Fixture Author" }, + record: { + text: `Post ${key}`, + createdAt: "2026-01-01T10:00:00Z", + ...(parent ? { reply: { parent: { uri: parent } } } : {}), + }, + }; +} + +function status( + id: string, + uri: string, + parent?: string, + author = "fixture", +): Record { + return { + id, + uri, + url: uri.replace("/users/fixture/statuses/", "/@fixture/"), + account: { + id: author, + url: `https://home.example/@${author}`, + acct: `${author}@home.example`, + display_name: "Fixture Author", + }, + content: `

Post ${id}

`, + created_at: "2026-01-01T10:00:00Z", + in_reply_to_id: parent ?? null, + media_attachments: [], + }; +} + +function json(value: unknown): Response { + return new Response(JSON.stringify(value), { headers: { "content-type": "application/json" } }); +} + +function requestUrl(input: string | URL | Request): string { + return typeof input === "string" ? input : input instanceof URL ? input.href : input.url; +} + +function mockApi(handler: (url: string) => Response): Mock { + return vi.fn((input) => Promise.resolve(handler(requestUrl(input)))); +} + +beforeEach(() => window.getSelection()?.removeAllRanges()); + +describe("post identity", () => { + it.each([ + "https://x.com/fixture/status/100?s=20&t=tracking#photo", + "https://twitter.com/other/status/100", + "https://mobile.twitter.com/fixture/status/100/photo/1", + "https://mobile.x.com/i/web/status/100", + "https://x.com/i/status/100", + ])("normalises X variant %s by status ID, not author or tracking", (url) => { + expect(postAddress(url)).toMatchObject({ id: "100", url: "https://x.com/i/status/100" }); + expect(normalizedSourceUrl(url)).toBe("https://x.com/i/status/100"); + }); + it("keeps an AT URI and DID permalink identical", () => { + expect(postAddress("at://did:plc:fixture/app.bsky.feed.post/first")).toEqual( + postAddress("https://bsky.app/profile/did:plc:fixture/post/first?ref=test"), + ); + }); + it.each([ + "https://x.com/home", + "https://x.com/fixture/status/100/quotes", + "https://bsky.app/profile/fixture.test", + "https://bsky.app/search", + "https://bad.example/@fixture/no-id", + "https://x.com.evil.example/fixture/status/100", + "https://user:secret@x.com/fixture/status/100", + "http://x.com/fixture/status/100", + ])("does not make a post identity from %s", (url) => expect(postAddress(url)).toBeNull()); + it("recognises Mastodon address forms without trusting an instance hostname", () => { + expect(postAddress("https://home.example/users/fixture/statuses/100?ref=test")).toMatchObject({ + platform: "mastodon", + id: "100", + }); + expect(postAddress("https://local.example/@fixture@home.example/555")).toMatchObject({ + platform: "mastodon", + id: "555", + }); + }); +}); + +describe("saved DOM contracts", () => { + it.each([ + ["x", "https://x.com/home", xAdapter, 4], + ["bluesky", "https://bsky.app/profile/fixture.test", blueskyAdapter, 2], + ["mastodon", "https://local.example/public/local", mastodonAdapter, 2], + ] as const)( + "detects, lists and resolves %s loaded posts, not their feed", + (name, url, adapter, count) => { + const doc = fixture(name, url); + expect(adapter.detect(doc)).toBe("multiple"); + expect(adapterFor(doc)).toBe(adapter); + const refs = adapter.items(doc); + expect(refs).toHaveLength(count); + const root = adapter.element(refs[0]!, doc)!; + expect(adapter.resolve(root.querySelector("time")!)).toEqual(refs[0]); + expect(adapter.resolve(doc.body)).toBeNull(); + expect(refs[0]?.snippet).toContain("first durable post"); + window.location.href = refs[0]!.url; + expect(adapter.detect(doc)).toBe("single"); + }, + ); + it("handles Bluesky's expanded item without a self-permalink or postText marker", () => { + const doc = fixture("bluesky-expanded", "https://bsky.app/profile/fixture.test/post/first"); + const refs = blueskyAdapter.items(doc); + expect(refs).toHaveLength(1); + expect(refs[0]).toMatchObject({ + snippet: "The first durable post.", + id: "at://fixture.test/app.bsky.feed.post/first", + }); + expect(blueskyAdapter.resolve(doc.querySelector(".r-1xnzce8")!)).toEqual(refs[0]); + }); + it("falls through for ordinary sites, including arbitrary numerical paths", () => { + window.location.href = "https://ordinary.example/@user/100"; + document.body.innerHTML = "
An ordinary article
"; + document.head.innerHTML = ""; + expect(siteAdapters.every((adapter) => adapter.detect(document) === null)).toBe(true); + }); + it("extracts X content, media, quoted context and explicit self replies from DOM only", async () => { + const doc = fixture("x", "https://x.com/home"); + const post = await xAdapter.extract(xAdapter.items(doc)[1]!, doc); + expect(post.id).toBe("100"); + expect(post.parts.map((part) => part.id)).toEqual(["100", "101"]); + expect(post.quoted?.text).toContain("Quoted context"); + expect(post.media[0]).toMatchObject({ alt: "A blue square" }); + expect(post.author).toMatchObject({ handle: "fixture", name: "Fixture Author" }); + expect(post.threadPartial).toBe(true); + }); + it("unrolls an X thread from the author's run around the focal post, without reply markup", async () => { + const doc = fixture("x-conversation", "https://x.com/fixture/status/100"); + const focal = xAdapter.items(doc).find((item) => item.id === "100")!; + const post = await xAdapter.extract(focal, doc); + expect(post.threadRoot).toBe("99"); + expect(post.parts.map((part) => part.id)).toEqual(["99", "100", "101"]); + expect(post.threadPartial).toBe(true); + }); + it("never joins X posts into a thread on a feed", async () => { + const doc = fixture("x-conversation", "https://x.com/home"); + const ref = xAdapter.items(doc).find((item) => item.id === "100")!; + expect((await xAdapter.extract(ref, doc)).parts.map((part) => part.id)).toEqual(["100"]); + }); +}); + +describe("public API adapters", () => { + it("resolves Bluesky handles to DID identity and unrolls only same-author replies via AppView", async () => { + const doc = fixture("bluesky", "https://bsky.app/profile/fixture.test"); + const root = bskyPost("first"); + const fetcher = mockApi((url) => + url.includes("resolveHandle") + ? json({ did: "did:plc:fixture" }) + : json({ + thread: { + post: root, + replies: [ + { post: bskyPost("second", "did:plc:fixture", root.uri) }, + { post: bskyPost("other", "did:plc:other", root.uri) }, + ], + }, + }), + ); + const post = await blueskyAdapter.extract(blueskyAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe(root.uri); + expect(post.parts.map((part) => part.text)).toEqual(["Post first", "Post second"]); + expect(post.url).toBe("https://bsky.app/profile/did:plc:fixture/post/first"); + expect( + fetcher.mock.calls.every(([url]) => + requestUrl(url).startsWith("https://public.api.bsky.app/xrpc/"), + ), + ).toBe(true); + }); + it("uses a resolved-DID Bluesky DOM fallback when AppView thread retrieval fails", async () => { + const doc = fixture("bluesky", "https://bsky.app/profile/fixture.test"); + const fetcher = mockApi((url) => + url.includes("resolveHandle") + ? json({ did: "did:plc:fixture" }) + : new Response("", { status: 503 }), + ); + const post = await blueskyAdapter.extract(blueskyAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe("at://did:plc:fixture/app.bsky.feed.post/first"); + expect(post.text).toBe("The first durable post."); + expect(post.threadPartial).toBe(true); + }); + it("normalises federated Mastodon local IDs and home URI forms to one record", async () => { + const doc = fixture("mastodon", "https://local.example/public/local"); + const uri = "https://home.example/users/fixture/statuses/100"; + const fetcher = mockApi((url) => + url.endsWith("/context") + ? json({ + ancestors: [], + descendants: [ + status("556", "https://home.example/users/fixture/statuses/101", "555"), + status("557", "https://local.example/users/other/statuses/557", "555", "other"), + ], + }) + : json(status("555", uri)), + ); + const post = await mastodonAdapter.extract(mastodonAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.id).toBe(uri); + expect(post.url).toBe(uri); + expect(post.parts.map((part) => part.text)).toEqual(["Post 555", "Post 556"]); + expect(post.parts[1]?.inReplyTo).toBe(uri); + expect( + fetcher.mock.calls.some( + ([url]) => requestUrl(url) === "https://local.example/api/v1/statuses/555/context", + ), + ).toBe(true); + }); + it("Mastodon DOM fallback requires a home URI rather than guessing from an instance ID", async () => { + const doc = fixture("mastodon", "https://local.example/public/local"); + const fetcher = mockApi(() => new Response("", { status: 503 })); + const post = await mastodonAdapter.extract(mastodonAdapter.items(doc)[0]!, doc, { + fetch: fetcher, + }); + expect(post.url).toBe("https://home.example/users/fixture/statuses/100"); + await expect( + mastodonAdapter.extract(mastodonAdapter.items(doc)[1]!, doc, { fetch: fetcher }), + ).rejects.toThrow("HTTP 503"); + }); +}); + +describe("Mastodon anonymous profile placeholders", () => { + it("maps only loaded placeholders to official status identities; unseen API posts are not picker items", async () => { + const doc = fixture("mastodon", "https://local.example/@fixture"); + doc.body.innerHTML = + '
Fixture Author<p>Post 555</p>
'; + const known = status("555", "https://home.example/users/fixture/statuses/100"); + const fetcher = mockApi((url) => + url.includes("accounts/lookup") + ? json({ id: "20" }) + : json([known, status("777", "https://home.example/users/fixture/statuses/777")]), + ); + expect(mastodonAdapter.items(doc)).toHaveLength(0); + await mastodonAdapter.prepare(doc, { fetch: fetcher }); + const items = mastodonAdapter.items(doc); + expect(items).toHaveLength(1); + expect(items[0]).toMatchObject({ + id: "555", + snippet: "Post 555", + author: "fixture@home.example", + }); + expect(mastodonAdapter.resolve(doc.querySelector("span")!)?.id).toBe("555"); + }); + it("does not guess when two API statuses have the same author and text", async () => { + const doc = fixture("mastodon", "https://local.example/@fixture"); + doc.body.innerHTML = + '
Fixture Author<p>Post 555</p>
'; + const first = status("555", "https://home.example/users/fixture/statuses/100"); + const fetcher = mockApi((url) => + url.includes("accounts/lookup") ? json({ id: "20" }) : json([first, { ...first, id: "556" }]), + ); + await mastodonAdapter.prepare(doc, { fetch: fetcher }); + expect(mastodonAdapter.items(doc)).toHaveLength(0); + }); +}); + +const part = (id: string, author: string, parent?: string): PostPart => ({ + platform: "x", + id, + url: `https://x.com/i/status/${id}`, + text: `Part ${id}`, + author: { id: author, handle: author, name: author }, + media: [], + ...(parent ? { inReplyTo: parent } : {}), +}); + +describe("threads and picker", () => { + it("unrolls ancestors and self-reply branches but excludes other authors and unrelated posts", () => { + const root = part("1", "author"); + const middle = part("2", "author", "1"); + expect( + unrollThread( + middle, + [root, middle, part("3", "author", "2"), part("4", "other", "2"), part("5", "author")], + false, + ).parts.map((item) => item.id), + ).toEqual(["1", "2", "3"]); + }); + it("terminates cyclic reply graphs", () => { + expect(unrollThread(part("1", "a", "2"), [part("2", "a", "1")], true).parts).toHaveLength(2); + }); + it("preselects the visible post closest to viewport centre", () => { + const refs = xAdapter.items(fixture("x", "https://x.com/home")); + expect( + nearestPost( + refs.map((ref, index) => ({ ...ref, top: index * 300 - 100, bottom: index * 300 + 100 })), + 800, + ), + ).toBe("101"); + expect(nearestPost([], 800)).toBeNull(); + }); + it("embeds Bluesky quoted-post context from official embed data", () => { + const quoted = { ...bskyPost("quoted", "did:plc:quoted"), value: { text: "Quoted context" } }; + const post = blueskyPart({ ...bskyPost("first"), embed: { record: quoted } }); + expect(post.quoted?.text).toBe("Quoted context"); + }); +}); + +describe("portable post imports and capture safety", () => { + it("does not mistake another site's generic initial-state script for a Mastodon page", () => { + const parsed = new DOMParser().parseFromString( + '
Ordinary article
', + "text/html", + ); + expect(mastodonAdapter.detect(parsed, "https://ordinary-site.test/@fixture/123")).toBeNull(); + expect(adapterFor(parsed, "https://ordinary-site.test/article")).toBeNull(); + }); + it("retains Mastodon audio attachment URLs/types without pretending to archive playback", async () => { + const root = mastodonPart( + { + uri: "https://any-instance.test/users/fixture/statuses/100", + content: "

Audio note

", + account: { + url: "https://any-instance.test/@fixture", + acct: "fixture", + display_name: "Fixture Author", + }, + media_attachments: [ + { type: "audio", url: "https://media.example/note.mp3", description: "" }, + ], + }, + document, + ); + expect(root.media[0]?.kind).toBe("audio"); + const prepared = await socialPostImport(unrollThread(root, [], false), { + retrievedAt: "2026-01-01T10:00:00Z", + }); + expect(new TextDecoder().decode(prepared.bytes)).toContain( + "Audio (original URL kept in the archived copy)", + ); + expect(prepared.metadata.socialPost?.media[0]?.url).toBe("https://media.example/note.mp3"); + expect(new TextDecoder().decode(prepared.archive.bytes)).toContain( + 'href="https://media.example/note.mp3"', + ); + }); + it("drops an image that cannot be made small enough to embed, keeping its caption", async () => { + const root = { + ...part("1", "author"), + media: [ + { kind: "image" as const, url: "https://media.example/huge.png", alt: "A huge image" }, + ], + }; + const prepared = await socialPostImport( + unrollThread(root, [], false), + { retrievedAt: "2026-01-01T10:00:00Z" }, + { + fetch: mockApi( + () => + new Response(new Uint8Array(2 * 1024 * 1024), { + headers: { "content-type": "image/png" }, + }), + ), + }, + ); + const readable = new TextDecoder().decode(prepared.bytes); + expect(readable).not.toContain("data:image"); + expect(readable).toContain("A huge image — image unavailable offline"); + expect(new TextDecoder().decode(prepared.archive.bytes)).toContain( + 'src="https://media.example/huge.png"', + ); + }); + it("stores safe readable HTML, a post-only archive and portable metadata with offline images", async () => { + const root = { + ...part("1", "author"), + text: '', + media: [ + { kind: "image" as const, url: "https://media.example/one.png", alt: "A blue square" }, + ], + }; + const prepared = await socialPostImport( + unrollThread(root, [], false), + { retrievedAt: "2026-01-01T10:00:00Z" }, + { + fetch: mockApi( + () => + new Response(new Uint8Array([1, 2, 3]), { headers: { "content-type": "image/png" } }), + ), + }, + ); + const readable = new TextDecoder().decode(prepared.bytes); + expect(readable).toContain("data:image/png;base64,"); + expect(readable).not.toContain("