From 20ef143f4849286c6dc4013bf1ccf1f26531676b Mon Sep 17 00:00:00 2001 From: Dam Minh Tuan Anh Date: Tue, 22 Sep 2026 16:15:22 +0700 Subject: [PATCH 1/5] feat(playwright-notion): extract comments and download attachments Notion's native export drops every comment, and the converter fallback only linked attachments by their signed URL, which expires within days. Both gaps hit requirement pages hardest: that is where scope changes and final formulas get decided. download.mjs now: - keeps discussion/comment/notion_user from the record map and writes .comments.md, numbered #1..#n with anchors, each thread linking back to its Notion discussion; commented blocks reference it inline - downloads attachments to assets// and rewrites links to the local copies, including files held in database row properties (NOTION_MAX_MB, NOTION_MEDIA, NOTION_NO_ASSETS) - deep-links headings and toggles to the exact block in Notion - resolves person and page mentions to names instead of @user / [[page]] - prefers loadCachedPageChunkV2 and takes spaceId from the response - reports block, file and comment counts per page, and names every file skipped for size or media type The driver is skipped under NOTION_SELFTEST and playwright is imported lazily, so test-download.mjs can exercise the renderers without deps. --- .agents/skills/playwright-notion/SKILL.md | 45 ++- .../playwright-notion/scripts/download.mjs | 287 +++++++++++++++--- .../scripts/test-download.mjs | 46 +++ 3 files changed, 324 insertions(+), 54 deletions(-) create mode 100644 .agents/skills/playwright-notion/scripts/test-download.mjs diff --git a/.agents/skills/playwright-notion/SKILL.md b/.agents/skills/playwright-notion/SKILL.md index fa7a9e9..a43ed04 100644 --- a/.agents/skills/playwright-notion/SKILL.md +++ b/.agents/skills/playwright-notion/SKILL.md @@ -1,6 +1,6 @@ --- name: playwright-notion -description: Use when downloading or reading Notion pages without an API token — the workspace is company-owned, there is no integration secret, the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a Notion scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text mixed into content) or when a headless browser lands on the Notion login screen. +description: Use when downloading or reading Notion pages without an API token — the workspace is company-owned, there is no integration secret, the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a Notion scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text mixed into content), when a headless browser lands on the Notion login screen, or when a page's comments and attachments are needed alongside its body because that is where the spec was actually decided. --- # Playwright Notion @@ -36,13 +36,15 @@ Skipping either one wastes an hour. Both were verified by failure on macOS. | Script | Endpoint | Output | Use | |---|---|---|---| | `scripts/export.mjs` | `enqueueTask` (`exportBlock`) → zip | Notion's own markdown + images downloaded | **Try this first** | -| `scripts/download.mjs` | `loadPageChunk` + `queryCollection` | markdown built by a local converter | Fallback if export is blocked | +| `scripts/download.mjs` | `loadCachedPageChunkV2` + `queryCollection` + `getSignedFileUrls` | markdown + `comments.md` + downloaded `assets/` | Fallback if export is blocked, **and whenever comments matter** | **A disabled Export button does not mean export is blocked.** In many workspaces the button is only hidden client-side by role, while the server still accepts the export task. That was true in the case this skill was built from: the UI offered no Export, yet `enqueueTask` returned `200` and produced a proper zip. So always test `export.mjs` on one page before falling back. -Native export is better where it works: it resolves person mentions to real names (`佐藤珠未/Tamami Sato`, not `@user`), resolves page mentions to titles plus URLs (not `[[page]]`), and downloads embedded images into a folder beside the markdown. +Native export is better where it works: it is Notion's own renderer, so tables, nested content and formatting come out exactly as the page reads. -Fall back to `download.mjs` only when `export.mjs` reports an `enqueueTask` `401`/`Unauthorized` — that means the workspace really did disable export server-side (an Enterprise setting). `download.mjs` still works there, because it uses nothing more than the read access the browser already has. +**But export drops every comment.** In most workspaces the decisions — a scope change, the answer to an open question, the final formula — live in comments, not in the body. When the page is a requirement and the comments matter, run `download.mjs` as well and keep both: export for the body, `download.mjs` for `.comments.md`. + +Fall back to `download.mjs` when `export.mjs` reports an `enqueueTask` `401`/`Unauthorized` — that means the workspace really did disable export server-side (an Enterprise setting). `download.mjs` still works there, because it uses nothing more than the read access the browser already has. ## Workflow @@ -90,6 +92,15 @@ node scripts/download.mjs urls.txt ./notion-docs 9222 # fallback `NOTION_RECURSIVE=1` on `export.mjs` exports each page's subtree as well. Leave it off by default — on a database page it pulls the entire table. +`download.mjs` environment variables: + +| Variable | Default | Effect | +|---|---|---| +| `NOTION_MAX_MB` | `30` | skip attachments larger than this | +| `NOTION_MEDIA` | *(off)* | `=1` also downloads video/audio (usually huge) | +| `NOTION_NO_ASSETS` | *(off)* | `=1` keeps the original URLs and downloads nothing | +| `NOTION_RAW` | *(off)* | `=1` also dumps `.raw.json` — use it when comments come out `0` but the UI shows some | + **Step 6 — verify.** Count `OK` lines against the URL count and report any `FAIL`/`GAVE UP` line. Never report success from an exit code alone. ```bash @@ -97,6 +108,8 @@ grep -c OK /tmp/notion_export.log # export.mjs grep -E "FAIL|GAVE UP" /tmp/notion_export.log ``` +`download.mjs` logs `N comments M files` per page. If a page shows `0 comments` while the Notion UI clearly has some, say so — do not let it pass silently. Re-run with `NOTION_RAW=1` and check whether `discussion` in the dump is empty (really none) or populated (a renderer bug). + ## Quick reference | Need | Command | @@ -107,6 +120,7 @@ grep -E "FAIL|GAVE UP" /tmp/notion_export.log | Native export (preferred) | `node scripts/export.mjs urls.txt ./out 9222` | | Converter (fallback) | `node scripts/download.mjs urls.txt ./out 9222` | | Check results | `grep -c OK /tmp/notion_export.log` | +| Self-check the converter | `node scripts/test-download.mjs` | Logs default to `/tmp/notion_export.log` and `/tmp/notion_dl.log`; override with `NOTION_DL_LOG`. @@ -114,7 +128,25 @@ Logs default to `/tmp/notion_export.log` and `/tmp/notion_dl.log`; override with **`export.mjs`** — Notion's own export, unpacked: one `.md` per page under a workspace-named folder, plus a sibling folder of downloaded images per page. Properties appear as a plain key/value block at the top, mentions and relations resolved to names and URLs. -**`download.mjs`** — one `.md` per page named by its Notion title (unsafe characters replaced, duplicates suffixed `(2)`), containing title, source URL, a `## Properties` section, headings, nested lists, to-do checkboxes, toggles, quotes, callouts as `> [!NOTE]`, code blocks with language tags, equations as `$$`, dividers, tables with correct columns (inline tables and embedded database views via `queryCollection`), and links/images/files as URLs. Its known limits, worth stating rather than hiding: page mentions render as `[[page]]` and person mentions as `@user`, because the API returns ids there. +**`download.mjs`** — per page: + +``` +/ + .md # body + .comments.md # every comment, numbered #1..#n, anchored + assets// # attachments downloaded from that page +``` + +The body holds title, source URL, an extraction header (block/file/comment counts), a `## Properties` section, headings, nested lists, to-do checkboxes, toggles, quotes, callouts as `> [!NOTE]`, code blocks with language tags, equations as `$$`, dividers, and tables with correct columns (inline tables and embedded database views via `queryCollection`). + +Traceability — the reason to prefer this over a plain scrape: + +- a commented block carries `> 💬 n comment → [#a–#b](.comments.md#c-a)`; +- each heading and toggle carries `[↗](#)`, opening that exact block in Notion; +- each comment thread carries `[↗](?d=)`, opening that exact thread; +- person mentions resolve to real names and page mentions to titles plus URLs, from the same recordMap. + +Attachments are downloaded rather than linked, because Notion's file URLs are signed and expire — a document that only links them is empty within days. Files skipped for size or media type are named in the header; report them rather than letting them vanish. ## Common mistakes @@ -129,6 +161,8 @@ Logs default to `/tmp/notion_export.log` and `/tmp/notion_dl.log`; override with | Reusing one tab for 30+ pages | Renderer crashes; every later page fails | Both scripts recycle the tab every 5 pages and retry 3x | | Plain `unzip` on the export zip | `Illegal byte sequence`, Japanese/Vietnamese names destroyed | Decode cp437→utf-8 (`export.mjs` does) | | Scraping the sidebar for the URL list | Gets database views, not the wanted rows | Ask the user for explicit URLs | +| Treating `export.mjs` output as the whole requirement | Every comment is missing — that is where the spec gets decided | Also run `download.mjs` for `comments.md` | +| Keeping Notion's file URLs instead of the files | Signed URLs expire; the doc is empty a few days later | `download.mjs` saves them under `assets/` | | Piping a long run through `tail` | Output buffered, progress invisible | Log to a file, `grep` it | ## Red flags — stop and re-read this skill @@ -138,6 +172,7 @@ Logs default to `/tmp/notion_export.log` and `/tmp/notion_dl.log`; override with - About to skip `export.mjs` because the UI hides the Export button → test it anyway. - About to launch a browser with `open -a ... --args` → flags get dropped. - About to report "downloaded all pages" without grepping the log → verify first. +- About to hand over an export as "the requirement" without its comments → the decisions are in the comments. ## Warn the user before starting diff --git a/.agents/skills/playwright-notion/scripts/download.mjs b/.agents/skills/playwright-notion/scripts/download.mjs index 4a4d37b..bf2e55f 100644 --- a/.agents/skills/playwright-notion/scripts/download.mjs +++ b/.agents/skills/playwright-notion/scripts/download.mjs @@ -1,4 +1,3 @@ -import { chromium } from 'playwright'; import fs from 'fs'; import path from 'path'; @@ -7,9 +6,13 @@ const URLS_FILE = process.argv[2] || './urls.txt'; const OUT = process.argv[3] || './notion-docs'; const PORT = process.argv[4] || '9222'; const LOG = process.env.NOTION_DL_LOG || '/tmp/notion_dl.log'; +const MAX_MB = Number(process.env.NOTION_MAX_MB || 30); +const WITH_MEDIA = process.env.NOTION_MEDIA === '1'; +const NO_ASSETS = process.env.NOTION_NO_ASSETS === '1'; +const WITH_RAW = process.env.NOTION_RAW === '1'; const log = (m) => { fs.appendFileSync(LOG, m + '\n'); console.log(m); }; -const URLS = fs.readFileSync(URLS_FILE, 'utf8').split('\n').map(s => s.trim()).filter(Boolean); +const ATTACHMENT = /amazonaws|secure\.notion|prod-files|attachment:/; function dashId(raw) { const hex = raw.replace(/-/g, ''); @@ -19,27 +22,33 @@ function idFromUrl(u) { const m = u.match(/([a-f0-9]{32})/i); return m ? dashId(m[1]) : null; } +const noDash = (id) => String(id).replace(/-/g, ''); +const safeName = (s) => (s || 'file').replace(/[/\\?%*:|"<>]/g, '_').replace(/\s+/g, '_').slice(0, 120); -// Fetch every block of a page (recursive chunks), run inside the browser +// Fetch every block of a page (recursive chunks), run inside the browser. +// discussion/comment/notion_user come in the same recordMap - keep them. async function fetchRecordMap(page, pageId) { return page.evaluate(async (pid) => { - const merged = { block: {}, collection: {}, collection_view: {} }; + const TABLES = ['block', 'collection', 'collection_view', 'discussion', 'comment', 'notion_user']; + const merged = Object.fromEntries(TABLES.map(t => [t, {}])); let cursor = { stack: [] }; + let spaceId = null; for (let i = 0; i < 40; i++) { - const r = await fetch('/api/v3/loadPageChunk', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ pageId: pid, limit: 100, cursor, chunkNumber: i, verticalColumns: false }) + const body = JSON.stringify({ pageId: pid, limit: 100, cursor, chunkNumber: i, verticalColumns: false }); + const post = (ep) => fetch('/api/v3/' + ep, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body }); + let r = await post('loadCachedPageChunkV2'); + if (r.status !== 200) r = await post('loadPageChunk'); if (r.status !== 200) break; const j = await r.json(); const rm = j.recordMap || {}; - for (const t of ['block', 'collection', 'collection_view']) { - Object.assign(merged[t], rm[t] || {}); - } + for (const t of TABLES) Object.assign(merged[t], rm[t] || {}); + spaceId = spaceId || j.spaceId || null; if (!j.cursor || !j.cursor.stack || j.cursor.stack.length === 0) break; cursor = j.cursor; } - return merged; + return { rm: merged, spaceId }; }, pageId); } @@ -59,8 +68,46 @@ async function fetchCollectionRows(page, collectionId, viewId, spaceId) { }, { cid: collectionId, vid: viewId, sid: spaceId }); } +// Exchange raw file sources for time-limited signed download URLs +async function fetchSignedUrls(page, refs) { + return page.evaluate(async (list) => { + const r = await fetch('/api/v3/getSignedFileUrls', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ urls: list.map(f => ({ url: f.source, permissionRecord: { table: 'block', id: f.id } })) }) + }); + if (r.status !== 200) return []; + return (await r.json()).signedUrls || []; + }, refs); +} + const unwrap = (w) => w?.value?.value || w?.value || null; +// per-page state, set in downloadPage +let RM = null; // record map, so rich() can resolve users and page mentions +let BASE = ''; // page url without hash, for [↗] deep links +let ASSETS = {}; // original file url -> local relative path +let COMMENTS = []; // markdown chunks for .comments.md +let COMMENT_SEEN = new Set(); +let CNUM = 0; + +function setPageContext(rm, baseUrl, assets = {}) { + RM = rm; BASE = baseUrl; ASSETS = assets; + COMMENTS = []; COMMENT_SEEN = new Set(); CNUM = 0; + return { comments: COMMENTS }; +} + +const userName = (id) => { + const u = unwrap(RM?.notion_user?.[id]); + if (!u) return 'user'; + return u.name || [u.given_name, u.family_name].filter(Boolean).join(' ') || u.email || 'user'; +}; +const pageMention = (id) => { + const p = unwrap(RM?.block?.[id]); + const t = p ? rich(p.properties?.title) : ''; + return `[${t || 'page'}](https://www.notion.so/${noDash(id)})`; +}; +const stamp = (t) => (t ? new Date(t).toISOString().replace('T', ' ').slice(0, 16) : ''); + // Notion rich text array -> markdown inline function rich(arr) { if (!Array.isArray(arr)) return ''; @@ -71,8 +118,8 @@ function rich(arr) { if (t === '‣') { for (const f of fmts) { if (f[0] === 'd' && f[1]?.start_date) return f[1].start_date + (f[1].end_date ? ` → ${f[1].end_date}` : ''); - if (f[0] === 'p') return '[[page]]'; - if (f[0] === 'u') return '@user'; + if (f[0] === 'p') return pageMention(f[1]); + if (f[0] === 'u') return `@${userName(f[1])}`; } return ''; } @@ -88,13 +135,41 @@ function rich(arr) { case 'e': t = `$${f[1]}$`; break; } } - if (link) t = `[${t}](${link})`; + if (link) t = `[${t}](${ASSETS[link] || link})`; return t; }).join(''); } const propText = (v) => Array.isArray(v) ? rich(v) : (v == null ? '' : String(v)); +// Number every comment thread hanging off a block, stash the markdown for +// comments.md, and return the inline reference line for the body document. +function commentRef(b, anchorText) { + const nums = []; + for (const did of b.discussions || []) { + const d = unwrap(RM.discussion?.[did]); + if (!d || !(d.comments || []).length) continue; + const items = []; + for (const cid of d.comments) { + const c = unwrap(RM.comment?.[cid]); + if (!c || COMMENT_SEEN.has(cid)) continue; + COMMENT_SEEN.add(cid); + const n = ++CNUM; + nums.push(n); + const who = userName(c.created_by_id || c.created_by?.id); + items.push(`### #${n} — ${who} · ${stamp(c.created_time)}\n\n${rich(c.text) || '_(empty)_'}`); + } + if (!items.length) continue; + COMMENTS.push([ + `## On: ${anchorText || '(page)'}${d.resolved ? ' · _resolved_' : ''} · [↗](${BASE}?d=${noDash(did)})`, + '', items.join('\n\n'), '' + ].join('\n')); + } + if (!nums.length) return ''; + const label = nums.length === 1 ? `#${nums[0]}` : `#${nums[0]}–#${nums[nums.length - 1]}`; + return `💬 ${nums.length} comment → [${label}](COMMENTS_FILE#c-${nums[0]})`; +} + function renderBlocks(ids, rm, depth, collections, seen) { const out = []; const ind = ' '.repeat(depth); @@ -109,18 +184,22 @@ function renderBlocks(ids, rm, depth, collections, seen) { const txt = rich(b.properties?.title); const kids = b.content || []; const t = b.type; + const deep = ` [↗](${BASE}#${noDash(id)})`; if (t !== 'numbered_list') numCounter = 0; + const ref = commentRef(b, txt || t); + if (ref) out.push(`${ind}> ${ref}\n`); + switch (t) { - case 'header': out.push(`${ind}## ${txt}\n`); break; - case 'sub_header': out.push(`${ind}### ${txt}\n`); break; - case 'sub_sub_header': out.push(`${ind}#### ${txt}\n`); break; + case 'header': out.push(`${ind}## ${txt}${deep}\n`); break; + case 'sub_header': out.push(`${ind}### ${txt}${deep}\n`); break; + case 'sub_sub_header': out.push(`${ind}#### ${txt}${deep}\n`); break; case 'text': out.push(txt ? `${ind}${txt}\n` : ''); break; case 'bulleted_list': out.push(`${ind}- ${txt}`); break; case 'numbered_list': numCounter++; out.push(`${ind}${numCounter}. ${txt}`); break; case 'to_do': out.push(`${ind}- [${b.properties?.checked?.[0]?.[0] === 'Yes' ? 'x' : ' '}] ${txt}`); break; - case 'toggle': out.push(`${ind}
${txt}\n`); break; + case 'toggle': out.push(`${ind}
${txt}${deep}\n`); break; case 'quote': out.push(`${ind}> ${txt}\n`); break; case 'callout': out.push(`${ind}> [!NOTE]\n${ind}> ${txt.replace(/\n/g, `\n${ind}> `)}\n`); break; case 'code': { @@ -133,12 +212,12 @@ function renderBlocks(ids, rm, depth, collections, seen) { case 'image': { const src = b.properties?.source?.[0]?.[0] || ''; const cap = rich(b.properties?.caption); - out.push(`${ind}![${cap}](${src})\n`); + out.push(`${ind}![${cap}](${ASSETS[src] || src})\n`); break; } case 'file': case 'pdf': case 'video': case 'audio': { const src = b.properties?.source?.[0]?.[0] || ''; - out.push(`${ind}[${t}: ${rich(b.properties?.title) || src}](${src})\n`); + out.push(`${ind}[${t}: ${rich(b.properties?.title) || src}](${ASSETS[src] || src})\n`); break; } case 'bookmark': { @@ -219,6 +298,75 @@ function renderCollection(colWrap, queryResult, rm) { return lines.join('\n') + '\n'; } +// Every file this page points at: file/image blocks plus attachment links +// living inside rich text (database row properties keep their files there). +function collectFileRefs(rm) { + const refs = []; + const seen = new Set(); + const add = (id, source) => { + if (!source || seen.has(source)) return; + seen.add(source); + refs.push({ id, source }); + }; + for (const bid in rm.block) { + const b = unwrap(rm.block[bid]); + if (!b) continue; + if (['image', 'file', 'pdf', 'video', 'audio'].includes(b.type)) add(bid, b.properties?.source?.[0]?.[0]); + for (const key in b.properties || {}) { + for (const seg of b.properties[key] || []) { + for (const f of seg[1] || []) if (f[0] === 'a' && ATTACHMENT.test(f[1] || '')) add(bid, f[1]); + } + } + } + return refs; +} + +// Download attachments next to the markdown; signed Notion URLs expire, so a +// document that only links them is empty a few days later. +async function downloadAssets(page, rm, dir) { + const refs = collectFileRefs(rm); + if (!refs.length) return { map: {}, saved: 0, skipped: [] }; + + let signed = []; + try { signed = await fetchSignedUrls(page, refs); } catch { /* fall back to raw urls */ } + + const map = {}; + const skipped = []; + let saved = 0; + fs.mkdirSync(dir, { recursive: true }); + + for (let i = 0; i < refs.length; i++) { + const { source } = refs[i]; + const url = signed[i] || (/^https?:/.test(source) ? source : null); + if (!url) continue; + const rawName = decodeURIComponent((source.split('?')[0].split('/').pop() || 'file').replace(/^.*:/, '')); + if (!WITH_MEDIA && /\.(mov|mp4|m4v|avi|webm|mkv|mp3|wav|m4a)$/i.test(rawName)) { + skipped.push(`${rawName} (media, set NOTION_MEDIA=1)`); + continue; + } + try { + const resp = await page.context().request.get(url, { timeout: 180000 }); + if (!resp.ok()) throw new Error('HTTP ' + resp.status()); + const buf = await resp.body(); + if (buf.length > MAX_MB * 1024 * 1024) { + skipped.push(`${rawName} (${(buf.length / 1048576).toFixed(1)}MB > NOTION_MAX_MB)`); + continue; + } + let name = safeName(rawName); + if (!path.extname(name)) name += '.bin'; + let file = path.join(dir, name); + let n = 2; + while (fs.existsSync(file) && fs.statSync(file).size !== buf.length) file = path.join(dir, `${n++}_${name}`); + fs.writeFileSync(file, buf); + map[source] = path.relative(OUT, file).split(path.sep).join('/'); + saved++; + } catch (e) { + skipped.push(`${rawName} (${e.message})`); + } + } + return { map, saved, skipped }; +} + async function downloadPage(page, url, idx, total) { const pid = idFromUrl(url); if (!pid) { log(`[${idx}/${total}] SKIP (no id): ${url}`); return; } @@ -226,15 +374,23 @@ async function downloadPage(page, url, idx, total) { await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 }); await page.waitForTimeout(2500); - const rm = await fetchRecordMap(page, pid); + const { rm, spaceId: chunkSpaceId } = await fetchRecordMap(page, pid); const root = unwrap(rm.block[pid]); if (!root) { log(`[${idx}/${total}] FAIL no root block: ${url}`); return; } + RM = rm; + BASE = url.split('#')[0]; + ASSETS = {}; + COMMENTS = []; + COMMENT_SEEN = new Set(); + CNUM = 0; + const title = rich(root.properties?.title) || pid; + const safe = title.replace(/[/\\?%*:|"<>]/g, '-').replace(/\s+/g, ' ').trim().slice(0, 120) || pid; // Resolve any embedded collection views into markdown tables const collections = {}; - const spaceId = rm.block[pid]?.spaceId; + const spaceId = chunkSpaceId || root.space_id || rm.block[pid]?.spaceId; for (const bid in rm.block) { const b = unwrap(rm.block[bid]); if (!b || !['collection_view', 'collection_view_page'].includes(b.type)) continue; @@ -250,6 +406,10 @@ async function downloadPage(page, url, idx, total) { } catch (e) { /* ignore one view */ } } + let assets = { map: {}, saved: 0, skipped: [] }; + if (!NO_ASSETS) assets = await downloadAssets(page, rm, path.join(OUT, 'assets', safe)); + ASSETS = assets.map; + // If the page itself IS a database page, its own row properties are useful const propLines = []; const parentColId = root.parent_table === 'collection' ? root.parent_id : null; @@ -262,53 +422,82 @@ async function downloadPage(page, url, idx, total) { } } + const pageRef = commentRef(root, title); const body = renderBlocks(root.content, rm, 0, collections, new Set([pid])); + // name is only known now, so comment references are patched in at the end + const commentsName = `${safe}.comments.md`; + const fix = (s) => s.split('COMMENTS_FILE').join(encodeURI(commentsName)); + const md = [ `# ${title}`, '', `> Source: ${url}`, + `> Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${Object.keys(rm.block).length} blocks · ${assets.saved} files`, + ...(CNUM ? [`> 💬 ${CNUM} comments in [${commentsName}](${encodeURI(commentsName)}) — every commented block below links into it.`] : []), + ...(assets.skipped.length ? [`> ⚠️ not downloaded: ${assets.skipped.join(', ')}`] : []), '', + ...(pageRef ? [`> ${pageRef}`, ''] : []), ...(propLines.length ? ['## Properties', '', ...propLines, ''] : []), body ].join('\n'); - const safe = title.replace(/[/\\?%*:|"<>]/g, '-').replace(/\s+/g, ' ').trim().slice(0, 120) || pid; let file = path.join(OUT, `${safe}.md`); let n = 2; while (fs.existsSync(file)) { file = path.join(OUT, `${safe} (${n++}).md`); } - fs.writeFileSync(file, md, 'utf8'); - log(`[${idx}/${total}] OK ${Math.round(md.length/1024)}KB ${file}`); + fs.writeFileSync(file, fix(md), 'utf8'); + + if (CNUM) { + const cmd = [ + `# Comments — ${title}`, + '', + `> Source: ${url}`, + `> ${CNUM} comments · body: [${safe}.md](${encodeURI(safe + '.md')})`, + '', + ...COMMENTS + ].join('\n'); + fs.writeFileSync(path.join(OUT, commentsName), fix(cmd), 'utf8'); + } + if (WITH_RAW) fs.writeFileSync(path.join(OUT, `${safe}.raw.json`), JSON.stringify(rm), 'utf8'); + + log(`[${idx}/${total}] OK ${Math.round(md.length/1024)}KB ${CNUM} comments ${assets.saved} files ${file}`); } -fs.writeFileSync(LOG, ''); -fs.mkdirSync(OUT, { recursive: true }); -log(`URLs: ${URLS.length}`); +export { rich, renderBlocks, commentRef, setPageContext }; -const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); -const ctx = browser.contexts()[0]; -const page = await ctx.newPage(); +// skipped when imported by the self-check +if (!process.env.NOTION_SELFTEST) { + const URLS = fs.readFileSync(URLS_FILE, 'utf8').split('\n').map(s => s.trim()).filter(Boolean); + fs.writeFileSync(LOG, ''); + fs.mkdirSync(OUT, { recursive: true }); + log(`URLs: ${URLS.length}`); -let cur = page; -for (let i = 0; i < URLS.length; i++) { - // recycle the tab periodically: Notion leaks memory and crashes the renderer - if (i > 0 && i % 5 === 0) { - try { await cur.close(); } catch {} - cur = await ctx.newPage(); - } - let done = false; - for (let attempt = 1; attempt <= 3 && !done; attempt++) { - try { - await downloadPage(cur, URLS[i], i + 1, URLS.length); - done = true; - } catch (e) { - log(`[${i+1}/${URLS.length}] attempt ${attempt} failed: ${e.message}`); + const { chromium } = await import('playwright'); + const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); + const ctx = browser.contexts()[0]; + const page = await ctx.newPage(); + + let cur = page; + for (let i = 0; i < URLS.length; i++) { + // recycle the tab periodically: Notion leaks memory and crashes the renderer + if (i > 0 && i % 5 === 0) { try { await cur.close(); } catch {} cur = await ctx.newPage(); } + let done = false; + for (let attempt = 1; attempt <= 3 && !done; attempt++) { + try { + await downloadPage(cur, URLS[i], i + 1, URLS.length); + done = true; + } catch (e) { + log(`[${i+1}/${URLS.length}] attempt ${attempt} failed: ${e.message}`); + try { await cur.close(); } catch {} + cur = await ctx.newPage(); + } + } + if (!done) log(`[${i+1}/${URLS.length}] GAVE UP ${URLS[i]}`); } - if (!done) log(`[${i+1}/${URLS.length}] GAVE UP ${URLS[i]}`); + try { await cur.close(); } catch {} + log('DONE'); + process.exit(0); } -try { await cur.close(); } catch {} -log('DONE'); -process.exit(0); diff --git a/.agents/skills/playwright-notion/scripts/test-download.mjs b/.agents/skills/playwright-notion/scripts/test-download.mjs new file mode 100644 index 0000000..f22fec0 --- /dev/null +++ b/.agents/skills/playwright-notion/scripts/test-download.mjs @@ -0,0 +1,46 @@ +// self-check for the pure renderers: node test-download.mjs +process.env.NOTION_SELFTEST = '1'; +const { rich, renderBlocks, commentRef, setPageContext } = await import('./download.mjs'); +import assert from 'assert'; + +const B = (v) => ({ value: v }); +const rm = { + notion_user: { 'u-1': B({ id: 'u-1', name: 'Tamami Sato' }) }, + discussion: { 'd-1': B({ id: 'd-1', comments: ['c-1', 'c-2'], resolved: false }) }, + comment: { + 'c-1': B({ id: 'c-1', created_by_id: 'u-1', created_time: 1700000000000, text: [['spec chot: dung ty gia cuoi thang']] }), + 'c-2': B({ id: 'c-2', created_by_id: 'u-1', created_time: 1700000100000, text: [['ok']] }), + }, + block: { + 'p-9': B({ id: 'p-9', type: 'page', properties: { title: [['Linked Page']] } }), + 'h-1': B({ id: 'h-1', type: 'header', properties: { title: [['Requirements']] }, discussions: ['d-1'], content: [] }), + 'i-1': B({ id: 'i-1', type: 'image', properties: { source: [['https://s3.amazonaws.com/x/wire.png']] } }), + }, +}; + +const ctx = setPageContext(rm, 'https://notion.so/doc', { 'https://s3.amazonaws.com/x/wire.png': 'assets/doc/wire.png' }); + +// person + page mentions resolve to names, not @user / [[page]] +assert.strictEqual(rich([['‣', [['u', 'u-1']]]]), '@Tamami Sato'); +assert.strictEqual(rich([['‣', [['p', 'p-9']]]]), '[Linked Page](https://www.notion.so/p9)'); + +const out = renderBlocks(['h-1', 'i-1'], rm, 0, {}, new Set()); + +// comment thread is numbered, pulled out, and linked from the body +assert.match(out, /💬 2 comment → \[#1–#2\]\(COMMENTS_FILE#c-1\)/); +assert.strictEqual(ctx.comments.length, 1); +assert.match(ctx.comments[0], /#1 — Tamami Sato/); +assert.match(ctx.comments[0], /ty gia cuoi thang/); +assert.match(ctx.comments[0], /\?d=d1\)/); + +// heading carries a deep link back to the exact block +assert.match(out, /## Requirements \[↗\]\(https:\/\/notion\.so\/doc#h1\)/); + +// image points at the downloaded copy, not the expiring signed url +assert.match(out, /!\[\]\(assets\/doc\/wire\.png\)/); + +// a second call resets numbering instead of continuing from the last page +setPageContext(rm, 'https://notion.so/doc', {}); +assert.match(commentRef({ discussions: ['d-1'] }, 'x'), /#1–#2/); + +console.log('ok'); From b19272741ed5c6d899e011c170d8e1301b5ed331 Mon Sep 17 00:00:00 2001 From: Dam Minh Tuan Anh Date: Tue, 22 Sep 2026 16:31:43 +0700 Subject: [PATCH 2/5] fix(playwright-notion): fetch the blocks loadPageChunk only names loadPageChunk returns one level of the page and then reports an empty cursor, so every heading came out with its children missing: a real page extracted as 16 blocks and a single comment. Walk down from the root and pull the missing records with syncRecordValues - blocks, then their discussions, comments and authors. The walk is bounded to the page: a chunk also carries unrelated blocks from the sidebar, and following those crawls the whole workspace. Also make Notion's relative in-page links absolute, so they still resolve once the markdown lives outside the app, and depend on playwright-core instead of playwright - the scripts only attach over CDP, so the browser download was 500MB of nothing. Verified on a real page: 16 blocks/1 comment before, 230 blocks, 9 comments and 27 downloaded files after. --- .../playwright-notion/scripts/download.mjs | 71 ++++++++++++++++--- .../playwright-notion/scripts/export.mjs | 2 +- .../playwright-notion/scripts/package.json | 4 +- .../scripts/test-download.mjs | 3 + 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/.agents/skills/playwright-notion/scripts/download.mjs b/.agents/skills/playwright-notion/scripts/download.mjs index bf2e55f..9591fb3 100644 --- a/.agents/skills/playwright-notion/scripts/download.mjs +++ b/.agents/skills/playwright-notion/scripts/download.mjs @@ -25,29 +25,76 @@ function idFromUrl(u) { const noDash = (id) => String(id).replace(/-/g, ''); const safeName = (s) => (s || 'file').replace(/[/\\?%*:|"<>]/g, '_').replace(/\s+/g, '_').slice(0, 120); -// Fetch every block of a page (recursive chunks), run inside the browser. -// discussion/comment/notion_user come in the same recordMap - keep them. +// Fetch every block of a page, run inside the browser. +// loadPageChunk only returns one level deep and then reports an empty cursor, +// so the children it names are pulled in afterwards until nothing is missing. +// discussion/comment/notion_user ride along in the same record map. async function fetchRecordMap(page, pageId) { return page.evaluate(async (pid) => { const TABLES = ['block', 'collection', 'collection_view', 'discussion', 'comment', 'notion_user']; const merged = Object.fromEntries(TABLES.map(t => [t, {}])); + const post = (ep, body) => fetch('/api/v3/' + ep, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) + }); + const absorb = (rm) => { for (const t of TABLES) Object.assign(merged[t], rm?.[t] || {}); }; + const unwrap = (w) => w?.value?.value || w?.value || null; + let cursor = { stack: [] }; let spaceId = null; for (let i = 0; i < 40; i++) { - const body = JSON.stringify({ pageId: pid, limit: 100, cursor, chunkNumber: i, verticalColumns: false }); - const post = (ep) => fetch('/api/v3/' + ep, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body - }); - let r = await post('loadCachedPageChunkV2'); - if (r.status !== 200) r = await post('loadPageChunk'); + const body = { pageId: pid, limit: 100, cursor, chunkNumber: i, verticalColumns: false }; + let r = await post('loadCachedPageChunkV2', body); + if (r.status !== 200) r = await post('loadPageChunk', body); if (r.status !== 200) break; const j = await r.json(); - const rm = j.recordMap || {}; - for (const t of TABLES) Object.assign(merged[t], rm[t] || {}); + absorb(j.recordMap); spaceId = spaceId || j.spaceId || null; if (!j.cursor || !j.cursor.stack || j.cursor.stack.length === 0) break; cursor = j.cursor; } + spaceId = spaceId || unwrap(merged.block[pid])?.space_id || null; + + // loadPageChunk names children without returning them, so walk down from + // the root and fetch what is missing. Bounded to this page: a chunk also + // carries unrelated blocks, and following those crawls the workspace. + const fetchMissing = async (pointers) => { + for (let k = 0; k < pointers.length; k += 100) { + const r = await post('syncRecordValues', { + requests: pointers.slice(k, k + 100).map(p => ({ pointer: { ...p, spaceId }, version: -1 })) + }); + if (r.status === 200) absorb((await r.json()).recordMap); + } + }; + + const mine = new Set([pid]); + let frontier = [pid]; + for (let depth = 0; depth < 30 && frontier.length; depth++) { + await fetchMissing(frontier.filter(id => !merged.block[id]).map(id => ({ table: 'block', id }))); + const next = []; + for (const id of frontier) { + const b = unwrap(merged.block[id]); + if (!b) continue; + if (id !== pid && b.type === 'page') continue; // child pages are listed, not inlined + for (const c of b.content || []) if (!mine.has(c)) { mine.add(c); next.push(c); } + } + frontier = next; + } + + const threads = []; + for (const id of mine) for (const d of unwrap(merged.block[id])?.discussions || []) threads.push(d); + await fetchMissing(threads.filter(d => !merged.discussion[d]).map(id => ({ table: 'discussion', id }))); + + const msgs = []; + for (const d of threads) for (const c of unwrap(merged.discussion[d])?.comments || []) msgs.push(c); + await fetchMissing(msgs.filter(c => !merged.comment[c]).map(id => ({ table: 'comment', id }))); + + const people = new Set(); + for (const c of msgs) { + const by = unwrap(merged.comment[c])?.created_by_id; + if (by && !merged.notion_user[by]) people.add(by); + } + await fetchMissing([...people].map(id => ({ table: 'notion_user', id }))); + return { rm: merged, spaceId }; }, pageId); } @@ -135,6 +182,8 @@ function rich(arr) { case 'e': t = `$${f[1]}$`; break; } } + // notion serves in-page links relative; they only resolve from the app + if (link?.startsWith('/')) link = 'https://www.notion.so' + link; if (link) t = `[${t}](${ASSETS[link] || link})`; return t; }).join(''); @@ -472,7 +521,7 @@ if (!process.env.NOTION_SELFTEST) { fs.mkdirSync(OUT, { recursive: true }); log(`URLs: ${URLS.length}`); - const { chromium } = await import('playwright'); + const { chromium } = await import('playwright-core'); const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); const ctx = browser.contexts()[0]; const page = await ctx.newPage(); diff --git a/.agents/skills/playwright-notion/scripts/export.mjs b/.agents/skills/playwright-notion/scripts/export.mjs index ca749fa..aa5023d 100644 --- a/.agents/skills/playwright-notion/scripts/export.mjs +++ b/.agents/skills/playwright-notion/scripts/export.mjs @@ -5,7 +5,7 @@ // usage: node export.mjs [cdp-port] // // Read-only: enqueueTask/getTasks only produce a download; nothing in Notion is modified. -import { chromium } from 'playwright'; +import { chromium } from 'playwright-core'; import fs from 'fs'; import path from 'path'; import { execFileSync } from 'child_process'; diff --git a/.agents/skills/playwright-notion/scripts/package.json b/.agents/skills/playwright-notion/scripts/package.json index cde68d2..2f9c0a2 100644 --- a/.agents/skills/playwright-notion/scripts/package.json +++ b/.agents/skills/playwright-notion/scripts/package.json @@ -2,8 +2,8 @@ "name": "playwright-notion-scripts", "private": true, "type": "module", - "description": "Dependencies for the playwright-notion skill scripts. Run `npm install` here once.", + "description": "Dependencies for the playwright-notion skill scripts. Run `npm install` here once. playwright-core only: the scripts attach to a running browser over CDP, so there is nothing to download.", "dependencies": { - "playwright": "^1.40.0" + "playwright-core": "^1.40.0" } } diff --git a/.agents/skills/playwright-notion/scripts/test-download.mjs b/.agents/skills/playwright-notion/scripts/test-download.mjs index f22fec0..61e3a5b 100644 --- a/.agents/skills/playwright-notion/scripts/test-download.mjs +++ b/.agents/skills/playwright-notion/scripts/test-download.mjs @@ -33,6 +33,9 @@ assert.match(ctx.comments[0], /#1 — Tamami Sato/); assert.match(ctx.comments[0], /ty gia cuoi thang/); assert.match(ctx.comments[0], /\?d=d1\)/); +// relative in-page links are made absolute, or they break outside the app +assert.strictEqual(rich([['see', [['a', '/abc#def']]]]), '[see](https://www.notion.so/abc#def)'); + // heading carries a deep link back to the exact block assert.match(out, /## Requirements \[↗\]\(https:\/\/notion\.so\/doc#h1\)/); From 2e1f98f0b99f72ce856604ce3a01b2ab4cfaa5ad Mon Sep 17 00:00:00 2001 From: Dam Minh Tuan Anh Date: Tue, 22 Sep 2026 16:39:35 +0700 Subject: [PATCH 3/5] feat(playwright-notion): run in a browser profile the skill owns Attaching to the user's everyday browser meant closing it, and CDP on a branded build's default profile dir is refused outright. agent-browser.sh launches the Chrome for Testing that ships with Playwright against a profile under ~/.cache instead: the window appears once for the login, and every run after that is headless, with the everyday browser never touched. Nothing is copied, so App-Bound Encryption is not in the way - the same binary writes and reads its own cookies. start-browser.sh stays for the case where a page is only reachable from the personal profile. --- .../scripts/agent-browser.sh | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100755 .agents/skills/playwright-notion/scripts/agent-browser.sh diff --git a/.agents/skills/playwright-notion/scripts/agent-browser.sh b/.agents/skills/playwright-notion/scripts/agent-browser.sh new file mode 100755 index 0000000..c1ce351 --- /dev/null +++ b/.agents/skills/playwright-notion/scripts/agent-browser.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Launch a browser the skill owns, in its own profile, with CDP enabled. +# Nothing is copied and the user's everyday browser is never touched: you log in +# once in the window this opens, and that profile keeps the session afterwards. +# Runs headless once the profile holds a session; the window only appears for +# the first login, or when --headed is asked for to refresh an expired one. +# usage: ./agent-browser.sh [port] | --headed | --stop | --reset +set -uo pipefail + +PROFILE="${NOTION_PROFILE_DIR:-$HOME/.cache/playwright-notion-profile}" +PORT="9222" +HEADED="${NOTION_HEADED:-0}" + +case "${1:-}" in + --stop) + pkill -f -- "--user-data-dir=$PROFILE" 2>/dev/null && echo "stopped" || echo "not running" + exit 0 ;; + --reset) + # only for a session that went bad; it throws away the login + pkill -f -- "--user-data-dir=$PROFILE" 2>/dev/null; sleep 2 + rm -rf "$PROFILE" && echo "profile cleared: $PROFILE" + exit 0 ;; + --headed) HEADED=1 ;; + "") ;; + *) PORT="$1" ;; +esac + +# Chrome for Testing ships with Playwright and is built for automation, so it +# takes CDP on a custom profile dir without the refusals a branded build makes. +BIN="${NOTION_BROWSER_BIN:-}" +if [ -z "$BIN" ]; then + BIN=$(find "$HOME/Library/Caches/ms-playwright" -maxdepth 6 -type f \ + -path "*Chrome for Testing.app/Contents/MacOS/*" 2>/dev/null | sort | tail -1) +fi +[ -x "$BIN" ] || { + echo "no automation browser found. Install one with:" >&2 + echo " npx playwright install chromium" >&2 + echo "or point NOTION_BROWSER_BIN at a Chromium-family binary." >&2 + exit 1 +} + +if curl -s --max-time 3 "http://127.0.0.1:$PORT/json/version" >/dev/null 2>&1; then + echo "CDP already listening on $PORT" + exit 0 +fi + +# No profile yet means nobody has logged in, so the window has to be visible. +FIRST_RUN=0 +[ -d "$PROFILE" ] || { FIRST_RUN=1; HEADED=1; } +mkdir -p "$PROFILE" + +MODE=(--headless=new) +[ "$HEADED" = 1 ] && MODE=() + +nohup "$BIN" --remote-debugging-port="$PORT" --user-data-dir="$PROFILE" \ + "${MODE[@]}" --no-first-run --no-default-browser-check \ + >"/tmp/notion-agent-browser.log" 2>&1 & +disown + +for i in $(seq 1 20); do + if curl -s --max-time 2 "http://127.0.0.1:$PORT/json/version" >/dev/null 2>&1; then + echo "CDP ready on $PORT (profile: $PROFILE, $([ "$HEADED" = 1 ] && echo windowed || echo headless))" + [ "$FIRST_RUN" = 1 ] && echo "FIRST RUN: log in to Notion in the window that just opened, then re-run the extract." + [ "$FIRST_RUN" = 1 ] || [ "$HEADED" = 1 ] || echo "running hidden; if an extract reports the login screen, re-run with --headed and sign in again." + exit 0 + fi + sleep 1 +done + +echo "CDP did not come up on $PORT. Log tail:" >&2 +tail -5 /tmp/notion-agent-browser.log >&2 +exit 1 From a2f27f1de741cf44d8e604b82abe50ab2237746b Mon Sep 17 00:00:00 2001 From: Dam Minh Tuan Anh Date: Tue, 22 Sep 2026 16:48:57 +0700 Subject: [PATCH 4/5] feat(playwright-cdp): add Slack and GitHub, rename from playwright-notion The skill was never really about Notion: it is about pulling a requirement, its comments and its attachments out of a source that only a logged-in browser can reach. Slack and GitHub have the same problem, so they now share the skill and its output contract. doc.mjs owns that contract - comment numbering and anchors, the reference line the body carries, the attachment download - so a new source only writes the part that is actually specific to it. slack.mjs reads the web client's token and calls conversations.history / replies in the page; the DOM is useless there because Slack virtual-scrolls. Replies become the comments document. github.mjs needs no browser at all: gh already holds the credentials. It takes the description, conversation comments, review verdicts and the review comments left on lines of the diff, which are a separate API and usually where the code-level decisions are. Scripts are now named for their source, and all of them take a bare URL as well as a file of URLs. --- .agents/skills/playwright-cdp/SKILL.md | 168 ++++++++++++++ .../scripts/agent-browser.sh | 0 .agents/skills/playwright-cdp/scripts/doc.mjs | 122 ++++++++++ .../skills/playwright-cdp/scripts/github.mjs | 125 +++++++++++ .../scripts/notion-export.mjs} | 3 +- .../scripts/notion.mjs} | 139 ++++-------- .../scripts/package.json | 0 .../skills/playwright-cdp/scripts/slack.mjs | 208 ++++++++++++++++++ .../scripts/start-browser.sh | 0 .../playwright-cdp/scripts/test-doc.mjs | 91 ++++++++ .agents/skills/playwright-notion/SKILL.md | 181 --------------- .../scripts/test-download.mjs | 49 ----- .gitignore | 6 +- 13 files changed, 764 insertions(+), 328 deletions(-) create mode 100644 .agents/skills/playwright-cdp/SKILL.md rename .agents/skills/{playwright-notion => playwright-cdp}/scripts/agent-browser.sh (100%) create mode 100644 .agents/skills/playwright-cdp/scripts/doc.mjs create mode 100644 .agents/skills/playwright-cdp/scripts/github.mjs rename .agents/skills/{playwright-notion/scripts/export.mjs => playwright-cdp/scripts/notion-export.mjs} (98%) rename .agents/skills/{playwright-notion/scripts/download.mjs => playwright-cdp/scripts/notion.mjs} (81%) rename .agents/skills/{playwright-notion => playwright-cdp}/scripts/package.json (100%) create mode 100644 .agents/skills/playwright-cdp/scripts/slack.mjs rename .agents/skills/{playwright-notion => playwright-cdp}/scripts/start-browser.sh (100%) create mode 100644 .agents/skills/playwright-cdp/scripts/test-doc.mjs delete mode 100644 .agents/skills/playwright-notion/SKILL.md delete mode 100644 .agents/skills/playwright-notion/scripts/test-download.mjs diff --git a/.agents/skills/playwright-cdp/SKILL.md b/.agents/skills/playwright-cdp/SKILL.md new file mode 100644 index 0000000..710b316 --- /dev/null +++ b/.agents/skills/playwright-cdp/SKILL.md @@ -0,0 +1,168 @@ +--- +name: playwright-cdp +description: Use when pulling a requirement out of Notion, Slack or a GitHub PR/issue into local markdown — body, every comment, and the attachments — because the decisions live in the comments and the attachments expire. Covers the case where there is no API token (company workspace, no integration allowed), the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text in the body), when only the top level of a Notion page came out, or when a headless browser lands on a login screen. +--- + +# Playwright CDP + +## Overview + +Read Notion pages, Slack threads and GitHub PRs/issues into local markdown, by calling each service's own web API from inside a browser that is already logged in. + +Core principle: **do not scrape the DOM, and do not copy a browser profile.** Attach over the Chrome DevTools Protocol (CDP) to a browser this skill owns, then call the same endpoints the web app itself calls. GitHub needs no browser at all — `gh` already holds the credentials. + +Read-only. Every endpoint used fetches data or produces a download; nothing is created, edited or deleted. + +## What comes out + +The same shape for all three sources, per document: + +``` +/ + .md # body + .comments.md # every comment, numbered #1..#n, anchored + assets// # attachments, downloaded +``` + +The reason for that shape, and the thing to preserve in any new extractor: + +- **Comments are half the requirement.** Scope changes, the answer to an open question and the final formula are decided in comments, not in the body. Notion's own export drops them entirely. +- **A commented item in the body carries `> 💬 n comment → [#a–#b](.comments.md#c-a)`**, so a claim can be walked back to the comment that decided it. +- **Everything deep-links back**: Notion headings and toggles to the exact block, Slack messages and replies to their permalink, GitHub comments to their anchor. +- **Attachments are copied, not linked.** Notion and Slack hand out signed, session-bound URLs that expire — a document that only links them is empty within days. The spec is often in the PDF or the wireframe, not in the text. +- Files skipped for size or media type are named in the header. Report them; do not let them vanish. + +## Two hard-won facts + +Skipping either one wastes an hour. Both were verified by failure on macOS. + +**1. Copying a browser profile can never carry the session.** Chromium 127+ encrypts cookies with App-Bound Encryption: the key is bound to the original profile and the OS, not stored in the copied files. A copied profile shows `os_crypt: {}` in `Local State` and is served the login screen. `Default`, `Profile 1`, `Profile 2` all fail the same way. This is about copying *someone else's* profile — a profile the skill creates and owns is fine, because the same binary writes and reads its own cookies. + +**2. Scraping the rendered DOM produces wrong markdown.** Notion nests `.notion-selectable` inside `.notion-selectable`, so a selector walk emits every table once per nesting level (typically 3x) and every cell twice, sweeps the sidebar into the body, and drops all properties. Slack virtual-scrolls, so its DOM holds only a few dozen messages of a long channel. Use the APIs — that is the fix, not a better selector. + +## Step 1 — the browser + +```bash +scripts/agent-browser.sh # headless; opens a window only on first run +``` + +It launches the Chrome for Testing that ships with Playwright against a profile under `~/.cache/playwright-notion-profile`, so **the everyday browser is never touched**. The first run shows a window: sign in to Notion and Slack there once. Every run after that is headless and the session persists. + +| Need | Command | +|---|---| +| Sign in again (session expired) | `scripts/agent-browser.sh --headed` | +| Stop it | `scripts/agent-browser.sh --stop` | +| Throw the profile away (loses the login) | `scripts/agent-browser.sh --reset` | +| Check CDP is up | `curl -s http://127.0.0.1:9222/json/version` | + +No automation browser on the machine → `npx playwright install chromium`, or point `NOTION_BROWSER_BIN` at a Chromium-family binary. + +`scripts/start-browser.sh [brave|chrome|edge]` remains for the one case the owned profile cannot cover: a page reachable only from the personal profile. It **closes the user's browser** (the profile lock blocks the debug port) — say so before running it. If it reports `remote debugging requires a non-default data directory`, that build refuses CDP on its default profile dir; Brave commonly works where Chrome refuses. + +## Step 2 — install deps once + +```bash +cd scripts && npm install +``` + +`playwright-core` only (~13MB): the scripts attach to a running browser, so there is no browser to download. + +## Step 3 — extract + +Every script takes a single URL or a file of URLs, one per line. + +### Notion + +```bash +node scripts/notion-export.mjs urls.txt ./out 9222 # try first: Notion's own export +node scripts/notion.mjs urls.txt ./out 9222 # the converter +``` + +| Script | Endpoint | Gets | +|---|---|---| +| `notion-export.mjs` | `enqueueTask` (`exportBlock`) → zip | Notion's own markdown + images, formatting exactly as the page reads | +| `notion.mjs` | `loadCachedPageChunkV2` + `syncRecordValues` + `queryCollection` + `getSignedFileUrls` | body + **comments** + attachments | + +**A disabled Export button does not mean export is blocked.** In many workspaces the button is only hidden client-side by role while the server still accepts the task — that was true in the workspace this skill was built from. Test `notion-export.mjs` on one page before assuming otherwise; fall back when it reports `enqueueTask` `401`/`Unauthorized`, which means export really is disabled server-side. + +**But export drops every comment**, so when the page is a requirement, run `notion.mjs` as well and keep both. + +`notion.mjs` environment: `NOTION_MAX_MB` (30), `NOTION_MEDIA=1` (also fetch video/audio), `NOTION_NO_ASSETS=1` (link instead of download), `NOTION_RAW=1` (dump `.raw.json` — use it when comments come out `0` but the UI shows some). `NOTION_RECURSIVE=1` on `notion-export.mjs` exports each page's subtree; leave it off, on a database page it pulls the whole table. + +### Slack + +```bash +node scripts/slack.mjs 'https://.slack.com/archives/C0.../p1712345678901234' ./out 9222 +``` + +| URL form | Gets | +|---|---| +| `.../archives//p` | that message and its whole thread | +| `.../archives/` | the last `SLACK_LIMIT` messages (default 200) and each one's thread | +| `app.slack.com/client///thread/-` | same as the first form | + +Replies are comments: the body keeps the top-level messages, `comments.md` holds every reply. Environment: `SLACK_LIMIT`, `SLACK_MAX_MB`, `SLACK_MEDIA=1`. + +The web client's token lives only on the `app.slack.com` origin — an `/archives/` link is a stub page that redirects to the desktop app, so the script hops origins by itself. `no localConfig_v2` / `no token for team` means that profile is not signed in to Slack: `agent-browser.sh --headed`, sign in, retry. + +### GitHub + +```bash +node scripts/github.mjs https://github.com///pull/123 ./out +``` + +No browser and no CDP port — it shells out to `gh`, so `gh auth login` must have been run. Pulls the description, conversation comments, review verdicts **and review comments left on a line of the diff** (a separate API, and usually where the code-level decisions are), all numbered in time order and grouped by where they were left. + +## Step 4 — verify + +Never report success from an exit code. Each script prints its counts per document: + +``` +OK 9 comments 27 files /.md +``` + +- `notion.mjs` also logs to `/tmp/notion_dl.log` (`NOTION_DL_LOG` to override); `notion-export.mjs` to `/tmp/notion_export.log`. Count `OK` lines against the URL count and report any `FAIL`/`GAVE UP`. +- A page that reports `0 comments` while the UI clearly shows some is a finding, not a pass. Re-run with `NOTION_RAW=1` and check whether `discussion` in the dump is empty (really none) or populated (a renderer bug). +- A Notion page that comes out with headings but no content under them is the one-level bug — see the mistakes table. + +```bash +node scripts/test-doc.mjs # self-check for the renderers, needs no browser +``` + +## Common mistakes + +| Mistake | What happens | Fix | +|---|---|---| +| Treating an export as the whole requirement | Every comment is missing — that is where the spec gets decided | Also run `notion.mjs` | +| Assuming a greyed-out Export button means export is blocked | Skips the best path for no reason | Test `notion-export.mjs` on one page | +| Trusting one `loadPageChunk` call | It returns one level and then reports an empty cursor: a real page came out as 16 blocks and 1 comment instead of 230 and 9 | Walk down from the root with `syncRecordValues` (`notion.mjs` does) | +| Following every block in the chunk | A chunk also carries unrelated sidebar blocks; following those crawls the workspace and never finishes | Bound the walk to the page | +| Keeping Notion/Slack file URLs instead of the files | Signed URLs expire; the doc is empty days later | The scripts save them under `assets/` | +| Copying a browser profile to a temp dir | Login screen, every time | Use the profile the skill owns | +| Scraping `.notion-selectable` / the Slack DOM | Tables 3x, cells 2x, sidebar in body; Slack yields a few dozen messages | Use the APIs | +| `open -a "Brave Browser" --args --remote-debugging-port=9222` | Flags silently dropped, port never opens | Exec the binary path directly (the scripts do) | +| Leaving a browser running when launching it with CDP | Profile lock blocks the port, with no error at the port | `agent-browser.sh` uses its own profile and sidesteps this | +| `waitUntil: 'networkidle'` | 30s timeout — Notion and Slack sync continuously | `domcontentloaded` + a short fixed wait | +| Reusing one tab for 30+ Notion pages | Renderer crashes; every later page fails | `notion.mjs` recycles the tab every 5 pages and retries 3x | +| Plain `unzip` on the export zip | `Illegal byte sequence`, Japanese/Vietnamese names destroyed | Decode cp437→utf-8 (`notion-export.mjs` does) | +| Scraping the sidebar for the URL list | Gets database views, not the wanted rows | Ask the user for explicit URLs | +| Piping a long run through `tail` | Output buffered, progress invisible | Log to a file, `grep` it | + +## Red flags — stop and re-read this skill + +- About to hand over an export as "the requirement" without its comments. +- About to copy `Cookies`, `Local State` or a whole profile folder. +- About to write a `querySelector` walk over Notion blocks or Slack messages. +- About to skip `notion-export.mjs` because the UI hides the Export button. +- About to accept a Notion page whose headings have no content under them. +- About to report "downloaded everything" without reading the counts. + +## Adding a source + +Keep the output contract: import `commentBook`, `writeDoc` and `saveAsset` from `scripts/doc.mjs`, which own the numbering, the anchors, the reference lines and the attachment handling. `slack.mjs` is the shortest example to copy. Reach for the DOM only when the service has no API worth calling, and say so in the header of what it produces. + +## Warn the user before starting + +- These are internal company documents being copied to a local disk. Whether that fits their company policy is their call, not something to assume. +- While a run is active the browser listens on a local debug port, so any local process can drive it. `agent-browser.sh --stop` when done. +- `start-browser.sh` (the fallback path only) closes their everyday browser; unsaved work in it is at risk. diff --git a/.agents/skills/playwright-notion/scripts/agent-browser.sh b/.agents/skills/playwright-cdp/scripts/agent-browser.sh similarity index 100% rename from .agents/skills/playwright-notion/scripts/agent-browser.sh rename to .agents/skills/playwright-cdp/scripts/agent-browser.sh diff --git a/.agents/skills/playwright-cdp/scripts/doc.mjs b/.agents/skills/playwright-cdp/scripts/doc.mjs new file mode 100644 index 0000000..c884490 --- /dev/null +++ b/.agents/skills/playwright-cdp/scripts/doc.mjs @@ -0,0 +1,122 @@ +// Output contract shared by every source: a body document, a numbered +// comments document beside it, and downloaded attachments. The point is +// traceability - each claim in the body can be walked back to the comment or +// the file it came from, and from there to the original page. +import fs from 'fs'; +import path from 'path'; + +export const safeName = (s) => + (s || 'file').replace(/[/\\?%*:|"<>]/g, '_').replace(/\s+/g, '_').slice(0, 120); + +export const docName = (s, fallback) => + (s || '').replace(/[/\\?%*:|"<>]/g, '-').replace(/\s+/g, ' ').trim().slice(0, 120) || fallback; + +export const stamp = (t) => + t ? new Date(t).toISOString().replace('T', ' ').slice(0, 16) : ''; + +// The body is written before its comments file has a name, so references are +// laid down as this placeholder and patched in at write time. +const PLACEHOLDER = 'COMMENTS_FILE'; + +// Collects comment threads in document order, numbering them #1..#n so the +// body can point at an exact comment and the reader can find it. +export function commentBook() { + const docs = []; + let n = 0; + return { + get count() { return n; }, + get docs() { return docs; }, + // items: [{ who, when, body, link }] -> the reference line for the body + thread({ on, link, items, note }) { + const real = items.filter(Boolean); + if (!real.length) return ''; + const nums = []; + const rendered = real.map((it) => { + const k = ++n; + nums.push(k); + const head = `#${k} — ${it.who}${it.when ? ` · ${it.when}` : ''}${it.link ? ` · [↗](${it.link})` : ''}`; + return `### ${head}\n\n${it.body || '_(empty)_'}`; + }); + docs.push([ + `## On: ${on || '(top)'}${note ? ` · _${note}_` : ''}${link ? ` · [↗](${link})` : ''}`, + '', rendered.join('\n\n'), '' + ].join('\n')); + const label = nums.length === 1 ? `#${nums[0]}` : `#${nums[0]}–#${nums[nums.length - 1]}`; + return `💬 ${nums.length} comment → [${label}](${PLACEHOLDER}#c-${nums[0]})`; + } + }; +} + +// Writes .md and, when there are any, .comments.md next to it. +export function writeDoc({ out, name, title, source, meta = [], body, book, suffix = [] }) { + fs.mkdirSync(out, { recursive: true }); + const commentsName = `${name}.comments.md`; + const patch = (s) => s.split(PLACEHOLDER).join(encodeURI(commentsName)); + const count = book ? book.count : 0; + + const md = [ + `# ${title}`, + '', + `> Source: ${source}`, + ...meta.map((m) => `> ${m}`), + ...(count ? [`> 💬 ${count} comments in [${commentsName}](${encodeURI(commentsName)}) — every commented item below links into it.`] : []), + '', + body, + ...suffix + ].join('\n'); + + let file = path.join(out, `${name}.md`); + let n = 2; + while (fs.existsSync(file)) file = path.join(out, `${name} (${n++}).md`); + fs.writeFileSync(file, patch(md), 'utf8'); + + if (count) { + fs.writeFileSync(path.join(out, commentsName), patch([ + `# Comments — ${title}`, + '', + `> Source: ${source}`, + `> ${count} comments · body: [${name}.md](${encodeURI(name + '.md')})`, + '', + ...book.docs + ].join('\n')), 'utf8'); + } + return { file, comments: count ? path.join(out, commentsName) : null }; +} + +const MEDIA = /\.(mov|mp4|m4v|avi|webm|mkv|mp3|wav|m4a)$/i; + +// Attachments are copied rather than linked: the URLs these services hand out +// are signed or session-bound, so a document that only links them goes empty. +export async function saveAsset(request, url, dir, opts = {}) { + const { name, maxMb = 30, media = false, headers } = opts; + const raw = name || decodeURIComponent((url.split('?')[0].split('/').pop() || 'file').replace(/^.*:/, '')); + if (!media && MEDIA.test(raw)) return { skipped: `${raw} (media, set *_MEDIA=1)` }; + try { + const resp = await request.get(url, { timeout: 180000, ...(headers ? { headers } : {}) }); + if (!resp.ok()) throw new Error('HTTP ' + resp.status()); + const buf = await resp.body(); + if (buf.length > maxMb * 1024 * 1024) { + return { skipped: `${raw} (${(buf.length / 1048576).toFixed(1)}MB > limit)` }; + } + fs.mkdirSync(dir, { recursive: true }); + let file = path.join(dir, safeName(raw)); + if (!path.extname(file)) file += '.bin'; + let n = 2; + while (fs.existsSync(file) && fs.statSync(file).size !== buf.length) { + file = path.join(dir, `${n++}_${safeName(raw)}`); + } + fs.writeFileSync(file, buf); + return { file, name: raw }; + } catch (e) { + return { skipped: `${raw} (${e.message})` }; + } +} + +// Sources take either a file of URLs or a single URL, so a one-off link does +// not need a file made for it. +export function readUrls(arg) { + if (arg && fs.existsSync(arg) && fs.statSync(arg).isFile()) { + return fs.readFileSync(arg, 'utf8').split('\n').map((s) => s.trim()).filter(Boolean); + } + return arg ? [arg] : []; +} diff --git a/.agents/skills/playwright-cdp/scripts/github.mjs b/.agents/skills/playwright-cdp/scripts/github.mjs new file mode 100644 index 0000000..76354b1 --- /dev/null +++ b/.agents/skills/playwright-cdp/scripts/github.mjs @@ -0,0 +1,125 @@ +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { commentBook, writeDoc, docName, readUrls, stamp, safeName } from './doc.mjs'; + +// usage: node github.mjs +// No browser: gh already holds the credentials, so this runs anywhere. +const URLS = readUrls(process.argv[2]); +const OUT = process.argv[3] || './github-docs'; +const MAX_MB = Number(process.env.GH_MAX_MB || 30); + +const gh = (args) => JSON.parse(execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })); + +function parseUrl(u) { + const m = u.match(/github\.com\/([^/]+)\/([^/]+)\/(pull|issues)\/(\d+)/); + if (!m) throw new Error('not a PR or issue URL: ' + u); + return { repo: `${m[1]}/${m[2]}`, kind: m[3] === 'pull' ? 'pr' : 'issue', number: m[4] }; +} + +// Attachments are links in the markdown; private ones need the gh token. +function saveAttachments(md, dir, source) { + const urls = [...md.matchAll(/!?\[[^\]]*\]\((https:\/\/[^)\s]*(?:user-images\.githubusercontent\.com|github\.com\/user-attachments\/assets)[^)\s]*)\)/g)] + .map((m) => m[1]); + const map = {}; + const skipped = []; + for (const url of new Set(urls)) { + const name = safeName(decodeURIComponent(url.split('?')[0].split('/').pop() || 'file')); + const file = path.join(dir, name); + try { + fs.mkdirSync(dir, { recursive: true }); + execFileSync('gh', ['api', '--method', 'GET', url.replace('https://github.com/', ''), '--cache', '0'], + { stdio: 'ignore' }); + } catch { /* gh api cannot serve these; fall back to curl with the token */ } + try { + const token = execFileSync('gh', ['auth', 'token'], { encoding: 'utf8' }).trim(); + execFileSync('curl', ['-sL', '--max-filesize', String(MAX_MB * 1024 * 1024), + '-H', `Authorization: Bearer ${token}`, '-o', file, url], { stdio: 'ignore' }); + if (fs.existsSync(file) && fs.statSync(file).size > 0) { + map[url] = path.relative(OUT, file).split(path.sep).join('/'); + } else { + fs.rmSync(file, { force: true }); + skipped.push(name); + } + } catch { skipped.push(name); } + } + let out = md; + for (const [url, local] of Object.entries(map)) out = out.split(url).join(local); + return { md: out, saved: Object.keys(map).length, skipped, source }; +} + +function extract(url) { + const { repo, kind, number } = parseUrl(url); + const fields = kind === 'pr' + ? 'title,body,author,createdAt,state,comments,reviews,files,baseRefName,headRefName' + : 'title,body,author,createdAt,state,comments,labels'; + const d = gh([kind, 'view', number, '--repo', repo, '--json', fields]); + + // Comments left on a line of the diff are a separate API and are usually + // where the code-level decisions get made. + const lineComments = kind === 'pr' + ? gh(['api', `repos/${repo}/pulls/${number}/comments`, '--paginate']).map((c) => ({ + at: c.created_at, who: c.user?.login || 'unknown', link: c.html_url, body: c.body, + on: `${c.path}${c.line ? ':' + c.line : ''}` + })) + : []; + + const name = docName(`${repo.split('/')[1]} ${kind} ${number} ${d.title}`, `${kind}-${number}`); + const assetDir = path.join(OUT, 'assets', name); + const book = commentBook(); + + // A review carries a verdict plus its own line comments; both decide scope, + // so they sit in the same numbered stream as plain comments, in time order. + const entries = [ + ...(d.comments || []).map((c) => ({ + at: c.createdAt, who: c.author?.login || 'unknown', link: c.url, body: c.body, on: 'Conversation' + })), + ...(d.reviews || []).map((r) => ({ + at: r.submittedAt, who: r.author?.login || 'unknown', link: r.url, + body: [r.state && `**${r.state}**`, r.body].filter(Boolean).join('\n\n'), on: 'Review' + })) + , ...lineComments + ].filter((e) => e.body || e.on === 'Review').sort((a, b) => String(a.at).localeCompare(String(b.at))); + + // grouped by where they were left, each group kept in time order + const refs = {}; + for (const group of [...new Set(entries.map((e) => e.on))]) { + const items = entries.filter((e) => e.on === group); + refs[group] = book.thread({ + on: group, items: items.map((e) => ({ who: e.who, when: stamp(e.at), link: e.link, body: e.body })) + }); + } + + const changed = (d.files || []).map((f) => `- \`${f.path}\` (+${f.additions} −${f.deletions})`); + const body = [ + ...(kind === 'pr' ? [`**${d.state}** · \`${d.headRefName}\` → \`${d.baseRefName}\` · by ${d.author?.login}`, ''] : []), + ...(kind === 'issue' ? [`**${d.state}** · by ${d.author?.login} · ${(d.labels || []).map((l) => l.name).join(', ')}`, ''] : []), + d.body || '_(no description)_', + ...(Object.values(refs).length ? ['', ...Object.values(refs).map((r) => `> ${r}`)] : []), + ...(changed.length ? ['', '## Changed files', '', ...changed] : []) + ].join('\n'); + + const withAssets = saveAttachments(body, assetDir, url); + const commentsWithAssets = book.docs.map((doc) => saveAttachments(doc, assetDir, url)); + book.docs.length = 0; + book.docs.push(...commentsWithAssets.map((c) => c.md)); + const saved = withAssets.saved + commentsWithAssets.reduce((n, c) => n + c.saved, 0); + const skipped = [...withAssets.skipped, ...commentsWithAssets.flatMap((c) => c.skipped)]; + + const { file } = writeDoc({ + out: OUT, name, title: `${d.title} (${repo}#${number})`, source: url, book, + meta: [ + `Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${saved} files`, + ...(skipped.length ? [`⚠️ not downloaded: ${skipped.join(', ')}`] : []) + ], + body: withAssets.md + }); + console.log(`OK ${book.count} comments ${saved} files ${file}`); +} + +if (!URLS.length) { console.error('usage: node github.mjs '); process.exit(1); } +try { execFileSync('gh', ['auth', 'status'], { stdio: 'ignore' }); } +catch { console.error('gh is not authenticated: run `gh auth login`'); process.exit(1); } +for (const url of URLS) { + try { extract(url); } catch (e) { console.error(`FAIL ${url}: ${e.message}`); } +} diff --git a/.agents/skills/playwright-notion/scripts/export.mjs b/.agents/skills/playwright-cdp/scripts/notion-export.mjs similarity index 98% rename from .agents/skills/playwright-notion/scripts/export.mjs rename to .agents/skills/playwright-cdp/scripts/notion-export.mjs index aa5023d..c96b679 100644 --- a/.agents/skills/playwright-notion/scripts/export.mjs +++ b/.agents/skills/playwright-cdp/scripts/notion-export.mjs @@ -8,6 +8,7 @@ import { chromium } from 'playwright-core'; import fs from 'fs'; import path from 'path'; +import { readUrls } from './doc.mjs'; import { execFileSync } from 'child_process'; const URLS_FILE = process.argv[2] || './urls.txt'; @@ -19,7 +20,7 @@ const RECURSIVE = process.env.NOTION_RECURSIVE === '1'; const log = (m) => { fs.appendFileSync(LOG, m + '\n'); console.log(m); }; -const URLS = fs.readFileSync(URLS_FILE, 'utf8').split('\n').map(s => s.trim()).filter(Boolean); +const URLS = readUrls(URLS_FILE); function dashId(hex) { return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20,32)}`; diff --git a/.agents/skills/playwright-notion/scripts/download.mjs b/.agents/skills/playwright-cdp/scripts/notion.mjs similarity index 81% rename from .agents/skills/playwright-notion/scripts/download.mjs rename to .agents/skills/playwright-cdp/scripts/notion.mjs index 9591fb3..f5ae0ed 100644 --- a/.agents/skills/playwright-notion/scripts/download.mjs +++ b/.agents/skills/playwright-cdp/scripts/notion.mjs @@ -1,5 +1,6 @@ import fs from 'fs'; import path from 'path'; +import { commentBook, writeDoc, saveAsset, docName, stamp, readUrls } from './doc.mjs'; // usage: node download.mjs [cdp-port] const URLS_FILE = process.argv[2] || './urls.txt'; @@ -23,7 +24,6 @@ function idFromUrl(u) { return m ? dashId(m[1]) : null; } const noDash = (id) => String(id).replace(/-/g, ''); -const safeName = (s) => (s || 'file').replace(/[/\\?%*:|"<>]/g, '_').replace(/\s+/g, '_').slice(0, 120); // Fetch every block of a page, run inside the browser. // loadPageChunk only returns one level deep and then reports an empty cursor, @@ -133,14 +133,14 @@ const unwrap = (w) => w?.value?.value || w?.value || null; let RM = null; // record map, so rich() can resolve users and page mentions let BASE = ''; // page url without hash, for [↗] deep links let ASSETS = {}; // original file url -> local relative path -let COMMENTS = []; // markdown chunks for .comments.md +let BOOK = null; // shared comment numbering, see doc.mjs let COMMENT_SEEN = new Set(); -let CNUM = 0; function setPageContext(rm, baseUrl, assets = {}) { RM = rm; BASE = baseUrl; ASSETS = assets; - COMMENTS = []; COMMENT_SEEN = new Set(); CNUM = 0; - return { comments: COMMENTS }; + COMMENT_SEEN = new Set(); + BOOK = commentBook(); + return BOOK; } const userName = (id) => { @@ -153,7 +153,6 @@ const pageMention = (id) => { const t = p ? rich(p.properties?.title) : ''; return `[${t || 'page'}](https://www.notion.so/${noDash(id)})`; }; -const stamp = (t) => (t ? new Date(t).toISOString().replace('T', ' ').slice(0, 16) : ''); // Notion rich text array -> markdown inline function rich(arr) { @@ -191,32 +190,31 @@ function rich(arr) { const propText = (v) => Array.isArray(v) ? rich(v) : (v == null ? '' : String(v)); -// Number every comment thread hanging off a block, stash the markdown for -// comments.md, and return the inline reference line for the body document. +// Hand each comment thread on a block to the shared book, which numbers it and +// returns the reference line the body carries. function commentRef(b, anchorText) { - const nums = []; + const refs = []; for (const did of b.discussions || []) { const d = unwrap(RM.discussion?.[did]); - if (!d || !(d.comments || []).length) continue; - const items = []; - for (const cid of d.comments) { + if (!d) continue; + const items = (d.comments || []).map((cid) => { const c = unwrap(RM.comment?.[cid]); - if (!c || COMMENT_SEEN.has(cid)) continue; + if (!c || COMMENT_SEEN.has(cid)) return null; COMMENT_SEEN.add(cid); - const n = ++CNUM; - nums.push(n); - const who = userName(c.created_by_id || c.created_by?.id); - items.push(`### #${n} — ${who} · ${stamp(c.created_time)}\n\n${rich(c.text) || '_(empty)_'}`); - } - if (!items.length) continue; - COMMENTS.push([ - `## On: ${anchorText || '(page)'}${d.resolved ? ' · _resolved_' : ''} · [↗](${BASE}?d=${noDash(did)})`, - '', items.join('\n\n'), '' - ].join('\n')); + return { + who: userName(c.created_by_id || c.created_by?.id), + when: stamp(c.created_time), + body: rich(c.text) + }; + }); + const ref = BOOK.thread({ + on: anchorText, items, + note: d.resolved ? 'resolved' : '', + link: `${BASE}?d=${noDash(did)}` + }); + if (ref) refs.push(ref); } - if (!nums.length) return ''; - const label = nums.length === 1 ? `#${nums[0]}` : `#${nums[0]}–#${nums[nums.length - 1]}`; - return `💬 ${nums.length} comment → [${label}](COMMENTS_FILE#c-${nums[0]})`; + return refs.join(' · '); } function renderBlocks(ids, rm, depth, collections, seen) { @@ -382,36 +380,14 @@ async function downloadAssets(page, rm, dir) { const map = {}; const skipped = []; let saved = 0; - fs.mkdirSync(dir, { recursive: true }); - for (let i = 0; i < refs.length; i++) { const { source } = refs[i]; const url = signed[i] || (/^https?:/.test(source) ? source : null); if (!url) continue; - const rawName = decodeURIComponent((source.split('?')[0].split('/').pop() || 'file').replace(/^.*:/, '')); - if (!WITH_MEDIA && /\.(mov|mp4|m4v|avi|webm|mkv|mp3|wav|m4a)$/i.test(rawName)) { - skipped.push(`${rawName} (media, set NOTION_MEDIA=1)`); - continue; - } - try { - const resp = await page.context().request.get(url, { timeout: 180000 }); - if (!resp.ok()) throw new Error('HTTP ' + resp.status()); - const buf = await resp.body(); - if (buf.length > MAX_MB * 1024 * 1024) { - skipped.push(`${rawName} (${(buf.length / 1048576).toFixed(1)}MB > NOTION_MAX_MB)`); - continue; - } - let name = safeName(rawName); - if (!path.extname(name)) name += '.bin'; - let file = path.join(dir, name); - let n = 2; - while (fs.existsSync(file) && fs.statSync(file).size !== buf.length) file = path.join(dir, `${n++}_${name}`); - fs.writeFileSync(file, buf); - map[source] = path.relative(OUT, file).split(path.sep).join('/'); - saved++; - } catch (e) { - skipped.push(`${rawName} (${e.message})`); - } + const r = await saveAsset(page.context().request, url, dir, { maxMb: MAX_MB, media: WITH_MEDIA }); + if (r.skipped) { skipped.push(r.skipped); continue; } + map[source] = path.relative(OUT, r.file).split(path.sep).join('/'); + saved++; } return { map, saved, skipped }; } @@ -427,15 +403,10 @@ async function downloadPage(page, url, idx, total) { const root = unwrap(rm.block[pid]); if (!root) { log(`[${idx}/${total}] FAIL no root block: ${url}`); return; } - RM = rm; - BASE = url.split('#')[0]; - ASSETS = {}; - COMMENTS = []; - COMMENT_SEEN = new Set(); - CNUM = 0; + setPageContext(rm, url.split('#')[0]); const title = rich(root.properties?.title) || pid; - const safe = title.replace(/[/\\?%*:|"<>]/g, '-').replace(/\s+/g, ' ').trim().slice(0, 120) || pid; + const safe = docName(title, pid); // Resolve any embedded collection views into markdown tables const collections = {}; @@ -474,49 +445,29 @@ async function downloadPage(page, url, idx, total) { const pageRef = commentRef(root, title); const body = renderBlocks(root.content, rm, 0, collections, new Set([pid])); - // name is only known now, so comment references are patched in at the end - const commentsName = `${safe}.comments.md`; - const fix = (s) => s.split('COMMENTS_FILE').join(encodeURI(commentsName)); - - const md = [ - `# ${title}`, - '', - `> Source: ${url}`, - `> Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${Object.keys(rm.block).length} blocks · ${assets.saved} files`, - ...(CNUM ? [`> 💬 ${CNUM} comments in [${commentsName}](${encodeURI(commentsName)}) — every commented block below links into it.`] : []), - ...(assets.skipped.length ? [`> ⚠️ not downloaded: ${assets.skipped.join(', ')}`] : []), - '', - ...(pageRef ? [`> ${pageRef}`, ''] : []), - ...(propLines.length ? ['## Properties', '', ...propLines, ''] : []), - body - ].join('\n'); - - let file = path.join(OUT, `${safe}.md`); - let n = 2; - while (fs.existsSync(file)) { file = path.join(OUT, `${safe} (${n++}).md`); } - fs.writeFileSync(file, fix(md), 'utf8'); - - if (CNUM) { - const cmd = [ - `# Comments — ${title}`, - '', - `> Source: ${url}`, - `> ${CNUM} comments · body: [${safe}.md](${encodeURI(safe + '.md')})`, - '', - ...COMMENTS - ].join('\n'); - fs.writeFileSync(path.join(OUT, commentsName), fix(cmd), 'utf8'); - } + const { file } = writeDoc({ + out: OUT, name: safe, title, source: url, book: BOOK, + meta: [ + `Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${Object.keys(rm.block).length} blocks · ${assets.saved} files`, + ...(assets.skipped.length ? [`⚠️ not downloaded: ${assets.skipped.join(', ')}`] : []) + ], + body: [ + ...(pageRef ? [`> ${pageRef}`, ''] : []), + ...(propLines.length ? ['## Properties', '', ...propLines, ''] : []), + body + ].join('\n') + }); + if (WITH_RAW) fs.writeFileSync(path.join(OUT, `${safe}.raw.json`), JSON.stringify(rm), 'utf8'); - log(`[${idx}/${total}] OK ${Math.round(md.length/1024)}KB ${CNUM} comments ${assets.saved} files ${file}`); + log(`[${idx}/${total}] OK ${BOOK.count} comments ${assets.saved} files ${file}`); } export { rich, renderBlocks, commentRef, setPageContext }; // skipped when imported by the self-check if (!process.env.NOTION_SELFTEST) { - const URLS = fs.readFileSync(URLS_FILE, 'utf8').split('\n').map(s => s.trim()).filter(Boolean); + const URLS = readUrls(URLS_FILE); fs.writeFileSync(LOG, ''); fs.mkdirSync(OUT, { recursive: true }); log(`URLs: ${URLS.length}`); diff --git a/.agents/skills/playwright-notion/scripts/package.json b/.agents/skills/playwright-cdp/scripts/package.json similarity index 100% rename from .agents/skills/playwright-notion/scripts/package.json rename to .agents/skills/playwright-cdp/scripts/package.json diff --git a/.agents/skills/playwright-cdp/scripts/slack.mjs b/.agents/skills/playwright-cdp/scripts/slack.mjs new file mode 100644 index 0000000..dbbd7da --- /dev/null +++ b/.agents/skills/playwright-cdp/scripts/slack.mjs @@ -0,0 +1,208 @@ +import path from 'path'; +import { commentBook, writeDoc, saveAsset, docName, readUrls } from './doc.mjs'; + +// usage: node slack.mjs [cdp-port] +// Accepts a channel link, a message permalink, or a thread link. +const URLS = readUrls(process.argv[2]); +const OUT = process.argv[3] || './slack-docs'; +const PORT = process.argv[4] || '9222'; +const LIMIT = Number(process.env.SLACK_LIMIT || 200); +const MAX_MB = Number(process.env.SLACK_MAX_MB || 30); +const WITH_MEDIA = process.env.SLACK_MEDIA === '1'; + +// channel id and, when the link points at one message, its timestamp +function parseUrl(u) { + const url = new URL(u); + let channel = null, ts = null; + const arch = url.pathname.match(/\/archives\/([A-Z0-9]+)(?:\/p(\d{10})(\d{6}))?/i); + if (arch) { channel = arch[1]; if (arch[2]) ts = `${arch[2]}.${arch[3]}`; } + const thread = url.pathname.match(/\/client\/[A-Z0-9]+\/([A-Z0-9]+)(?:\/thread\/[A-Z0-9]+-(\d+\.\d+))?/i); + if (thread) { channel = channel || thread[1]; ts = ts || thread[2] || null; } + ts = url.searchParams.get('thread_ts') || ts; + channel = url.searchParams.get('cid') || channel; + if (!channel) throw new Error('no channel id in URL: ' + u); + return { channel, ts }; +} + +const when = (ts) => new Date(Number(String(ts).split('.')[0]) * 1000).toISOString().replace('T', ' ').slice(0, 16); + +// The web client's own token, which the in-page API calls need alongside the +// session cookie. It only exists on the app.slack.com origin - a /archives/ +// link is a stub page that redirects to the desktop app. +async function readToken(page) { + const read = () => page.evaluate(() => { + const raw = localStorage.getItem('localConfig_v2'); + if (!raw) return { error: 'no localConfig_v2 (not signed in to Slack in this profile?)' }; + const cfg = JSON.parse(raw); + const teams = cfg.teams || {}; + const fromUrl = (location.pathname.match(/\/client\/(T[A-Z0-9]+)/i) || [])[1]; + const id = (fromUrl && teams[fromUrl] && fromUrl) || cfg.lastActiveTeamId || Object.keys(teams)[0]; + const team = teams[id]; + if (!team?.token) return { error: 'no token for team ' + id }; + return { token: team.token, domain: team.domain, name: team.name }; + }); + let cfg = await read(); + if (cfg.error && !page.url().startsWith('https://app.slack.com/')) { + await page.goto('https://app.slack.com/client', { waitUntil: 'domcontentloaded', timeout: 90000 }); + await page.waitForTimeout(10000); + cfg = await read(); + } + return cfg; +} + +// Slack virtual-scrolls, so the DOM only ever holds a few dozen messages. +// These calls run in the page: same origin, session cookie attached. +async function fetchConversation(page, { token, channel, ts, limit }) { + return page.evaluate(async ({ token, channel, ts, limit }) => { + const api = async (method, params) => { + const fd = new FormData(); + fd.append('token', token); + for (const [k, v] of Object.entries(params)) fd.append(k, String(v)); + const r = await fetch('/api/' + method, { method: 'POST', body: fd, credentials: 'include' }); + const j = await r.json(); + if (!j.ok) throw new Error(method + ' -> ' + (j.error || 'unknown')); + return j; + }; + + const info = await api('conversations.info', { channel }).catch((e) => ({ error: String(e.message) })); + let messages = []; + const threads = {}; + + if (ts) { + const r = await api('conversations.replies', { channel, ts, limit: 1000 }); + messages = r.messages.slice(0, 1); + threads[ts] = r.messages.slice(1); + } else { + const h = await api('conversations.history', { channel, limit }); + messages = (h.messages || []).slice().reverse(); // oldest first + let pulled = 0; + for (const m of messages) { + if (!m.reply_count || pulled >= 60) continue; + pulled++; + try { + const r = await api('conversations.replies', { channel, ts: m.ts, limit: 1000 }); + threads[m.ts] = r.messages.slice(1); + } catch (e) { + threads[m.ts] = [{ text: `_(thread unavailable: ${e.message})_`, user: '' }]; + } + } + } + + const ids = new Set(); + const scan = (m) => { + if (m.user) ids.add(m.user); + (m.text || '').replace(/<@([UW][A-Z0-9]+)>/g, (_, u) => ids.add(u)); + }; + messages.forEach(scan); + Object.values(threads).flat().forEach(scan); + + const users = {}; + for (const id of ids) { + try { + const u = await api('users.info', { user: id }); + users[id] = u.user.profile.display_name || u.user.profile.real_name || u.user.name; + } catch { users[id] = id; } + } + return { info: info.channel || null, infoError: info.error || null, messages, threads, users }; + }, { token, channel, ts, limit }); +} + +async function extract(page, url, out) { + const { channel, ts } = parseUrl(url); + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(6000); + + const cfg = await readToken(page); + if (cfg.error) throw new Error(cfg.error + ' — run agent-browser.sh --headed and sign in to Slack'); + + const host = `https://${cfg.domain}.slack.com`; + if (!page.url().startsWith(host)) { + await page.goto(`${host}/archives/${channel}`, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(5000); + } + + const data = await fetchConversation(page, { token: cfg.token, channel, ts, limit: LIMIT }); + const name = docName(`slack ${data.info?.name || channel}${ts ? ' ' + ts : ''}`, channel); + const title = `#${data.info?.name || channel}${ts ? ` — thread ${when(ts)}` : ` — last ${LIMIT} messages`}`; + + // files live behind the session; the token goes in as a bearer header + const files = {}; + const skipped = []; + for (const m of [...data.messages, ...Object.values(data.threads).flat()]) { + for (const f of m.files || []) { + const src = f.url_private_download || f.url_private; + if (!src || files[f.id]) continue; + const r = await saveAsset(page.context().request, src, path.join(out, 'assets', name), { + name: f.name || f.id, maxMb: MAX_MB, media: WITH_MEDIA, + headers: { Authorization: 'Bearer ' + cfg.token } + }); + if (r.skipped) { skipped.push(r.skipped); continue; } + files[f.id] = { name: r.name, local: path.relative(out, r.file).split(path.sep).join('/') }; + } + } + + const who = (id) => data.users[id] || id || 'unknown'; + const mrkdwn = (t) => (t || '') + .replace(/<@([UW][A-Z0-9]+)>/g, (_, u) => '@' + who(u)) + .replace(/<#([CG][A-Z0-9]+)\|([^>]*)>/g, (_, c, n) => '#' + (n || c)) + .replace(//g, '@$1') + .replace(/<(https?:\/\/[^|>]+)\|([^>]+)>/g, '[$2]($1)') + .replace(/<(https?:\/\/[^>]+)>/g, '$1') + .replace(/>/g, '>').replace(/</g, '<').replace(/&/g, '&') + .replace(/\*([^*\n]+)\*/g, '**$1**'); + + const attached = (m) => (m.files || []).map((f) => { + const hit = files[f.id]; + if (!hit) return `📎 ${f.name || f.id} _(not downloaded)_`; + return /\.(png|jpe?g|gif|webp)$/i.test(hit.name) ? `![${hit.name}](${hit.local})` : `📎 [${hit.name}](${hit.local})`; + }).join('\n'); + + const permalink = (mts, parent) => + `${host}/archives/${channel}/p${String(mts).replace('.', '')}` + + (parent && parent !== mts ? `?thread_ts=${parent}&cid=${channel}` : ''); + + const book = commentBook(); + const refs = {}; + for (const [parent, replies] of Object.entries(data.threads)) { + const root = data.messages.find((m) => m.ts === parent); + refs[parent] = book.thread({ + on: root ? `${who(root.user)} · ${when(parent)} — ${mrkdwn(root.text).split('\n')[0].slice(0, 120)}` : when(parent), + link: permalink(parent), + items: replies.map((r) => ({ + who: who(r.user), when: when(r.ts), link: permalink(r.ts, parent), + body: [mrkdwn(r.text), attached(r)].filter(Boolean).join('\n\n') + })) + }); + } + + const body = data.messages.map((m) => [ + `### ${who(m.user)} · ${when(m.ts)} [↗](${permalink(m.ts)})`, '', + mrkdwn(m.text), + ...(attached(m) ? ['', attached(m)] : []), + ...((m.reactions || []).length ? ['', m.reactions.map((r) => `:${r.name}: ${r.count}`).join(' ')] : []), + ...(refs[m.ts] ? ['', `> ${refs[m.ts]}`] : []) + ].join('\n')).join('\n\n---\n\n'); + + const { file } = writeDoc({ + out, name, title, source: url, book, + meta: [ + `Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${data.messages.length} messages · ${Object.keys(files).length} files`, + ...(data.infoError ? [`⚠️ channel info unavailable: ${data.infoError}`] : []), + ...(skipped.length ? [`⚠️ not downloaded: ${skipped.join(', ')}`] : []) + ], + body + }); + console.log(`OK ${data.messages.length} messages ${book.count} replies ${Object.keys(files).length} files ${file}`); +} + +if (!URLS.length) { console.error('usage: node slack.mjs [cdp-port]'); process.exit(1); } +const { chromium } = await import('playwright-core'); +const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); +const ctx = browser.contexts()[0]; +for (const url of URLS) { + const page = await ctx.newPage(); + try { await extract(page, url, OUT); } + catch (e) { console.error(`FAIL ${url}: ${e.message}`); } + finally { await page.close().catch(() => {}); } +} +process.exit(0); diff --git a/.agents/skills/playwright-notion/scripts/start-browser.sh b/.agents/skills/playwright-cdp/scripts/start-browser.sh similarity index 100% rename from .agents/skills/playwright-notion/scripts/start-browser.sh rename to .agents/skills/playwright-cdp/scripts/start-browser.sh diff --git a/.agents/skills/playwright-cdp/scripts/test-doc.mjs b/.agents/skills/playwright-cdp/scripts/test-doc.mjs new file mode 100644 index 0000000..9906fec --- /dev/null +++ b/.agents/skills/playwright-cdp/scripts/test-doc.mjs @@ -0,0 +1,91 @@ +// self-check for the pure renderers: node test-download.mjs +process.env.NOTION_SELFTEST = '1'; +const { rich, renderBlocks, commentRef, setPageContext } = await import('./notion.mjs'); +import assert from 'assert'; + +const B = (v) => ({ value: v }); +const rm = { + notion_user: { 'u-1': B({ id: 'u-1', name: 'Tamami Sato' }) }, + discussion: { 'd-1': B({ id: 'd-1', comments: ['c-1', 'c-2'], resolved: false }) }, + comment: { + 'c-1': B({ id: 'c-1', created_by_id: 'u-1', created_time: 1700000000000, text: [['spec chot: dung ty gia cuoi thang']] }), + 'c-2': B({ id: 'c-2', created_by_id: 'u-1', created_time: 1700000100000, text: [['ok']] }), + }, + block: { + 'p-9': B({ id: 'p-9', type: 'page', properties: { title: [['Linked Page']] } }), + 'h-1': B({ id: 'h-1', type: 'header', properties: { title: [['Requirements']] }, discussions: ['d-1'], content: [] }), + 'i-1': B({ id: 'i-1', type: 'image', properties: { source: [['https://s3.amazonaws.com/x/wire.png']] } }), + }, +}; + +const book = setPageContext(rm, 'https://notion.so/doc', { 'https://s3.amazonaws.com/x/wire.png': 'assets/doc/wire.png' }); + +// person + page mentions resolve to names, not @user / [[page]] +assert.strictEqual(rich([['‣', [['u', 'u-1']]]]), '@Tamami Sato'); +assert.strictEqual(rich([['‣', [['p', 'p-9']]]]), '[Linked Page](https://www.notion.so/p9)'); + +const out = renderBlocks(['h-1', 'i-1'], rm, 0, {}, new Set()); + +// comment thread is numbered, pulled out, and linked from the body +assert.match(out, /💬 2 comment → \[#1–#2\]\(COMMENTS_FILE#c-1\)/); +assert.strictEqual(book.count, 2); +assert.strictEqual(book.docs.length, 1); +assert.match(book.docs[0], /#1 — Tamami Sato/); +assert.match(book.docs[0], /ty gia cuoi thang/); +assert.match(book.docs[0], /\?d=d1\)/); + +// relative in-page links are made absolute, or they break outside the app +assert.strictEqual(rich([['see', [['a', '/abc#def']]]]), '[see](https://www.notion.so/abc#def)'); + +// heading carries a deep link back to the exact block +assert.match(out, /## Requirements \[↗\]\(https:\/\/notion\.so\/doc#h1\)/); + +// image points at the downloaded copy, not the expiring signed url +assert.match(out, /!\[\]\(assets\/doc\/wire\.png\)/); + +// a second call resets numbering instead of continuing from the last page +setPageContext(rm, 'https://notion.so/doc', {}); +assert.match(commentRef({ discussions: ['d-1'] }, 'x'), /#1–#2/); + +console.log('ok'); + +// --- shared output contract (doc.mjs), used by all three sources --- +import os from 'os'; +import fsp from 'fs'; +import pathp from 'path'; +const { commentBook, writeDoc } = await import('./doc.mjs'); + +const dir = fsp.mkdtempSync(pathp.join(os.tmpdir(), 'doc-test-')); +const bk = commentBook(); +const r1 = bk.thread({ on: 'Section A', link: 'https://x/#a', items: [ + { who: 'ann', when: '2026-01-01 10:00', body: 'first' }, + { who: 'bo', when: '2026-01-01 11:00', body: 'second', link: 'https://x/#c2' }, +] }); +const r2 = bk.thread({ on: 'Section B', items: [{ who: 'ann', body: 'third' }] }); + +// numbering continues across threads, so a reference names one exact comment +assert.strictEqual(r1, '💬 2 comment → [#1–#2](COMMENTS_FILE#c-1)'); +assert.strictEqual(r2, '💬 1 comment → [#3](COMMENTS_FILE#c-3)'); +assert.strictEqual(bk.count, 3); + +// an empty thread produces no reference and no section +assert.strictEqual(bk.thread({ on: 'C', items: [null, undefined] }), ''); +assert.strictEqual(bk.docs.length, 2); + +const w = writeDoc({ out: dir, name: 'Doc', title: 'Doc', source: 'https://x', book: bk, body: r1 }); +const body = fsp.readFileSync(w.file, 'utf8'); +const comments = fsp.readFileSync(w.comments, 'utf8'); + +// the placeholder is resolved to the real comments file, in both documents +assert.ok(!body.includes('COMMENTS_FILE') && !comments.includes('COMMENTS_FILE')); +assert.match(body, /\(Doc\.comments\.md#c-1\)/); +assert.match(comments, /<\/a>#3 — ann/); +assert.match(body, /💬 3 comments in \[Doc\.comments\.md\]/); + +// no comments means no stray empty comments file +const w2 = writeDoc({ out: dir, name: 'Bare', title: 'Bare', source: 'https://x', book: commentBook(), body: 'x' }); +assert.strictEqual(w2.comments, null); +assert.ok(!fsp.existsSync(pathp.join(dir, 'Bare.comments.md'))); + +fsp.rmSync(dir, { recursive: true, force: true }); +console.log('ok (doc)'); diff --git a/.agents/skills/playwright-notion/SKILL.md b/.agents/skills/playwright-notion/SKILL.md deleted file mode 100644 index a43ed04..0000000 --- a/.agents/skills/playwright-notion/SKILL.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -name: playwright-notion -description: Use when downloading or reading Notion pages without an API token — the workspace is company-owned, there is no integration secret, the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a Notion scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text mixed into content), when a headless browser lands on the Notion login screen, or when a page's comments and attachments are needed alongside its body because that is where the spec was actually decided. ---- - -# Playwright Notion - -## Overview - -Read Notion pages through a browser that is already logged in, by calling Notion's own internal web API from inside that browser tab. - -Core principle: **do not scrape the DOM, and do not copy the browser profile.** Attach to a running browser over the Chrome DevTools Protocol (CDP), then call the same endpoints the Notion web app itself calls. - -Read-only. The endpoints used only fetch data or produce a download; nothing in Notion is created, edited, or deleted. - -## When to use - -- No Notion API token available (company workspace, no integration allowed). -- UI Export is disabled/greyed out or absent because the account is read-only or guest. -- MCP Notion server is not an option; only browser access exists. -- A previous scrape returned garbage markdown: duplicated tables, doubled cells, sidebar links inside the body, lost properties. -- A Playwright/Puppeteer script keeps landing on Notion's login page even though the user is logged in normally. - -**Do NOT use when** an API token or working MCP Notion connection exists — use those, they are simpler and supported. - -## Two hard-won facts that dictate the approach - -Skipping either one wastes an hour. Both were verified by failure on macOS. - -**1. Copying the browser profile can never carry the session.** Chromium 127+ encrypts cookies with App-Bound Encryption: the key is bound to the original profile and the OS, not stored in the copied files. A copied profile shows `os_crypt: {}` in `Local State`, cannot decrypt the original cookies, and Notion serves the login screen. Copying `Cookies` + `Local State` + `Preferences` from `Default`, `Profile 1`, and `Profile 2` all fail the same way. Do not try it — attach to the real running browser instead. - -**2. Scraping the rendered DOM produces wrong markdown.** Notion nests `.notion-selectable` inside `.notion-selectable`, so a selector-based walk emits every table once per nesting level (typically 3x) and every cell twice. It also sweeps the sidebar navigation into the page body and drops all page properties. Use the API instead — that is the fix, not a better selector. - -## Two scripts: try export first - -| Script | Endpoint | Output | Use | -|---|---|---|---| -| `scripts/export.mjs` | `enqueueTask` (`exportBlock`) → zip | Notion's own markdown + images downloaded | **Try this first** | -| `scripts/download.mjs` | `loadCachedPageChunkV2` + `queryCollection` + `getSignedFileUrls` | markdown + `comments.md` + downloaded `assets/` | Fallback if export is blocked, **and whenever comments matter** | - -**A disabled Export button does not mean export is blocked.** In many workspaces the button is only hidden client-side by role, while the server still accepts the export task. That was true in the case this skill was built from: the UI offered no Export, yet `enqueueTask` returned `200` and produced a proper zip. So always test `export.mjs` on one page before falling back. - -Native export is better where it works: it is Notion's own renderer, so tables, nested content and formatting come out exactly as the page reads. - -**But export drops every comment.** In most workspaces the decisions — a scope change, the answer to an open question, the final formula — live in comments, not in the body. When the page is a requirement and the comments matter, run `download.mjs` as well and keep both: export for the body, `download.mjs` for `.comments.md`. - -Fall back to `download.mjs` when `export.mjs` reports an `enqueueTask` `401`/`Unauthorized` — that means the workspace really did disable export server-side (an Enterprise setting). `download.mjs` still works there, because it uses nothing more than the read access the browser already has. - -## Workflow - -**Step 1 — start the browser with CDP enabled.** - -```bash -scripts/start-browser.sh brave 9222 # or: chrome | edge -``` - -The script lists available profiles, closes any running instance (its profile lock blocks the debug port), relaunches detached against the **real** profile dir, and waits for the port. It is idempotent — if CDP already listens it exits immediately. - -This closes the user's browser windows. Say so before running it, and re-run it whenever a later step reports a connection refusal. - -If it reports `remote debugging requires a non-default data directory`, that browser build refuses CDP on its default profile dir. Try another browser — **Brave commonly works where Chrome refuses**. - -**Step 2 — install script deps once.** - -```bash -cd scripts && npm install -``` - -**Step 3 — collect the target URLs.** One per line in a plain text file. Any Notion URL form works; both scripts extract the 32-hex page id themselves, so `?v=...&source=copy_link` query strings can stay. - -``` -https://app.notion.com/p/883f894a7fd44a1b9bfa0d6af0ff4a28 -https://app.notion.com/p/RQ-01-31136f82f94b4a16adb8b434404db850 -``` - -Ask the user for the list. Do not guess links — sidebar `` scraping returns the navigation menu (a dozen or so database views), not the rows the user means. - -**Step 4 — smoke-test one URL.** Run the chosen script against a single-URL file and open the output. A whole batch against a logged-out browser produces a directory of useless files. - -```bash -node scripts/export.mjs one-url.txt ./out 9222 -``` - -If it fails with `Unauthorized`, switch to `scripts/download.mjs` and smoke-test that instead. - -**Step 5 — run the full list.** - -```bash -node scripts/export.mjs urls.txt ./notion-docs 9222 # preferred -node scripts/download.mjs urls.txt ./notion-docs 9222 # fallback -``` - -`NOTION_RECURSIVE=1` on `export.mjs` exports each page's subtree as well. Leave it off by default — on a database page it pulls the entire table. - -`download.mjs` environment variables: - -| Variable | Default | Effect | -|---|---|---| -| `NOTION_MAX_MB` | `30` | skip attachments larger than this | -| `NOTION_MEDIA` | *(off)* | `=1` also downloads video/audio (usually huge) | -| `NOTION_NO_ASSETS` | *(off)* | `=1` keeps the original URLs and downloads nothing | -| `NOTION_RAW` | *(off)* | `=1` also dumps `.raw.json` — use it when comments come out `0` but the UI shows some | - -**Step 6 — verify.** Count `OK` lines against the URL count and report any `FAIL`/`GAVE UP` line. Never report success from an exit code alone. - -```bash -grep -c OK /tmp/notion_export.log # export.mjs -grep -E "FAIL|GAVE UP" /tmp/notion_export.log -``` - -`download.mjs` logs `N comments M files` per page. If a page shows `0 comments` while the Notion UI clearly has some, say so — do not let it pass silently. Re-run with `NOTION_RAW=1` and check whether `discussion` in the dump is empty (really none) or populated (a renderer bug). - -## Quick reference - -| Need | Command | -|---|---| -| Start/verify CDP | `scripts/start-browser.sh brave 9222` | -| Check CDP is up | `curl -s http://127.0.0.1:9222/json/version` | -| Install deps | `cd scripts && npm install` | -| Native export (preferred) | `node scripts/export.mjs urls.txt ./out 9222` | -| Converter (fallback) | `node scripts/download.mjs urls.txt ./out 9222` | -| Check results | `grep -c OK /tmp/notion_export.log` | -| Self-check the converter | `node scripts/test-download.mjs` | - -Logs default to `/tmp/notion_export.log` and `/tmp/notion_dl.log`; override with `NOTION_DL_LOG`. - -## What the output contains - -**`export.mjs`** — Notion's own export, unpacked: one `.md` per page under a workspace-named folder, plus a sibling folder of downloaded images per page. Properties appear as a plain key/value block at the top, mentions and relations resolved to names and URLs. - -**`download.mjs`** — per page: - -``` -/ - .md # body - .comments.md # every comment, numbered #1..#n, anchored - assets// # attachments downloaded from that page -``` - -The body holds title, source URL, an extraction header (block/file/comment counts), a `## Properties` section, headings, nested lists, to-do checkboxes, toggles, quotes, callouts as `> [!NOTE]`, code blocks with language tags, equations as `$$`, dividers, and tables with correct columns (inline tables and embedded database views via `queryCollection`). - -Traceability — the reason to prefer this over a plain scrape: - -- a commented block carries `> 💬 n comment → [#a–#b](.comments.md#c-a)`; -- each heading and toggle carries `[↗](#)`, opening that exact block in Notion; -- each comment thread carries `[↗](?d=)`, opening that exact thread; -- person mentions resolve to real names and page mentions to titles plus URLs, from the same recordMap. - -Attachments are downloaded rather than linked, because Notion's file URLs are signed and expire — a document that only links them is empty within days. Files skipped for size or media type are named in the header; report them rather than letting them vanish. - -## Common mistakes - -| Mistake | What happens | Fix | -|---|---|---| -| Assuming a greyed-out Export button means export is blocked | Skips the best path for no reason | Test `export.mjs` on one page first | -| Copying the browser profile to a temp dir | Login screen, every time | Attach to the running browser over CDP | -| Scraping `.notion-selectable` / DOM | Tables 3x, cells 2x, sidebar in body, no properties | Use the API scripts | -| `open -a "Brave Browser" --args --remote-debugging-port=9222` | Flags silently dropped, port never opens | Exec the binary path directly (the script does) | -| Leaving the browser running when launching with CDP | Profile lock blocks the port, with no error at the port | Close it first (the script does) | -| `waitUntil: 'networkidle'` | 30s timeout — Notion syncs continuously | `domcontentloaded` + a short fixed wait | -| Reusing one tab for 30+ pages | Renderer crashes; every later page fails | Both scripts recycle the tab every 5 pages and retry 3x | -| Plain `unzip` on the export zip | `Illegal byte sequence`, Japanese/Vietnamese names destroyed | Decode cp437→utf-8 (`export.mjs` does) | -| Scraping the sidebar for the URL list | Gets database views, not the wanted rows | Ask the user for explicit URLs | -| Treating `export.mjs` output as the whole requirement | Every comment is missing — that is where the spec gets decided | Also run `download.mjs` for `comments.md` | -| Keeping Notion's file URLs instead of the files | Signed URLs expire; the doc is empty a few days later | `download.mjs` saves them under `assets/` | -| Piping a long run through `tail` | Output buffered, progress invisible | Log to a file, `grep` it | - -## Red flags — stop and re-read this skill - -- About to copy `Cookies`, `Local State`, or a whole profile folder → will not work, see fact 1. -- About to write a `querySelector` walk over Notion blocks → will produce duplicates, see fact 2. -- About to skip `export.mjs` because the UI hides the Export button → test it anyway. -- About to launch a browser with `open -a ... --args` → flags get dropped. -- About to report "downloaded all pages" without grepping the log → verify first. -- About to hand over an export as "the requirement" without its comments → the decisions are in the comments. - -## Warn the user before starting - -- Their browser will be closed and relaunched; unsaved work in it is at risk. -- While the run is active the browser listens on a local debug port, so any local process can drive it. Restart the browser normally afterwards. -- These are internal company documents being copied to a local disk. Whether that fits their company policy is their call to make, not something to assume. diff --git a/.agents/skills/playwright-notion/scripts/test-download.mjs b/.agents/skills/playwright-notion/scripts/test-download.mjs deleted file mode 100644 index 61e3a5b..0000000 --- a/.agents/skills/playwright-notion/scripts/test-download.mjs +++ /dev/null @@ -1,49 +0,0 @@ -// self-check for the pure renderers: node test-download.mjs -process.env.NOTION_SELFTEST = '1'; -const { rich, renderBlocks, commentRef, setPageContext } = await import('./download.mjs'); -import assert from 'assert'; - -const B = (v) => ({ value: v }); -const rm = { - notion_user: { 'u-1': B({ id: 'u-1', name: 'Tamami Sato' }) }, - discussion: { 'd-1': B({ id: 'd-1', comments: ['c-1', 'c-2'], resolved: false }) }, - comment: { - 'c-1': B({ id: 'c-1', created_by_id: 'u-1', created_time: 1700000000000, text: [['spec chot: dung ty gia cuoi thang']] }), - 'c-2': B({ id: 'c-2', created_by_id: 'u-1', created_time: 1700000100000, text: [['ok']] }), - }, - block: { - 'p-9': B({ id: 'p-9', type: 'page', properties: { title: [['Linked Page']] } }), - 'h-1': B({ id: 'h-1', type: 'header', properties: { title: [['Requirements']] }, discussions: ['d-1'], content: [] }), - 'i-1': B({ id: 'i-1', type: 'image', properties: { source: [['https://s3.amazonaws.com/x/wire.png']] } }), - }, -}; - -const ctx = setPageContext(rm, 'https://notion.so/doc', { 'https://s3.amazonaws.com/x/wire.png': 'assets/doc/wire.png' }); - -// person + page mentions resolve to names, not @user / [[page]] -assert.strictEqual(rich([['‣', [['u', 'u-1']]]]), '@Tamami Sato'); -assert.strictEqual(rich([['‣', [['p', 'p-9']]]]), '[Linked Page](https://www.notion.so/p9)'); - -const out = renderBlocks(['h-1', 'i-1'], rm, 0, {}, new Set()); - -// comment thread is numbered, pulled out, and linked from the body -assert.match(out, /💬 2 comment → \[#1–#2\]\(COMMENTS_FILE#c-1\)/); -assert.strictEqual(ctx.comments.length, 1); -assert.match(ctx.comments[0], /#1 — Tamami Sato/); -assert.match(ctx.comments[0], /ty gia cuoi thang/); -assert.match(ctx.comments[0], /\?d=d1\)/); - -// relative in-page links are made absolute, or they break outside the app -assert.strictEqual(rich([['see', [['a', '/abc#def']]]]), '[see](https://www.notion.so/abc#def)'); - -// heading carries a deep link back to the exact block -assert.match(out, /## Requirements \[↗\]\(https:\/\/notion\.so\/doc#h1\)/); - -// image points at the downloaded copy, not the expiring signed url -assert.match(out, /!\[\]\(assets\/doc\/wire\.png\)/); - -// a second call resets numbering instead of continuing from the last page -setPageContext(rm, 'https://notion.so/doc', {}); -assert.match(commentRef({ discussions: ['d-1'] }, 'x'), /#1–#2/); - -console.log('ok'); diff --git a/.gitignore b/.gitignore index 63464f8..781758e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .code-review-graph/ .DS_Store -# playwright-notion skill: local deps + downloaded docs -.agents/skills/playwright-notion/scripts/node_modules/ -.agents/skills/playwright-notion/scripts/package-lock.json +# playwright-cdp skill: local deps + downloaded docs +.agents/skills/playwright-cdp/scripts/node_modules/ +.agents/skills/playwright-cdp/scripts/package-lock.json __pycache__/ From 1bb4e86ba65f8502d3ba75a0b3a74bbb6b0eb302 Mon Sep 17 00:00:00 2001 From: Dam Minh Tuan Anh Date: Tue, 22 Sep 2026 17:02:56 +0700 Subject: [PATCH 5/5] fix(playwright-cdp): attach when no tab is open, drop the github script Closing the last window leaves the browser process alive with an empty target list, and connectOverCDP then fails with "Browser context management is not supported", which reads like a version problem rather than an empty browser. connectCdp opens a target first. The github script only ever wrapped `gh pr view --json` and the review comments API, which the agent can call directly, so it earned nothing by being a script. The skill says which calls to use instead. Also fix agent-browser.sh --headed: an empty array expansion aborts under `set -u` on the bash 3.2 that ships with macOS, so asking for a window was the one path that could not launch. Slack verified end to end on a real thread: message plus 5 replies, user names and mentions resolved, permalinks carrying thread_ts. --- .agents/skills/playwright-cdp/SKILL.md | 21 ++- .../playwright-cdp/scripts/agent-browser.sh | 8 +- .agents/skills/playwright-cdp/scripts/doc.mjs | 16 +++ .../skills/playwright-cdp/scripts/github.mjs | 125 ------------------ .../skills/playwright-cdp/scripts/notion.mjs | 5 +- .../skills/playwright-cdp/scripts/slack.mjs | 5 +- 6 files changed, 33 insertions(+), 147 deletions(-) delete mode 100644 .agents/skills/playwright-cdp/scripts/github.mjs diff --git a/.agents/skills/playwright-cdp/SKILL.md b/.agents/skills/playwright-cdp/SKILL.md index 710b316..f2edf7e 100644 --- a/.agents/skills/playwright-cdp/SKILL.md +++ b/.agents/skills/playwright-cdp/SKILL.md @@ -1,21 +1,23 @@ --- name: playwright-cdp -description: Use when pulling a requirement out of Notion, Slack or a GitHub PR/issue into local markdown — body, every comment, and the attachments — because the decisions live in the comments and the attachments expire. Covers the case where there is no API token (company workspace, no integration allowed), the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text in the body), when only the top level of a Notion page came out, or when a headless browser lands on a login screen. +description: Use when pulling a requirement out of Notion or Slack into local markdown — body, every comment, and the attachments — because the decisions live in the comments and the attachments expire. Covers the case where there is no API token (company workspace, no integration allowed), the UI Export button is disabled or missing by permission, and access exists only through a logged-in browser. Also use when a scrape produced wrong markdown (tables repeated, cells duplicated, sidebar text in the body), when only the top level of a Notion page came out, or when a headless browser lands on a login screen. --- # Playwright CDP ## Overview -Read Notion pages, Slack threads and GitHub PRs/issues into local markdown, by calling each service's own web API from inside a browser that is already logged in. +Read Notion pages and Slack threads into local markdown, by calling each service's own web API from inside a browser that is already logged in. -Core principle: **do not scrape the DOM, and do not copy a browser profile.** Attach over the Chrome DevTools Protocol (CDP) to a browser this skill owns, then call the same endpoints the web app itself calls. GitHub needs no browser at all — `gh` already holds the credentials. +Core principle: **do not scrape the DOM, and do not copy a browser profile.** Attach over the Chrome DevTools Protocol (CDP) to a browser this skill owns, then call the same endpoints the web app itself calls. + +For GitHub there is nothing to build: `gh pr view --json body,comments,reviews` and `gh api repos//pulls//comments` already return everything, with credentials `gh` holds. Use those directly. Read-only. Every endpoint used fetches data or produces a download; nothing is created, edited or deleted. ## What comes out -The same shape for all three sources, per document: +The same shape for both sources, per document: ``` / @@ -28,7 +30,7 @@ The reason for that shape, and the thing to preserve in any new extractor: - **Comments are half the requirement.** Scope changes, the answer to an open question and the final formula are decided in comments, not in the body. Notion's own export drops them entirely. - **A commented item in the body carries `> 💬 n comment → [#a–#b](.comments.md#c-a)`**, so a claim can be walked back to the comment that decided it. -- **Everything deep-links back**: Notion headings and toggles to the exact block, Slack messages and replies to their permalink, GitHub comments to their anchor. +- **Everything deep-links back**: Notion headings and toggles to the exact block, Slack messages and replies to their permalink. - **Attachments are copied, not linked.** Notion and Slack hand out signed, session-bound URLs that expire — a document that only links them is empty within days. The spec is often in the PDF or the wireframe, not in the text. - Files skipped for size or media type are named in the header. Report them; do not let them vanish. @@ -105,14 +107,6 @@ Replies are comments: the body keeps the top-level messages, `comments.md` holds The web client's token lives only on the `app.slack.com` origin — an `/archives/` link is a stub page that redirects to the desktop app, so the script hops origins by itself. `no localConfig_v2` / `no token for team` means that profile is not signed in to Slack: `agent-browser.sh --headed`, sign in, retry. -### GitHub - -```bash -node scripts/github.mjs https://github.com///pull/123 ./out -``` - -No browser and no CDP port — it shells out to `gh`, so `gh auth login` must have been run. Pulls the description, conversation comments, review verdicts **and review comments left on a line of the diff** (a separate API, and usually where the code-level decisions are), all numbered in time order and grouped by where they were left. - ## Step 4 — verify Never report success from an exit code. Each script prints its counts per document: @@ -143,6 +137,7 @@ node scripts/test-doc.mjs # self-check for the renderers, needs no browse | `open -a "Brave Browser" --args --remote-debugging-port=9222` | Flags silently dropped, port never opens | Exec the binary path directly (the scripts do) | | Leaving a browser running when launching it with CDP | Profile lock blocks the port, with no error at the port | `agent-browser.sh` uses its own profile and sidesteps this | | `waitUntil: 'networkidle'` | 30s timeout — Notion and Slack sync continuously | `domcontentloaded` + a short fixed wait | +| Attaching while the browser has no tab open | `connectOverCDP` fails with `Browser context management is not supported` | The scripts open a target first; a closed window is not a dead browser | | Reusing one tab for 30+ Notion pages | Renderer crashes; every later page fails | `notion.mjs` recycles the tab every 5 pages and retries 3x | | Plain `unzip` on the export zip | `Illegal byte sequence`, Japanese/Vietnamese names destroyed | Decode cp437→utf-8 (`notion-export.mjs` does) | | Scraping the sidebar for the URL list | Gets database views, not the wanted rows | Ask the user for explicit URLs | diff --git a/.agents/skills/playwright-cdp/scripts/agent-browser.sh b/.agents/skills/playwright-cdp/scripts/agent-browser.sh index c1ce351..a49893d 100755 --- a/.agents/skills/playwright-cdp/scripts/agent-browser.sh +++ b/.agents/skills/playwright-cdp/scripts/agent-browser.sh @@ -49,11 +49,13 @@ FIRST_RUN=0 [ -d "$PROFILE" ] || { FIRST_RUN=1; HEADED=1; } mkdir -p "$PROFILE" -MODE=(--headless=new) -[ "$HEADED" = 1 ] && MODE=() +# a plain string, not an array: bash 3.2 (the macOS default) errors on an +# empty array expansion under `set -u` +MODE="--headless=new" +[ "$HEADED" = 1 ] && MODE="" nohup "$BIN" --remote-debugging-port="$PORT" --user-data-dir="$PROFILE" \ - "${MODE[@]}" --no-first-run --no-default-browser-check \ + $MODE --no-first-run --no-default-browser-check \ >"/tmp/notion-agent-browser.log" 2>&1 & disown diff --git a/.agents/skills/playwright-cdp/scripts/doc.mjs b/.agents/skills/playwright-cdp/scripts/doc.mjs index c884490..4e36320 100644 --- a/.agents/skills/playwright-cdp/scripts/doc.mjs +++ b/.agents/skills/playwright-cdp/scripts/doc.mjs @@ -120,3 +120,19 @@ export function readUrls(arg) { } return arg ? [arg] : []; } + +// Attaching fails outright when the browser has no page open - closing the last +// window leaves the process alive with an empty target list - so make one first. +export async function connectCdp(port) { + const base = `http://127.0.0.1:${port}`; + try { + const targets = await (await fetch(`${base}/json/list`)).json(); + if (!targets.some((t) => t.type === 'page')) { + await fetch(`${base}/json/new?about:blank`, { method: 'PUT' }); + } + } catch { + throw new Error(`no browser on ${port}: run scripts/agent-browser.sh`); + } + const { chromium } = await import('playwright-core'); + return chromium.connectOverCDP(base); +} diff --git a/.agents/skills/playwright-cdp/scripts/github.mjs b/.agents/skills/playwright-cdp/scripts/github.mjs deleted file mode 100644 index 76354b1..0000000 --- a/.agents/skills/playwright-cdp/scripts/github.mjs +++ /dev/null @@ -1,125 +0,0 @@ -import { execFileSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; -import { commentBook, writeDoc, docName, readUrls, stamp, safeName } from './doc.mjs'; - -// usage: node github.mjs -// No browser: gh already holds the credentials, so this runs anywhere. -const URLS = readUrls(process.argv[2]); -const OUT = process.argv[3] || './github-docs'; -const MAX_MB = Number(process.env.GH_MAX_MB || 30); - -const gh = (args) => JSON.parse(execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })); - -function parseUrl(u) { - const m = u.match(/github\.com\/([^/]+)\/([^/]+)\/(pull|issues)\/(\d+)/); - if (!m) throw new Error('not a PR or issue URL: ' + u); - return { repo: `${m[1]}/${m[2]}`, kind: m[3] === 'pull' ? 'pr' : 'issue', number: m[4] }; -} - -// Attachments are links in the markdown; private ones need the gh token. -function saveAttachments(md, dir, source) { - const urls = [...md.matchAll(/!?\[[^\]]*\]\((https:\/\/[^)\s]*(?:user-images\.githubusercontent\.com|github\.com\/user-attachments\/assets)[^)\s]*)\)/g)] - .map((m) => m[1]); - const map = {}; - const skipped = []; - for (const url of new Set(urls)) { - const name = safeName(decodeURIComponent(url.split('?')[0].split('/').pop() || 'file')); - const file = path.join(dir, name); - try { - fs.mkdirSync(dir, { recursive: true }); - execFileSync('gh', ['api', '--method', 'GET', url.replace('https://github.com/', ''), '--cache', '0'], - { stdio: 'ignore' }); - } catch { /* gh api cannot serve these; fall back to curl with the token */ } - try { - const token = execFileSync('gh', ['auth', 'token'], { encoding: 'utf8' }).trim(); - execFileSync('curl', ['-sL', '--max-filesize', String(MAX_MB * 1024 * 1024), - '-H', `Authorization: Bearer ${token}`, '-o', file, url], { stdio: 'ignore' }); - if (fs.existsSync(file) && fs.statSync(file).size > 0) { - map[url] = path.relative(OUT, file).split(path.sep).join('/'); - } else { - fs.rmSync(file, { force: true }); - skipped.push(name); - } - } catch { skipped.push(name); } - } - let out = md; - for (const [url, local] of Object.entries(map)) out = out.split(url).join(local); - return { md: out, saved: Object.keys(map).length, skipped, source }; -} - -function extract(url) { - const { repo, kind, number } = parseUrl(url); - const fields = kind === 'pr' - ? 'title,body,author,createdAt,state,comments,reviews,files,baseRefName,headRefName' - : 'title,body,author,createdAt,state,comments,labels'; - const d = gh([kind, 'view', number, '--repo', repo, '--json', fields]); - - // Comments left on a line of the diff are a separate API and are usually - // where the code-level decisions get made. - const lineComments = kind === 'pr' - ? gh(['api', `repos/${repo}/pulls/${number}/comments`, '--paginate']).map((c) => ({ - at: c.created_at, who: c.user?.login || 'unknown', link: c.html_url, body: c.body, - on: `${c.path}${c.line ? ':' + c.line : ''}` - })) - : []; - - const name = docName(`${repo.split('/')[1]} ${kind} ${number} ${d.title}`, `${kind}-${number}`); - const assetDir = path.join(OUT, 'assets', name); - const book = commentBook(); - - // A review carries a verdict plus its own line comments; both decide scope, - // so they sit in the same numbered stream as plain comments, in time order. - const entries = [ - ...(d.comments || []).map((c) => ({ - at: c.createdAt, who: c.author?.login || 'unknown', link: c.url, body: c.body, on: 'Conversation' - })), - ...(d.reviews || []).map((r) => ({ - at: r.submittedAt, who: r.author?.login || 'unknown', link: r.url, - body: [r.state && `**${r.state}**`, r.body].filter(Boolean).join('\n\n'), on: 'Review' - })) - , ...lineComments - ].filter((e) => e.body || e.on === 'Review').sort((a, b) => String(a.at).localeCompare(String(b.at))); - - // grouped by where they were left, each group kept in time order - const refs = {}; - for (const group of [...new Set(entries.map((e) => e.on))]) { - const items = entries.filter((e) => e.on === group); - refs[group] = book.thread({ - on: group, items: items.map((e) => ({ who: e.who, when: stamp(e.at), link: e.link, body: e.body })) - }); - } - - const changed = (d.files || []).map((f) => `- \`${f.path}\` (+${f.additions} −${f.deletions})`); - const body = [ - ...(kind === 'pr' ? [`**${d.state}** · \`${d.headRefName}\` → \`${d.baseRefName}\` · by ${d.author?.login}`, ''] : []), - ...(kind === 'issue' ? [`**${d.state}** · by ${d.author?.login} · ${(d.labels || []).map((l) => l.name).join(', ')}`, ''] : []), - d.body || '_(no description)_', - ...(Object.values(refs).length ? ['', ...Object.values(refs).map((r) => `> ${r}`)] : []), - ...(changed.length ? ['', '## Changed files', '', ...changed] : []) - ].join('\n'); - - const withAssets = saveAttachments(body, assetDir, url); - const commentsWithAssets = book.docs.map((doc) => saveAttachments(doc, assetDir, url)); - book.docs.length = 0; - book.docs.push(...commentsWithAssets.map((c) => c.md)); - const saved = withAssets.saved + commentsWithAssets.reduce((n, c) => n + c.saved, 0); - const skipped = [...withAssets.skipped, ...commentsWithAssets.flatMap((c) => c.skipped)]; - - const { file } = writeDoc({ - out: OUT, name, title: `${d.title} (${repo}#${number})`, source: url, book, - meta: [ - `Extracted: ${new Date().toISOString().slice(0, 16).replace('T', ' ')} UTC · ${saved} files`, - ...(skipped.length ? [`⚠️ not downloaded: ${skipped.join(', ')}`] : []) - ], - body: withAssets.md - }); - console.log(`OK ${book.count} comments ${saved} files ${file}`); -} - -if (!URLS.length) { console.error('usage: node github.mjs '); process.exit(1); } -try { execFileSync('gh', ['auth', 'status'], { stdio: 'ignore' }); } -catch { console.error('gh is not authenticated: run `gh auth login`'); process.exit(1); } -for (const url of URLS) { - try { extract(url); } catch (e) { console.error(`FAIL ${url}: ${e.message}`); } -} diff --git a/.agents/skills/playwright-cdp/scripts/notion.mjs b/.agents/skills/playwright-cdp/scripts/notion.mjs index f5ae0ed..8c8b696 100644 --- a/.agents/skills/playwright-cdp/scripts/notion.mjs +++ b/.agents/skills/playwright-cdp/scripts/notion.mjs @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import { commentBook, writeDoc, saveAsset, docName, stamp, readUrls } from './doc.mjs'; +import { commentBook, writeDoc, saveAsset, docName, stamp, readUrls, connectCdp } from './doc.mjs'; // usage: node download.mjs [cdp-port] const URLS_FILE = process.argv[2] || './urls.txt'; @@ -472,8 +472,7 @@ if (!process.env.NOTION_SELFTEST) { fs.mkdirSync(OUT, { recursive: true }); log(`URLs: ${URLS.length}`); - const { chromium } = await import('playwright-core'); - const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); + const browser = await connectCdp(PORT); const ctx = browser.contexts()[0]; const page = await ctx.newPage(); diff --git a/.agents/skills/playwright-cdp/scripts/slack.mjs b/.agents/skills/playwright-cdp/scripts/slack.mjs index dbbd7da..acad71f 100644 --- a/.agents/skills/playwright-cdp/scripts/slack.mjs +++ b/.agents/skills/playwright-cdp/scripts/slack.mjs @@ -1,5 +1,5 @@ import path from 'path'; -import { commentBook, writeDoc, saveAsset, docName, readUrls } from './doc.mjs'; +import { commentBook, writeDoc, saveAsset, docName, readUrls, connectCdp } from './doc.mjs'; // usage: node slack.mjs [cdp-port] // Accepts a channel link, a message permalink, or a thread link. @@ -196,8 +196,7 @@ async function extract(page, url, out) { } if (!URLS.length) { console.error('usage: node slack.mjs [cdp-port]'); process.exit(1); } -const { chromium } = await import('playwright-core'); -const browser = await chromium.connectOverCDP(`http://127.0.0.1:${PORT}`); +const browser = await connectCdp(PORT); const ctx = browser.contexts()[0]; for (const url of URLS) { const page = await ctx.newPage();