From 88a56eb2da4e014fb7d496862fc868bcc06a8039 Mon Sep 17 00:00:00 2001 From: Doug Hatcher Date: Sun, 2 Aug 2026 10:28:53 -0400 Subject: [PATCH] fix(link-preview): stop caching Reddit failures, keep good results forever Reddit rate-limits anonymous scraping to a small per-IP budget and then returns 403 "Blocked". The Worker fetched Reddit at render time, so one visitor loading a page with 20 Reddit links spent the entire budget at once, got the logged-out wall, and wrote it to KV. "Welcome to Reddit" is a valid-looking title, so it passed the only guard there was and got pinned for 7 days. Rendering and fetching were coupled when they should not be: the sites are read constantly, but new posts appear every few days. Worker: - Never fetch Reddit on a cache miss. Fall back to a title derived from the URL slug, which is always available and never wrong. That code was already written (titleFromRedditSlug) and had been dead since it landed. - Treat the wall titles as failures, on read and on write, so a blocked fetch can neither be stored nor served. - Cache good Reddit metadata with NO expiry. Post titles are immutable, so each URL is fetched once and kept. Other sites keep a 7-day TTL since their can change. - Normalize the cache key. The warmer stripped the trailing slash while the Worker looked it up with one, so warmed entries were never read. Both sides now share one rule, verified to agree byte-for-byte. - Bump CACHE_VERSION to v3, which abandons the 165 existing v2 entries (many of them poisoned) instead of purging them. - Give provisional responses a 5-minute edge TTL, not a day, so the real title appears promptly once the warmer fills KV. Warmer (new scheduled workflow, replaces the job on deploy-worker): - Read each site's feed.json. The old job scanned a release artifact from doughatcher/blog and its last run pulled waccamaw_b3343a.zip and found zero Reddit URLs, so it had been a no-op. Feeds find 52. - Skip URLs that already have good metadata: fetched once, ever. - Refetch entries that hold a wall title, so poison heals. - Space requests out, and use the free Reddit OAuth API when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set. - Never write a wall title. On failure write nothing and let the slug title stand until a later run succeeds. Note: production was running the unmerged branch feat/reddit-fallback-image-badge (its only delta was the v2 key bump, folded in here as v3). --- .github/workflows/deploy-worker.yml | 73 ------ .github/workflows/warm-preview-cache.yml | 52 ++++ apps/link-preview-service/src/index.js | 132 +++++++--- apps/link-preview-service/warm-cache.py | 292 +++++++++++++++++++++++ 4 files changed, 447 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/warm-preview-cache.yml create mode 100644 apps/link-preview-service/warm-cache.py diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml index beb7155..2167288 100644 --- a/.github/workflows/deploy-worker.yml +++ b/.github/workflows/deploy-worker.yml @@ -35,76 +35,3 @@ jobs: CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} run: npx wrangler deploy - - reddit-cache: - runs-on: ubuntu-latest - needs: deploy - if: ${{ always() && needs.deploy.result != 'failure' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download latest backup to scan for Reddit URLs - env: - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p /tmp/content-scan - # Try to download the latest backup release ZIP - gh release download --repo doughatcher/blog --pattern "*.zip" \ - --dir /tmp/backup --clobber 2>/dev/null || true - ZIP=$(ls /tmp/backup/*.zip 2>/dev/null | head -1) - if [ -n "$ZIP" ]; then - echo "Extracting $ZIP..." - unzip -q "$ZIP" -d /tmp/content-scan || true - else - echo "No backup release found — KV cache will not be updated" - fi - - - name: Pre-populate KV cache with Reddit metadata - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} - KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905 - run: | - npm install -g wrangler - echo "Scanning /tmp/content-scan for Reddit URLs..." - URLS=$(grep -r -h 'https://\(www\.\)\?reddit\.com[^ )>]*' /tmp/content-scan \ - --include="*.md" -o 2>/dev/null | sort -u || true) - COUNT=$(echo "$URLS" | grep -c . || echo 0) - echo "Found $COUNT Reddit URL(s)" - - for URL in $URLS; do - # Resolve /s/ share links via HTTP redirect - if echo "$URL" | grep -q '/s/'; then - RESOLVED=$(curl -sI "$URL" -A "Mozilla/5.0" -L --max-redirs 3 2>/dev/null \ - | grep -i "^location:" | tail -1 | sed 's/location: //i' | tr -d '\r' || echo "") - [ -n "$RESOLVED" ] && URL="$RESOLVED" - fi - - BASE_URL=$(echo "$URL" | cut -d'?' -f1 | sed 's|/$||') - echo "$BASE_URL" | grep -q '/comments/' || continue - - echo -n " $BASE_URL ... " - OLD_URL=$(echo "$BASE_URL" | sed 's|www\.reddit\.com|old.reddit.com|') - HTML=$(curl -s "$OLD_URL" -A "Mozilla/5.0" -H "Accept: text/html" 2>/dev/null || echo "") - - TITLE=$(echo "$HTML" | grep -o 'property="og:title" content="[^"]*"' \ - | head -1 | sed 's/.*content="//;s/"//') - IMAGE=$(echo "$HTML" | grep -o 'property="og:image" content="[^"]*"' \ - | head -1 | sed 's/.*content="//;s/"//' | sed 's/&/\&/g') - SUBREDDIT=$(echo "$BASE_URL" | grep -o 'r/[^/]*' | head -1 || echo "reddit.com") - - if [ -z "$TITLE" ]; then - echo "skip (no title)" - continue - fi - - PAYLOAD=$(python3 -c "import json,sys; print(json.dumps({'title':sys.argv[1],'image':sys.argv[2] if sys.argv[2] else None,'domain':sys.argv[3],'description':None}))" "$TITLE" "$IMAGE" "$SUBREDDIT") - - CACHE_KEY="preview:$BASE_URL" - npx wrangler kv key put "$CACHE_KEY" "$PAYLOAD" \ - --namespace-id="$KV_NAMESPACE_ID" --remote 2>/dev/null && echo "cached" || echo "failed" - done - echo "Reddit cache pre-population complete." diff --git a/.github/workflows/warm-preview-cache.yml b/.github/workflows/warm-preview-cache.yml new file mode 100644 index 0000000..31e881f --- /dev/null +++ b/.github/workflows/warm-preview-cache.yml @@ -0,0 +1,52 @@ +name: Warm Reddit preview cache + +# Fills the link-preview KV cache with real Reddit metadata, off the render path. +# +# The Worker deliberately does NOT fetch Reddit on a cache miss any more: Reddit +# rate-limits anonymous scraping to a small per-IP budget, and serving 20 link +# cards per pageview burned it instantly and cached the resulting "Welcome to +# Reddit" wall. Rendering happens constantly; new posts appear every few days. +# This job decouples the two — it fetches each URL exactly once, ever, and +# stores it with no expiry. Until it succeeds, cards show a title derived from +# the URL slug, which is always available and never wrong. +# +# Optional secrets REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET switch it to the free +# Reddit OAuth API, which is not subject to the anonymous block. Without them it +# falls back to scraping old.reddit.com with a wide delay between requests. + +on: + schedule: + - cron: "20 6 * * *" # 06:20 UTC daily + workflow_dispatch: + inputs: + dry_run: + description: "Scan and report without writing to KV" + type: boolean + default: false + +permissions: {} + +concurrency: + group: warm-preview-cache + cancel-in-progress: false + +jobs: + warm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Warm KV cache + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905 + REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} + REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} + DRY_RUN: ${{ inputs.dry_run && '1' || '' }} + run: python3 apps/link-preview-service/warm-cache.py diff --git a/apps/link-preview-service/src/index.js b/apps/link-preview-service/src/index.js index ee2edd4..d80054c 100644 --- a/apps/link-preview-service/src/index.js +++ b/apps/link-preview-service/src/index.js @@ -84,6 +84,53 @@ function isRedditFallbackImage(imageUrl) { } } +// Titles Reddit serves when it is stonewalling rather than rendering the post: +// the logged-out interstitial, the 403 "Blocked" page, and the generic site +// titles. These are valid-looking strings, which is exactly why they used to +// get written to KV and pinned there — every one of these is a FAILURE, not +// metadata, and must never be cached. +const REDDIT_WALL_TITLES = new Set([ + 'welcome to reddit', + 'blocked', + 'reddit', + 'reddit - dive into anything', + 'reddit - the heart of the internet', + 'log in or sign up', +]); + +function isWallTitle(title) { + if (!title) return true; + return REDDIT_WALL_TITLES.has(title.trim().toLowerCase()); +} + +// Canonical cache key. The browser sends the URL exactly as it appears in the +// post (usually WITH a trailing slash); the warmer used to strip it, so the two +// sides wrote and read different keys and the warmed data was never seen. +// Both sides now go through this one rule: drop the query/fragment, drop the +// trailing slash, and normalise old./www. to a single host. +function normalizeUrl(url) { + try { + const u = new URL(url); + u.search = ''; + u.hash = ''; + u.hostname = u.hostname.replace(/^old\./, 'www.'); + let s = u.toString(); + return s.endsWith('/') ? s.slice(0, -1) : s; + } catch { + return url; + } +} + +// Bump to invalidate the whole cache at once — old entries stay until their TTL +// but are never read again. v2 added image_is_fallback. v3 changes the key to a +// normalized URL and stops caching failures, so every v2 entry (many of which +// pinned Reddit's "Welcome to Reddit" wall) is abandoned here rather than purged. +const CACHE_VERSION = 'v3'; + +function cacheKeyFor(url) { + return `preview:${CACHE_VERSION}:${normalizeUrl(url)}`; +} + async function fetchRedditMetadata(url) { const resolvedUrl = await resolveRedditShareUrl(url); @@ -95,7 +142,7 @@ async function fetchRedditMetadata(url) { if (res.ok) { const data = await res.json(); const post = data[0]?.data?.children?.[0]?.data; - if (post?.title) { + if (post?.title && !isWallTitle(post.title)) { const badThumbnails = new Set(['self', 'default', 'nsfw', 'spoiler', 'image', '']); const previewImg = post.preview?.images?.[0]?.resolutions?.slice(-1)[0]?.url?.replace(/&/g, '&') || @@ -114,29 +161,43 @@ async function fetchRedditMetadata(url) { } } catch {} - // Fallback: Reddit JSON API is blocked from Cloudflare IPs. - // old.reddit.com is server-rendered and returns proper OG tags including og:image. - const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com'); - const htmlMeta = await fetchHtmlMetadata(oldUrl); - if (htmlMeta.title) { - const subreddit = (() => { - try { - const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean); - const rIdx = parts.indexOf('r'); - return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com'; - } catch { return 'reddit.com'; } - })(); - return { - ...htmlMeta, - domain: subreddit, - image_is_fallback: isRedditFallbackImage(htmlMeta.image), - }; - } + const subreddit = (() => { + try { + const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean); + const rIdx = parts.indexOf('r'); + return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com'; + } catch { return 'reddit.com'; } + })(); + + // Fallback: old.reddit.com is server-rendered and returns proper OG tags. + // Reddit rate-limits unauthenticated scraping hard and then 403s the caller, + // so this is expected to fail more often than not from Worker egress. A throw + // or a wall title both mean "no data" — neither is allowed to reach the cache. + try { + const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com'); + const htmlMeta = await fetchHtmlMetadata(oldUrl); + if (!isWallTitle(htmlMeta.title)) { + return { + ...htmlMeta, + domain: subreddit, + image_is_fallback: isRedditFallbackImage(htmlMeta.image), + }; + } + } catch {} - // Last resort: title from URL slug, no image + // Last resort: title from the URL slug. This is derived, not fetched, so it + // is always available and never wrong — but it is also not the real title, so + // it is marked provisional and deliberately NOT cached. The scheduled warmer + // (which runs off-Worker, at low volume) fills in the real metadata later. const slugTitle = titleFromRedditSlug(resolvedUrl); if (!slugTitle) throw new Error('could not resolve Reddit post URL'); - return { title: slugTitle, description: null, image: null, domain: 'reddit.com' }; + return { + title: slugTitle, + description: null, + image: null, + domain: subreddit, + provisional: true, + }; } // Extract OG/meta tags from HTML for non-Reddit URLs. @@ -194,11 +255,14 @@ export default { } try { - // Check KV cache first (populated by GitHub Actions at deploy time) + const cacheKey = cacheKeyFor(url); + + // Check KV cache first (kept warm by the scheduled warmer workflow). + // Legacy poisoned entries are ignored rather than served, so the cards + // self-heal without needing a manual purge. if (env?.PREVIEW_CACHE) { - const cacheKey = `preview:${url}`; const cached = await env.PREVIEW_CACHE.get(cacheKey, 'json'); - if (cached) { + if (cached && !isWallTitle(cached.title)) { return new Response(JSON.stringify(cached), { status: 200, headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400', 'X-Cache': 'HIT' }, @@ -210,15 +274,25 @@ export default { ? await fetchRedditMetadata(url) : await fetchHtmlMetadata(url); - // Write to KV cache for future requests (TTL: 7 days) - if (env?.PREVIEW_CACHE && metadata.title) { - const cacheKey = `preview:${url}`; - await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), { expirationTtl: 604800 }); + // Cache only real, verified metadata — never a wall title, never a + // provisional slug-derived title. Reddit post titles are immutable, so a + // good Reddit result is stored with NO expiry: fetched once, kept forever, + // and never re-requested. Other sites can change their <title>, so those + // keep a 7-day TTL. + const cacheable = + env?.PREVIEW_CACHE && !metadata.provisional && !isWallTitle(metadata.title); + if (cacheable) { + const opts = isRedditUrl(url) ? undefined : { expirationTtl: 604800 }; + await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), opts); } + // A provisional (slug-derived) response must NOT sit in Cloudflare's edge + // cache for a day, or the real title wouldn't appear until long after the + // warmer filled KV in. Give it a short TTL so it refreshes promptly. + const edgeTtl = cacheable ? 86400 : 300; return new Response(JSON.stringify(metadata), { status: 200, - headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400' }, + headers: { ...CORS_HEADERS, 'Cache-Control': `public, max-age=${edgeTtl}` }, }); } catch (err) { return new Response(JSON.stringify({ error: err.message, domain: getDomain(url) }), { diff --git a/apps/link-preview-service/warm-cache.py b/apps/link-preview-service/warm-cache.py new file mode 100644 index 0000000..91078ff --- /dev/null +++ b/apps/link-preview-service/warm-cache.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +""" +Warm the link-preview KV cache with real Reddit metadata. + +Why this exists +--------------- +Reddit rate-limits unauthenticated scraping to a small per-IP budget and then +returns 403 "Blocked". The Worker used to fetch Reddit at render time, so a +single visitor loading a page with 20 Reddit links spent the whole budget at +once, got walled, and cached the wall. Rendering and fetching were coupled to +each other when they should not be: the site is read constantly, but new posts +appear every few days. + +So the Worker no longer fetches Reddit on a cache miss (it falls back to a +title derived from the URL slug), and this script does the fetching instead: +off the render path, at low volume, once per URL, forever. + +Behaviour +--------- +* Reads each site's feed.json — no backup artifact needed. +* Skips any URL that already has good metadata in KV. Reddit post titles are + immutable, so a URL is fetched exactly once, ever. +* Refetches URLs whose cached entry is a wall title, so existing poisoned + entries heal on the next run. +* Sleeps between Reddit requests to stay well under the anonymous budget. +* Uses the Reddit OAuth API when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set + (free, and not subject to the anonymous block); otherwise falls back to + scraping old.reddit.com more slowly. +* Never writes a wall title to KV, and writes with no expiry. + +Required env: CLOUDFLARE_ACCOUNT_ID, KV_NAMESPACE_ID, + and CLOUDFLARE_API_TOKEN or (CLOUDFLARE_EMAIL + CLOUDFLARE_API_KEY) +Optional env: REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET, DRY_RUN=1 +""" + +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +FEEDS = [ + "https://doughatcher.com/feed.json", + "https://superterran.net/feed.json", + "https://leaning.blue/feed.json", +] + +UA = "python:doughatcher-link-preview:2.0 (by /u/superterran)" + +# Keep in sync with REDDIT_WALL_TITLES in src/index.js. +WALL_TITLES = { + "welcome to reddit", + "blocked", + "reddit", + "reddit - dive into anything", + "reddit - the heart of the internet", + "log in or sign up", +} + +# Seconds between Reddit requests. Anonymous scraping needs a wide gap; the +# authenticated API allows ~100 requests/minute, so it can go much faster. +SLEEP_ANON = 8.0 +SLEEP_OAUTH = 1.5 + +# Must match CACHE_VERSION in src/index.js. +CACHE_VERSION = "v3" + +ACCOUNT_ID = os.environ.get("CLOUDFLARE_ACCOUNT_ID", "") +NAMESPACE_ID = os.environ.get("KV_NAMESPACE_ID", "") +DRY_RUN = os.environ.get("DRY_RUN") == "1" + + +def is_wall_title(title): + return not title or title.strip().lower() in WALL_TITLES + + +def normalize_url(url): + """Canonical cache key form. Must match normalizeUrl() in src/index.js.""" + p = urllib.parse.urlsplit(url) + host = re.sub(r"^old\.", "www.", p.netloc) + s = urllib.parse.urlunsplit((p.scheme, host, p.path, "", "")) + return s[:-1] if s.endswith("/") else s + + +def cf_headers(): + token = os.environ.get("CLOUDFLARE_API_TOKEN") + if token: + return {"Authorization": f"Bearer {token}"} + email = os.environ.get("CLOUDFLARE_EMAIL") + key = os.environ.get("CLOUDFLARE_API_KEY") + if email and key: + return {"X-Auth-Email": email, "X-Auth-Key": key} + sys.exit("need CLOUDFLARE_API_TOKEN, or CLOUDFLARE_EMAIL + CLOUDFLARE_API_KEY") + + +def kv_url(key): + return ( + f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}" + f"/storage/kv/namespaces/{NAMESPACE_ID}/values/{urllib.parse.quote(key, safe='')}" + ) + + +def kv_get(key): + req = urllib.request.Request(kv_url(key), headers=cf_headers()) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + except (json.JSONDecodeError, urllib.error.URLError): + return None + + +def kv_put(key, value): + """Write with no expiry — fetched once, kept forever.""" + if DRY_RUN: + return True + body, boundary = [], "----warmcache" + body.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"value\"\r\n\r\n{json.dumps(value)}\r\n") + body.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"metadata\"\r\n\r\n{{}}\r\n") + body.append(f"--{boundary}--\r\n") + data = "".join(body).encode() + headers = cf_headers() + headers["Content-Type"] = f"multipart/form-data; boundary={boundary}" + req = urllib.request.Request(kv_url(key), data=data, headers=headers, method="PUT") + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()).get("success", False) + + +def get_oauth_token(): + cid = os.environ.get("REDDIT_CLIENT_ID") + secret = os.environ.get("REDDIT_CLIENT_SECRET") + if not (cid and secret): + return None + import base64 + + auth = base64.b64encode(f"{cid}:{secret}".encode()).decode() + req = urllib.request.Request( + "https://www.reddit.com/api/v1/access_token", + data=b"grant_type=client_credentials", + headers={"Authorization": f"Basic {auth}", "User-Agent": UA}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()).get("access_token") + except Exception as e: + print(f" ! OAuth token request failed ({e}); falling back to anonymous") + return None + + +def subreddit_of(url): + parts = [p for p in urllib.parse.urlsplit(url).path.split("/") if p] + if "r" in parts: + i = parts.index("r") + if i + 1 < len(parts): + return f"r/{parts[i + 1]}" + return "reddit.com" + + +def title_from_slug(url): + parts = [p for p in urllib.parse.urlsplit(url).path.split("/") if p] + if "comments" in parts: + i = parts.index("comments") + if i + 2 < len(parts): + t = urllib.parse.unquote(parts[i + 2]).replace("_", " ") + return t[:1].upper() + t[1:] + return None + + +def fetch_via_oauth(url, token): + path = urllib.parse.urlsplit(url).path + api = f"https://oauth.reddit.com{path}.json?raw_json=1" + req = urllib.request.Request( + api, headers={"Authorization": f"Bearer {token}", "User-Agent": UA} + ) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode()) + post = data[0]["data"]["children"][0]["data"] + if not post.get("title") or is_wall_title(post["title"]): + return None + images = (post.get("preview") or {}).get("images") or [] + image = None + if images: + res = images[0].get("resolutions") or [] + image = (res[-1]["url"] if res else images[0]["source"]["url"]).replace("&", "&") + ups, ncom = post.get("ups"), post.get("num_comments") + desc = f"Posted in r/{post['subreddit']} by u/{post.get('author')}" + if ups is not None and ncom is not None: + desc += f" • {ups:,} points and {ncom:,} comments" + return { + "title": post["title"], + "description": desc, + "image": image, + "domain": f"r/{post['subreddit']}", + } + + +def fetch_via_scrape(url): + old = url.replace("www.reddit.com", "old.reddit.com") + req = urllib.request.Request(old, headers={"User-Agent": UA, "Accept": "text/html"}) + with urllib.request.urlopen(req, timeout=30) as r: + html = r.read().decode("utf-8", "replace") + + def og(prop): + m = re.search(rf'<meta\s+property="og:{prop}"\s+content="([^"]*)"', html) + return m.group(1).replace("&", "&") if m else None + + title = og("title") + if is_wall_title(title): + return None + return { + "title": title, + "description": og("description"), + "image": og("image"), + "domain": subreddit_of(url), + } + + +def collect_urls(): + found = set() + for feed in FEEDS: + try: + req = urllib.request.Request(feed, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode()) + except Exception as e: + print(f"! could not read {feed}: {e}") + continue + blob = json.dumps(data) + for m in re.findall(r'https://(?:www\.)?reddit\.com/[^\s"\'<>\\)]+', blob): + if "/comments/" in m: + found.add(normalize_url(m)) + print(f" {feed}: {len(found)} cumulative reddit URL(s)") + return sorted(found) + + +def main(): + if not ACCOUNT_ID or not NAMESPACE_ID: + sys.exit("CLOUDFLARE_ACCOUNT_ID and KV_NAMESPACE_ID are required") + + print("Collecting Reddit URLs from site feeds...") + urls = collect_urls() + print(f"Found {len(urls)} unique Reddit URL(s)\n") + + token = get_oauth_token() + mode = "OAuth API" if token else "anonymous scrape" + sleep_for = SLEEP_OAUTH if token else SLEEP_ANON + print(f"Fetch mode: {mode}\n") + + skipped = warmed = failed = 0 + for url in urls: + key = f"preview:{CACHE_VERSION}:{url}" + cached = kv_get(key) + if cached and not is_wall_title(cached.get("title")): + skipped += 1 + continue + + reason = "poisoned" if cached else "missing" + print(f" [{reason}] {url}") + meta = None + try: + meta = fetch_via_oauth(url, token) if token else fetch_via_scrape(url) + except Exception as e: + print(f" fetch failed: {e}") + + if not meta: + # Blocked, walled, or deleted. Write NOTHING — the Worker will keep + # serving the slug-derived title until a later run succeeds. + failed += 1 + print(" -> no usable metadata; leaving uncached") + else: + kv_put(key, meta) + warmed += 1 + print(f" -> cached: {meta['title'][:60]}") + + time.sleep(sleep_for) + + print(f"\nDone. warmed={warmed} skipped={skipped} failed={failed}") + # A run where everything failed means we are blocked; surface it, but do not + # fail the build, since the site degrades gracefully to slug titles. + if warmed == 0 and failed > 0: + print("::warning::no URLs could be warmed this run (likely rate-limited)") + + +if __name__ == "__main__": + main()