diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml index beb7155..2167288 100644 --- a/.github/workflows/deploy-worker.yml +++ b/.github/workflows/deploy-worker.yml @@ -35,76 +35,3 @@ jobs: CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} run: npx wrangler deploy - - reddit-cache: - runs-on: ubuntu-latest - needs: deploy - if: ${{ always() && needs.deploy.result != 'failure' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download latest backup to scan for Reddit URLs - env: - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p /tmp/content-scan - # Try to download the latest backup release ZIP - gh release download --repo doughatcher/blog --pattern "*.zip" \ - --dir /tmp/backup --clobber 2>/dev/null || true - ZIP=$(ls /tmp/backup/*.zip 2>/dev/null | head -1) - if [ -n "$ZIP" ]; then - echo "Extracting $ZIP..." - unzip -q "$ZIP" -d /tmp/content-scan || true - else - echo "No backup release found — KV cache will not be updated" - fi - - - name: Pre-populate KV cache with Reddit metadata - env: - CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} - CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} - CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} - KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905 - run: | - npm install -g wrangler - echo "Scanning /tmp/content-scan for Reddit URLs..." - URLS=$(grep -r -h 'https://\(www\.\)\?reddit\.com[^ )>]*' /tmp/content-scan \ - --include="*.md" -o 2>/dev/null | sort -u || true) - COUNT=$(echo "$URLS" | grep -c . || echo 0) - echo "Found $COUNT Reddit URL(s)" - - for URL in $URLS; do - # Resolve /s/ share links via HTTP redirect - if echo "$URL" | grep -q '/s/'; then - RESOLVED=$(curl -sI "$URL" -A "Mozilla/5.0" -L --max-redirs 3 2>/dev/null \ - | grep -i "^location:" | tail -1 | sed 's/location: //i' | tr -d '\r' || echo "") - [ -n "$RESOLVED" ] && URL="$RESOLVED" - fi - - BASE_URL=$(echo "$URL" | cut -d'?' -f1 | sed 's|/$||') - echo "$BASE_URL" | grep -q '/comments/' || continue - - echo -n " $BASE_URL ... " - OLD_URL=$(echo "$BASE_URL" | sed 's|www\.reddit\.com|old.reddit.com|') - HTML=$(curl -s "$OLD_URL" -A "Mozilla/5.0" -H "Accept: text/html" 2>/dev/null || echo "") - - TITLE=$(echo "$HTML" | grep -o 'property="og:title" content="[^"]*"' \ - | head -1 | sed 's/.*content="//;s/"//') - IMAGE=$(echo "$HTML" | grep -o 'property="og:image" content="[^"]*"' \ - | head -1 | sed 's/.*content="//;s/"//' | sed 's/&/\&/g') - SUBREDDIT=$(echo "$BASE_URL" | grep -o 'r/[^/]*' | head -1 || echo "reddit.com") - - if [ -z "$TITLE" ]; then - echo "skip (no title)" - continue - fi - - PAYLOAD=$(python3 -c "import json,sys; print(json.dumps({'title':sys.argv[1],'image':sys.argv[2] if sys.argv[2] else None,'domain':sys.argv[3],'description':None}))" "$TITLE" "$IMAGE" "$SUBREDDIT") - - CACHE_KEY="preview:$BASE_URL" - npx wrangler kv key put "$CACHE_KEY" "$PAYLOAD" \ - --namespace-id="$KV_NAMESPACE_ID" --remote 2>/dev/null && echo "cached" || echo "failed" - done - echo "Reddit cache pre-population complete." diff --git a/.github/workflows/warm-preview-cache.yml b/.github/workflows/warm-preview-cache.yml new file mode 100644 index 0000000..31e881f --- /dev/null +++ b/.github/workflows/warm-preview-cache.yml @@ -0,0 +1,52 @@ +name: Warm Reddit preview cache + +# Fills the link-preview KV cache with real Reddit metadata, off the render path. +# +# The Worker deliberately does NOT fetch Reddit on a cache miss any more: Reddit +# rate-limits anonymous scraping to a small per-IP budget, and serving 20 link +# cards per pageview burned it instantly and cached the resulting "Welcome to +# Reddit" wall. Rendering happens constantly; new posts appear every few days. +# This job decouples the two — it fetches each URL exactly once, ever, and +# stores it with no expiry. Until it succeeds, cards show a title derived from +# the URL slug, which is always available and never wrong. +# +# Optional secrets REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET switch it to the free +# Reddit OAuth API, which is not subject to the anonymous block. Without them it +# falls back to scraping old.reddit.com with a wide delay between requests. + +on: + schedule: + - cron: "20 6 * * *" # 06:20 UTC daily + workflow_dispatch: + inputs: + dry_run: + description: "Scan and report without writing to KV" + type: boolean + default: false + +permissions: {} + +concurrency: + group: warm-preview-cache + cancel-in-progress: false + +jobs: + warm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Warm KV cache + env: + CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }} + CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905 + REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} + REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} + DRY_RUN: ${{ inputs.dry_run && '1' || '' }} + run: python3 apps/link-preview-service/warm-cache.py diff --git a/apps/link-preview-service/src/index.js b/apps/link-preview-service/src/index.js index ee2edd4..d80054c 100644 --- a/apps/link-preview-service/src/index.js +++ b/apps/link-preview-service/src/index.js @@ -84,6 +84,53 @@ function isRedditFallbackImage(imageUrl) { } } +// Titles Reddit serves when it is stonewalling rather than rendering the post: +// the logged-out interstitial, the 403 "Blocked" page, and the generic site +// titles. These are valid-looking strings, which is exactly why they used to +// get written to KV and pinned there — every one of these is a FAILURE, not +// metadata, and must never be cached. +const REDDIT_WALL_TITLES = new Set([ + 'welcome to reddit', + 'blocked', + 'reddit', + 'reddit - dive into anything', + 'reddit - the heart of the internet', + 'log in or sign up', +]); + +function isWallTitle(title) { + if (!title) return true; + return REDDIT_WALL_TITLES.has(title.trim().toLowerCase()); +} + +// Canonical cache key. The browser sends the URL exactly as it appears in the +// post (usually WITH a trailing slash); the warmer used to strip it, so the two +// sides wrote and read different keys and the warmed data was never seen. +// Both sides now go through this one rule: drop the query/fragment, drop the +// trailing slash, and normalise old./www. to a single host. +function normalizeUrl(url) { + try { + const u = new URL(url); + u.search = ''; + u.hash = ''; + u.hostname = u.hostname.replace(/^old\./, 'www.'); + let s = u.toString(); + return s.endsWith('/') ? s.slice(0, -1) : s; + } catch { + return url; + } +} + +// Bump to invalidate the whole cache at once — old entries stay until their TTL +// but are never read again. v2 added image_is_fallback. v3 changes the key to a +// normalized URL and stops caching failures, so every v2 entry (many of which +// pinned Reddit's "Welcome to Reddit" wall) is abandoned here rather than purged. +const CACHE_VERSION = 'v3'; + +function cacheKeyFor(url) { + return `preview:${CACHE_VERSION}:${normalizeUrl(url)}`; +} + async function fetchRedditMetadata(url) { const resolvedUrl = await resolveRedditShareUrl(url); @@ -95,7 +142,7 @@ async function fetchRedditMetadata(url) { if (res.ok) { const data = await res.json(); const post = data[0]?.data?.children?.[0]?.data; - if (post?.title) { + if (post?.title && !isWallTitle(post.title)) { const badThumbnails = new Set(['self', 'default', 'nsfw', 'spoiler', 'image', '']); const previewImg = post.preview?.images?.[0]?.resolutions?.slice(-1)[0]?.url?.replace(/&/g, '&') || @@ -114,29 +161,43 @@ async function fetchRedditMetadata(url) { } } catch {} - // Fallback: Reddit JSON API is blocked from Cloudflare IPs. - // old.reddit.com is server-rendered and returns proper OG tags including og:image. - const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com'); - const htmlMeta = await fetchHtmlMetadata(oldUrl); - if (htmlMeta.title) { - const subreddit = (() => { - try { - const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean); - const rIdx = parts.indexOf('r'); - return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com'; - } catch { return 'reddit.com'; } - })(); - return { - ...htmlMeta, - domain: subreddit, - image_is_fallback: isRedditFallbackImage(htmlMeta.image), - }; - } + const subreddit = (() => { + try { + const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean); + const rIdx = parts.indexOf('r'); + return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com'; + } catch { return 'reddit.com'; } + })(); + + // Fallback: old.reddit.com is server-rendered and returns proper OG tags. + // Reddit rate-limits unauthenticated scraping hard and then 403s the caller, + // so this is expected to fail more often than not from Worker egress. A throw + // or a wall title both mean "no data" — neither is allowed to reach the cache. + try { + const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com'); + const htmlMeta = await fetchHtmlMetadata(oldUrl); + if (!isWallTitle(htmlMeta.title)) { + return { + ...htmlMeta, + domain: subreddit, + image_is_fallback: isRedditFallbackImage(htmlMeta.image), + }; + } + } catch {} - // Last resort: title from URL slug, no image + // Last resort: title from the URL slug. This is derived, not fetched, so it + // is always available and never wrong — but it is also not the real title, so + // it is marked provisional and deliberately NOT cached. The scheduled warmer + // (which runs off-Worker, at low volume) fills in the real metadata later. const slugTitle = titleFromRedditSlug(resolvedUrl); if (!slugTitle) throw new Error('could not resolve Reddit post URL'); - return { title: slugTitle, description: null, image: null, domain: 'reddit.com' }; + return { + title: slugTitle, + description: null, + image: null, + domain: subreddit, + provisional: true, + }; } // Extract OG/meta tags from HTML for non-Reddit URLs. @@ -194,11 +255,14 @@ export default { } try { - // Check KV cache first (populated by GitHub Actions at deploy time) + const cacheKey = cacheKeyFor(url); + + // Check KV cache first (kept warm by the scheduled warmer workflow). + // Legacy poisoned entries are ignored rather than served, so the cards + // self-heal without needing a manual purge. if (env?.PREVIEW_CACHE) { - const cacheKey = `preview:${url}`; const cached = await env.PREVIEW_CACHE.get(cacheKey, 'json'); - if (cached) { + if (cached && !isWallTitle(cached.title)) { return new Response(JSON.stringify(cached), { status: 200, headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400', 'X-Cache': 'HIT' }, @@ -210,15 +274,25 @@ export default { ? await fetchRedditMetadata(url) : await fetchHtmlMetadata(url); - // Write to KV cache for future requests (TTL: 7 days) - if (env?.PREVIEW_CACHE && metadata.title) { - const cacheKey = `preview:${url}`; - await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), { expirationTtl: 604800 }); + // Cache only real, verified metadata — never a wall title, never a + // provisional slug-derived title. Reddit post titles are immutable, so a + // good Reddit result is stored with NO expiry: fetched once, kept forever, + // and never re-requested. Other sites can change their , so those + // keep a 7-day TTL. + const cacheable = + env?.PREVIEW_CACHE && !metadata.provisional && !isWallTitle(metadata.title); + if (cacheable) { + const opts = isRedditUrl(url) ? undefined : { expirationTtl: 604800 }; + await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), opts); } + // A provisional (slug-derived) response must NOT sit in Cloudflare's edge + // cache for a day, or the real title wouldn't appear until long after the + // warmer filled KV in. Give it a short TTL so it refreshes promptly. + const edgeTtl = cacheable ? 86400 : 300; return new Response(JSON.stringify(metadata), { status: 200, - headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400' }, + headers: { ...CORS_HEADERS, 'Cache-Control': `public, max-age=${edgeTtl}` }, }); } catch (err) { return new Response(JSON.stringify({ error: err.message, domain: getDomain(url) }), { diff --git a/apps/link-preview-service/warm-cache.py b/apps/link-preview-service/warm-cache.py new file mode 100644 index 0000000..91078ff --- /dev/null +++ b/apps/link-preview-service/warm-cache.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +""" +Warm the link-preview KV cache with real Reddit metadata. + +Why this exists +--------------- +Reddit rate-limits unauthenticated scraping to a small per-IP budget and then +returns 403 "Blocked". The Worker used to fetch Reddit at render time, so a +single visitor loading a page with 20 Reddit links spent the whole budget at +once, got walled, and cached the wall. Rendering and fetching were coupled to +each other when they should not be: the site is read constantly, but new posts +appear every few days. + +So the Worker no longer fetches Reddit on a cache miss (it falls back to a +title derived from the URL slug), and this script does the fetching instead: +off the render path, at low volume, once per URL, forever. + +Behaviour +--------- +* Reads each site's feed.json — no backup artifact needed. +* Skips any URL that already has good metadata in KV. Reddit post titles are + immutable, so a URL is fetched exactly once, ever. +* Refetches URLs whose cached entry is a wall title, so existing poisoned + entries heal on the next run. +* Sleeps between Reddit requests to stay well under the anonymous budget. +* Uses the Reddit OAuth API when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set + (free, and not subject to the anonymous block); otherwise falls back to + scraping old.reddit.com more slowly. +* Never writes a wall title to KV, and writes with no expiry. + +Required env: CLOUDFLARE_ACCOUNT_ID, KV_NAMESPACE_ID, + and CLOUDFLARE_API_TOKEN or (CLOUDFLARE_EMAIL + CLOUDFLARE_API_KEY) +Optional env: REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET, DRY_RUN=1 +""" + +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +FEEDS = [ + "https://doughatcher.com/feed.json", + "https://superterran.net/feed.json", + "https://leaning.blue/feed.json", +] + +UA = "python:doughatcher-link-preview:2.0 (by /u/superterran)" + +# Keep in sync with REDDIT_WALL_TITLES in src/index.js. +WALL_TITLES = { + "welcome to reddit", + "blocked", + "reddit", + "reddit - dive into anything", + "reddit - the heart of the internet", + "log in or sign up", +} + +# Seconds between Reddit requests. Anonymous scraping needs a wide gap; the +# authenticated API allows ~100 requests/minute, so it can go much faster. +SLEEP_ANON = 8.0 +SLEEP_OAUTH = 1.5 + +# Must match CACHE_VERSION in src/index.js. +CACHE_VERSION = "v3" + +ACCOUNT_ID = os.environ.get("CLOUDFLARE_ACCOUNT_ID", "") +NAMESPACE_ID = os.environ.get("KV_NAMESPACE_ID", "") +DRY_RUN = os.environ.get("DRY_RUN") == "1" + + +def is_wall_title(title): + return not title or title.strip().lower() in WALL_TITLES + + +def normalize_url(url): + """Canonical cache key form. Must match normalizeUrl() in src/index.js.""" + p = urllib.parse.urlsplit(url) + host = re.sub(r"^old\.", "www.", p.netloc) + s = urllib.parse.urlunsplit((p.scheme, host, p.path, "", "")) + return s[:-1] if s.endswith("/") else s + + +def cf_headers(): + token = os.environ.get("CLOUDFLARE_API_TOKEN") + if token: + return {"Authorization": f"Bearer {token}"} + email = os.environ.get("CLOUDFLARE_EMAIL") + key = os.environ.get("CLOUDFLARE_API_KEY") + if email and key: + return {"X-Auth-Email": email, "X-Auth-Key": key} + sys.exit("need CLOUDFLARE_API_TOKEN, or CLOUDFLARE_EMAIL + CLOUDFLARE_API_KEY") + + +def kv_url(key): + return ( + f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}" + f"/storage/kv/namespaces/{NAMESPACE_ID}/values/{urllib.parse.quote(key, safe='')}" + ) + + +def kv_get(key): + req = urllib.request.Request(kv_url(key), headers=cf_headers()) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + except (json.JSONDecodeError, urllib.error.URLError): + return None + + +def kv_put(key, value): + """Write with no expiry — fetched once, kept forever.""" + if DRY_RUN: + return True + body, boundary = [], "----warmcache" + body.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"value\"\r\n\r\n{json.dumps(value)}\r\n") + body.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"metadata\"\r\n\r\n{{}}\r\n") + body.append(f"--{boundary}--\r\n") + data = "".join(body).encode() + headers = cf_headers() + headers["Content-Type"] = f"multipart/form-data; boundary={boundary}" + req = urllib.request.Request(kv_url(key), data=data, headers=headers, method="PUT") + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()).get("success", False) + + +def get_oauth_token(): + cid = os.environ.get("REDDIT_CLIENT_ID") + secret = os.environ.get("REDDIT_CLIENT_SECRET") + if not (cid and secret): + return None + import base64 + + auth = base64.b64encode(f"{cid}:{secret}".encode()).decode() + req = urllib.request.Request( + "https://www.reddit.com/api/v1/access_token", + data=b"grant_type=client_credentials", + headers={"Authorization": f"Basic {auth}", "User-Agent": UA}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()).get("access_token") + except Exception as e: + print(f" ! OAuth token request failed ({e}); falling back to anonymous") + return None + + +def subreddit_of(url): + parts = [p for p in urllib.parse.urlsplit(url).path.split("/") if p] + if "r" in parts: + i = parts.index("r") + if i + 1 < len(parts): + return f"r/{parts[i + 1]}" + return "reddit.com" + + +def title_from_slug(url): + parts = [p for p in urllib.parse.urlsplit(url).path.split("/") if p] + if "comments" in parts: + i = parts.index("comments") + if i + 2 < len(parts): + t = urllib.parse.unquote(parts[i + 2]).replace("_", " ") + return t[:1].upper() + t[1:] + return None + + +def fetch_via_oauth(url, token): + path = urllib.parse.urlsplit(url).path + api = f"https://oauth.reddit.com{path}.json?raw_json=1" + req = urllib.request.Request( + api, headers={"Authorization": f"Bearer {token}", "User-Agent": UA} + ) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode()) + post = data[0]["data"]["children"][0]["data"] + if not post.get("title") or is_wall_title(post["title"]): + return None + images = (post.get("preview") or {}).get("images") or [] + image = None + if images: + res = images[0].get("resolutions") or [] + image = (res[-1]["url"] if res else images[0]["source"]["url"]).replace("&", "&") + ups, ncom = post.get("ups"), post.get("num_comments") + desc = f"Posted in r/{post['subreddit']} by u/{post.get('author')}" + if ups is not None and ncom is not None: + desc += f" • {ups:,} points and {ncom:,} comments" + return { + "title": post["title"], + "description": desc, + "image": image, + "domain": f"r/{post['subreddit']}", + } + + +def fetch_via_scrape(url): + old = url.replace("www.reddit.com", "old.reddit.com") + req = urllib.request.Request(old, headers={"User-Agent": UA, "Accept": "text/html"}) + with urllib.request.urlopen(req, timeout=30) as r: + html = r.read().decode("utf-8", "replace") + + def og(prop): + m = re.search(rf'<meta\s+property="og:{prop}"\s+content="([^"]*)"', html) + return m.group(1).replace("&", "&") if m else None + + title = og("title") + if is_wall_title(title): + return None + return { + "title": title, + "description": og("description"), + "image": og("image"), + "domain": subreddit_of(url), + } + + +def collect_urls(): + found = set() + for feed in FEEDS: + try: + req = urllib.request.Request(feed, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode()) + except Exception as e: + print(f"! could not read {feed}: {e}") + continue + blob = json.dumps(data) + for m in re.findall(r'https://(?:www\.)?reddit\.com/[^\s"\'<>\\)]+', blob): + if "/comments/" in m: + found.add(normalize_url(m)) + print(f" {feed}: {len(found)} cumulative reddit URL(s)") + return sorted(found) + + +def main(): + if not ACCOUNT_ID or not NAMESPACE_ID: + sys.exit("CLOUDFLARE_ACCOUNT_ID and KV_NAMESPACE_ID are required") + + print("Collecting Reddit URLs from site feeds...") + urls = collect_urls() + print(f"Found {len(urls)} unique Reddit URL(s)\n") + + token = get_oauth_token() + mode = "OAuth API" if token else "anonymous scrape" + sleep_for = SLEEP_OAUTH if token else SLEEP_ANON + print(f"Fetch mode: {mode}\n") + + skipped = warmed = failed = 0 + for url in urls: + key = f"preview:{CACHE_VERSION}:{url}" + cached = kv_get(key) + if cached and not is_wall_title(cached.get("title")): + skipped += 1 + continue + + reason = "poisoned" if cached else "missing" + print(f" [{reason}] {url}") + meta = None + try: + meta = fetch_via_oauth(url, token) if token else fetch_via_scrape(url) + except Exception as e: + print(f" fetch failed: {e}") + + if not meta: + # Blocked, walled, or deleted. Write NOTHING — the Worker will keep + # serving the slug-derived title until a later run succeeds. + failed += 1 + print(" -> no usable metadata; leaving uncached") + else: + kv_put(key, meta) + warmed += 1 + print(f" -> cached: {meta['title'][:60]}") + + time.sleep(sleep_for) + + print(f"\nDone. warmed={warmed} skipped={skipped} failed={failed}") + # A run where everything failed means we are blocked; surface it, but do not + # fail the build, since the site degrades gracefully to slug titles. + if warmed == 0 and failed > 0: + print("::warning::no URLs could be warmed this run (likely rate-limited)") + + +if __name__ == "__main__": + main()