doughatcher · superterran · Aug 2, 2026 · Aug 2, 2026
diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml
@@ -35,76 +35,3 @@ jobs:
           CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }}
           CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
         run: npx wrangler deploy
-
-  reddit-cache:
-    runs-on: ubuntu-latest
-    needs: deploy
-    if: ${{ always() && needs.deploy.result != 'failure' }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Download latest backup to scan for Reddit URLs
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          mkdir -p /tmp/content-scan
-          # Try to download the latest backup release ZIP
-          gh release download --repo doughatcher/blog --pattern "*.zip" \
-            --dir /tmp/backup --clobber 2>/dev/null || true
-          ZIP=$(ls /tmp/backup/*.zip 2>/dev/null | head -1)
-          if [ -n "$ZIP" ]; then
-            echo "Extracting $ZIP..."
-            unzip -q "$ZIP" -d /tmp/content-scan || true
-          else
-            echo "No backup release found — KV cache will not be updated"
-          fi
-
-      - name: Pre-populate KV cache with Reddit metadata
-        env:
-          CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
-          CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }}
-          CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
-          KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905
-        run: |
-          npm install -g wrangler
-          echo "Scanning /tmp/content-scan for Reddit URLs..."
-          URLS=$(grep -r -h 'https://\(www\.\)\?reddit\.com[^ )>]*' /tmp/content-scan \
-            --include="*.md" -o 2>/dev/null | sort -u || true)
-          COUNT=$(echo "$URLS" | grep -c . || echo 0)
-          echo "Found $COUNT Reddit URL(s)"
-
-          for URL in $URLS; do
-            # Resolve /s/ share links via HTTP redirect
-            if echo "$URL" | grep -q '/s/'; then
-              RESOLVED=$(curl -sI "$URL" -A "Mozilla/5.0" -L --max-redirs 3 2>/dev/null \
-                | grep -i "^location:" | tail -1 | sed 's/location: //i' | tr -d '\r' || echo "")
-              [ -n "$RESOLVED" ] && URL="$RESOLVED"
-            fi
-
-            BASE_URL=$(echo "$URL" | cut -d'?' -f1 | sed 's|/$||')
-            echo "$BASE_URL" | grep -q '/comments/' || continue
-
-            echo -n "  $BASE_URL ... "
-            OLD_URL=$(echo "$BASE_URL" | sed 's|www\.reddit\.com|old.reddit.com|')
-            HTML=$(curl -s "$OLD_URL" -A "Mozilla/5.0" -H "Accept: text/html" 2>/dev/null || echo "")
-
-            TITLE=$(echo "$HTML" | grep -o 'property="og:title" content="[^"]*"' \
-              | head -1 | sed 's/.*content="//;s/"//')
-            IMAGE=$(echo "$HTML" | grep -o 'property="og:image" content="[^"]*"' \
-              | head -1 | sed 's/.*content="//;s/"//' | sed 's/&amp;/\&/g')
-            SUBREDDIT=$(echo "$BASE_URL" | grep -o 'r/[^/]*' | head -1 || echo "reddit.com")
-
-            if [ -z "$TITLE" ]; then
-              echo "skip (no title)"
-              continue
-            fi
-
-            PAYLOAD=$(python3 -c "import json,sys; print(json.dumps({'title':sys.argv[1],'image':sys.argv[2] if sys.argv[2] else None,'domain':sys.argv[3],'description':None}))" "$TITLE" "$IMAGE" "$SUBREDDIT")
-
-            CACHE_KEY="preview:$BASE_URL"
-            npx wrangler kv key put "$CACHE_KEY" "$PAYLOAD" \
-              --namespace-id="$KV_NAMESPACE_ID" --remote 2>/dev/null && echo "cached" || echo "failed"
-          done
-          echo "Reddit cache pre-population complete."
diff --git a/.github/workflows/warm-preview-cache.yml b/.github/workflows/warm-preview-cache.yml
@@ -0,0 +1,52 @@
+name: Warm Reddit preview cache
+
+# Fills the link-preview KV cache with real Reddit metadata, off the render path.
+#
+# The Worker deliberately does NOT fetch Reddit on a cache miss any more: Reddit
+# rate-limits anonymous scraping to a small per-IP budget, and serving 20 link
+# cards per pageview burned it instantly and cached the resulting "Welcome to
+# Reddit" wall. Rendering happens constantly; new posts appear every few days.
+# This job decouples the two — it fetches each URL exactly once, ever, and
+# stores it with no expiry. Until it succeeds, cards show a title derived from
+# the URL slug, which is always available and never wrong.
+#
+# Optional secrets REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET switch it to the free
+# Reddit OAuth API, which is not subject to the anonymous block. Without them it
+# falls back to scraping old.reddit.com with a wide delay between requests.
+
+on:
+  schedule:
+    - cron: "20 6 * * *" # 06:20 UTC daily
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Scan and report without writing to KV"
+        type: boolean
+        default: false
+
+permissions: {}
+
+concurrency:
+  group: warm-preview-cache
+  cancel-in-progress: false
+
+jobs:
+  warm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Warm KV cache
+        env:
+          CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
+          CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+          KV_NAMESPACE_ID: f48049994ef74c418fe7b8f491f6e905
+          REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
+          REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
+          DRY_RUN: ${{ inputs.dry_run && '1' || '' }}
+        run: python3 apps/link-preview-service/warm-cache.py
diff --git a/apps/link-preview-service/src/index.js b/apps/link-preview-service/src/index.js
@@ -84,6 +84,53 @@ function isRedditFallbackImage(imageUrl) {
   }
 }
 
+// Titles Reddit serves when it is stonewalling rather than rendering the post:
+// the logged-out interstitial, the 403 "Blocked" page, and the generic site
+// titles. These are valid-looking strings, which is exactly why they used to
+// get written to KV and pinned there — every one of these is a FAILURE, not
+// metadata, and must never be cached.
+const REDDIT_WALL_TITLES = new Set([
+  'welcome to reddit',
+  'blocked',
+  'reddit',
+  'reddit - dive into anything',
+  'reddit - the heart of the internet',
+  'log in or sign up',
+]);
+
+function isWallTitle(title) {
+  if (!title) return true;
+  return REDDIT_WALL_TITLES.has(title.trim().toLowerCase());
+}
+
+// Canonical cache key. The browser sends the URL exactly as it appears in the
+// post (usually WITH a trailing slash); the warmer used to strip it, so the two
+// sides wrote and read different keys and the warmed data was never seen.
+// Both sides now go through this one rule: drop the query/fragment, drop the
+// trailing slash, and normalise old./www. to a single host.
+function normalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    u.search = '';
+    u.hash = '';
+    u.hostname = u.hostname.replace(/^old\./, 'www.');
+    let s = u.toString();
+    return s.endsWith('/') ? s.slice(0, -1) : s;
+  } catch {
+    return url;
+  }
+}
+
+// Bump to invalidate the whole cache at once — old entries stay until their TTL
+// but are never read again. v2 added image_is_fallback. v3 changes the key to a
+// normalized URL and stops caching failures, so every v2 entry (many of which
+// pinned Reddit's "Welcome to Reddit" wall) is abandoned here rather than purged.
+const CACHE_VERSION = 'v3';
+
+function cacheKeyFor(url) {
+  return `preview:${CACHE_VERSION}:${normalizeUrl(url)}`;
+}
+
 async function fetchRedditMetadata(url) {
   const resolvedUrl = await resolveRedditShareUrl(url);
 
@@ -95,7 +142,7 @@ async function fetchRedditMetadata(url) {
     if (res.ok) {
       const data = await res.json();
       const post = data[0]?.data?.children?.[0]?.data;
-      if (post?.title) {
+      if (post?.title && !isWallTitle(post.title)) {
         const badThumbnails = new Set(['self', 'default', 'nsfw', 'spoiler', 'image', '']);
         const previewImg =
           post.preview?.images?.[0]?.resolutions?.slice(-1)[0]?.url?.replace(/&amp;/g, '&') ||
@@ -114,29 +161,43 @@ async function fetchRedditMetadata(url) {
     }
   } catch {}
 
-  // Fallback: Reddit JSON API is blocked from Cloudflare IPs.
-  // old.reddit.com is server-rendered and returns proper OG tags including og:image.
-  const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com');
-  const htmlMeta = await fetchHtmlMetadata(oldUrl);
-  if (htmlMeta.title) {
-    const subreddit = (() => {
-      try {
-        const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean);
-        const rIdx = parts.indexOf('r');
-        return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com';
-      } catch { return 'reddit.com'; }
-    })();
-    return {
-      ...htmlMeta,
-      domain: subreddit,
-      image_is_fallback: isRedditFallbackImage(htmlMeta.image),
-    };
-  }
+  const subreddit = (() => {
+    try {
+      const parts = new URL(resolvedUrl).pathname.split('/').filter(Boolean);
+      const rIdx = parts.indexOf('r');
+      return rIdx !== -1 ? `r/${parts[rIdx + 1]}` : 'reddit.com';
+    } catch { return 'reddit.com'; }
+  })();
+
+  // Fallback: old.reddit.com is server-rendered and returns proper OG tags.
+  // Reddit rate-limits unauthenticated scraping hard and then 403s the caller,
+  // so this is expected to fail more often than not from Worker egress. A throw
+  // or a wall title both mean "no data" — neither is allowed to reach the cache.
+  try {
+    const oldUrl = resolvedUrl.replace('www.reddit.com', 'old.reddit.com');
+    const htmlMeta = await fetchHtmlMetadata(oldUrl);
+    if (!isWallTitle(htmlMeta.title)) {
+      return {
+        ...htmlMeta,
+        domain: subreddit,
+        image_is_fallback: isRedditFallbackImage(htmlMeta.image),
+      };
+    }
+  } catch {}
 
-  // Last resort: title from URL slug, no image
+  // Last resort: title from the URL slug. This is derived, not fetched, so it
+  // is always available and never wrong — but it is also not the real title, so
+  // it is marked provisional and deliberately NOT cached. The scheduled warmer
+  // (which runs off-Worker, at low volume) fills in the real metadata later.
   const slugTitle = titleFromRedditSlug(resolvedUrl);
   if (!slugTitle) throw new Error('could not resolve Reddit post URL');
-  return { title: slugTitle, description: null, image: null, domain: 'reddit.com' };
+  return {
+    title: slugTitle,
+    description: null,
+    image: null,
+    domain: subreddit,
+    provisional: true,
+  };
 }
 
 // Extract OG/meta tags from HTML for non-Reddit URLs.
@@ -194,11 +255,14 @@ export default {
     }
 
     try {
-      // Check KV cache first (populated by GitHub Actions at deploy time)
+      const cacheKey = cacheKeyFor(url);
+
+      // Check KV cache first (kept warm by the scheduled warmer workflow).
+      // Legacy poisoned entries are ignored rather than served, so the cards
+      // self-heal without needing a manual purge.
       if (env?.PREVIEW_CACHE) {
-        const cacheKey = `preview:${url}`;
         const cached = await env.PREVIEW_CACHE.get(cacheKey, 'json');
-        if (cached) {
+        if (cached && !isWallTitle(cached.title)) {
           return new Response(JSON.stringify(cached), {
             status: 200,
             headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400', 'X-Cache': 'HIT' },
@@ -210,15 +274,25 @@ export default {
         ? await fetchRedditMetadata(url)
         : await fetchHtmlMetadata(url);
 
-      // Write to KV cache for future requests (TTL: 7 days)
-      if (env?.PREVIEW_CACHE && metadata.title) {
-        const cacheKey = `preview:${url}`;
-        await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), { expirationTtl: 604800 });
+      // Cache only real, verified metadata — never a wall title, never a
+      // provisional slug-derived title. Reddit post titles are immutable, so a
+      // good Reddit result is stored with NO expiry: fetched once, kept forever,
+      // and never re-requested. Other sites can change their <title>, so those
+      // keep a 7-day TTL.
+      const cacheable =
+        env?.PREVIEW_CACHE && !metadata.provisional && !isWallTitle(metadata.title);
+      if (cacheable) {
+        const opts = isRedditUrl(url) ? undefined : { expirationTtl: 604800 };
+        await env.PREVIEW_CACHE.put(cacheKey, JSON.stringify(metadata), opts);
       }
 
+      // A provisional (slug-derived) response must NOT sit in Cloudflare's edge
+      // cache for a day, or the real title wouldn't appear until long after the
+      // warmer filled KV in. Give it a short TTL so it refreshes promptly.
+      const edgeTtl = cacheable ? 86400 : 300;
       return new Response(JSON.stringify(metadata), {
         status: 200,
-        headers: { ...CORS_HEADERS, 'Cache-Control': 'public, max-age=86400' },
+        headers: { ...CORS_HEADERS, 'Cache-Control': `public, max-age=${edgeTtl}` },
       });
     } catch (err) {
       return new Response(JSON.stringify({ error: err.message, domain: getDomain(url) }), {