WebCheck 0.6.0: configurable crawler, Lighthouse and local scan history

.github/workflows/node.js.yml

@@ -1,31 +1,38 @@
-# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
-
 name: Node.js CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main", "upgrade/**"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 jobs:
-  build:
-
+  test:
     runs-on: ubuntu-latest
-
     strategy:
       matrix:
         node-version: [18.x, 20.x, 22.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
-
     steps:
-    - uses: actions/checkout@v4
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-    - run: npm ci
-    - run: npm run build --if-present
-    - run: npm test
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Check JavaScript syntax
+        run: npm run check
+      - name: Run tests
+        run: npm test
+      - name: Create standalone project archive
+        if: matrix.node-version == '22.x'
+        run: |
+          mkdir -p /tmp/webcheck-package
+          cp -R public test .github /tmp/webcheck-package/
+          cp server.js security-audit.js seo-audit.js performance-audit.js accessibility-audit.js link-audit.js rate-limit.js ip-safety.js site-crawler.js lighthouse.js package.json README.md LICENSE .gitignore /tmp/webcheck-package/
+          (cd /tmp/webcheck-package && zip -qr "$GITHUB_WORKSPACE/WebCheck-0.6.0.zip" .)
+      - name: Upload standalone project
+        if: matrix.node-version == '22.x'
+        uses: actions/upload-artifact@v4
+        with:
+          name: WebCheck-0.6.0
+          path: WebCheck-0.6.0.zip
+          retention-days: 14

accessibility-audit.js

@@ -0,0 +1,90 @@
+"use strict";
+
+const {parseAttributes}=require("./seo-audit");
+
+function runAccessibilityAudit({html=""}={}){
+  const source=String(html);
+  const checks=[];
+  const recommendations=[];
+  const addCheck=(key,label,description,status,value,weight=3)=>{
+    checks.push({category:"accessibility",key,label,description,status,value,weight});
+  };
+  const add=(condition,title,text,priority="medium")=>{
+    if(condition) recommendations.push({title,text,priority});
+  };
+
+  const images=[...source.matchAll(/<img\b[^>]*>/gi)].map(match=>parseAttributes(match[0]));
+  const missingAlt=images.filter(attrs=>!("alt" in attrs)).length;
+  addCheck("imageAlt","Image alternative text","Informative images need useful alt text; decorative images should use alt=\"\".",
+    missingAlt===0?"pass":"warn",missingAlt+"/"+images.length+" image(s) missing alt",5);
+  add(missingAlt>0,"Add image alternative text",
+    missingAlt+" image(s) do not declare an alt attribute. Add meaningful alt text or an empty alt for decorative images.");
+
+  const ids=[...source.matchAll(/\bid\s*=\s*["']([^"']+)["']/gi)].map(match=>match[1]);
+  const duplicates=[...new Set(ids.filter((id,index)=>ids.indexOf(id)!==index))];
+  addCheck("uniqueIds","Unique element IDs","Duplicate IDs can break labels, scripts and assistive-technology references.",
+    duplicates.length===0?"pass":"fail",duplicates.length?duplicates.join(", "):ids.length+" unique ID(s)",4);
+  add(duplicates.length>0,"Make element IDs unique","Duplicate IDs found: "+duplicates.join(", ")+".","high");
+
+  const labels=[...source.matchAll(/<label\b([^>]*)>([\s\S]*?)<\/label>/gi)];
+  const explicitLabelIds=new Set(labels.map(match=>parseAttributes("<label "+match[1]+">").for).filter(Boolean));
+  const wrappedLabels=labels.map(match=>match[0]);
+  const controls=[...source.matchAll(/<(input|select|textarea)\b([^>]*)>/gi)]
+    .map(match=>({tag:match[1].toLowerCase(),attrs:parseAttributes(match[0]),raw:match[0]}))
+    .filter(item=>!["hidden","submit","reset","button","image"].includes(String(item.attrs.type||"").toLowerCase()));
+  const unlabeled=controls.filter(item=>{
+    const id=item.attrs.id;
+    const explicitlyLinked=id&&explicitLabelIds.has(id);
+    const wrapped=wrappedLabels.some(label=>label.toLowerCase().includes("<"+item.tag));
+    return !explicitlyLinked&&!wrapped&&!item.attrs["aria-label"]&&!item.attrs["aria-labelledby"];
+  }).length;
+  addCheck("formLabels","Form control labels","Inputs, selects and textareas need programmatic labels or accessible names.",
+    unlabeled===0?"pass":"warn",unlabeled+"/"+controls.length+" control(s) without a detected label",5);
+  add(unlabeled>0,"Label form controls",
+    unlabeled+" form control(s) have no detected label, aria-label or aria-labelledby reference.","high");
+
+  const buttons=[...source.matchAll(/<button\b([^>]*)>([\s\S]*?)<\/button>/gi)];
+  const unlabeledButtons=buttons.filter(match=>{
+    const attrs=parseAttributes("<button "+match[1]+">");
+    const text=match[2].replace(/<[^>]+>/g,"").replace(/\s+/g,"").trim();
+    return !text&&!attrs["aria-label"]&&!attrs["aria-labelledby"]&&!attrs.title;
+  }).length;
+  addCheck("buttonNames","Button accessible names","Buttons should expose a readable name to assistive technologies.",
+    unlabeledButtons===0?"pass":"warn",unlabeledButtons+"/"+buttons.length+" button(s) without a detected name",4);
+  add(unlabeledButtons>0,"Add accessible button names",
+    unlabeledButtons+" button(s) have no visible text or accessible name.");
+
+  const links=[...source.matchAll(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi)];
+  const emptyLinks=links.filter(match=>{
+    const attrs=parseAttributes("<a "+match[1]+">");
+    const text=match[2].replace(/<[^>]+>/g,"").trim();
+    const hasImageAlt=/<img\b[^>]*\balt\s*=\s*["'][^"']+["']/i.test(match[2]);
+    return !text&&!hasImageAlt&&!attrs["aria-label"]&&!attrs["aria-labelledby"]&&!attrs.title;
+  }).length;
+  addCheck("linkNames","Link accessible names","Links should describe their destination or action.",
+    emptyLinks===0?"pass":"warn",emptyLinks+"/"+links.length+" link(s) without a detected name",3);
+  add(emptyLinks>0,"Give links descriptive names",
+    emptyLinks+" link(s) have no visible text or accessible name.");
+
+  const mainCount=(source.match(/<main\b/gi)||[]).length;
+  addCheck("mainLandmark","Main landmark","A page should expose one main landmark for its primary content.",
+    mainCount===1?"pass":"warn",mainCount+" main landmark(s)",3);
+  add(mainCount!==1,"Review the main landmark","Provide exactly one main landmark for the primary page content.");
+
+  const positiveTabindex=[...source.matchAll(/\btabindex\s*=\s*["']([1-9]\d*)["']/gi)].length;
+  addCheck("tabOrder","Keyboard tab order","Positive tabindex values can create a confusing focus order.",
+    positiveTabindex===0?"pass":"warn",positiveTabindex+" positive tabindex value(s)",2);
+  add(positiveTabindex>0,"Avoid positive tabindex values",
+    "Use the natural DOM order and tabindex=\"0\" or tabindex=\"-1\" rather than positive tabindex values.");
+
+  const tables=[...source.matchAll(/<table\b[^>]*>([\s\S]*?)<\/table>/gi)];
+  const tablesWithoutHeaders=tables.filter(match=>!/<th\b/i.test(match[1])).length;
+  addCheck("tableHeaders","Data table headers","Data tables should identify column or row headers with th elements.",
+    tablesWithoutHeaders===0?"pass":"warn",tablesWithoutHeaders+"/"+tables.length+" table(s) without th",2);
+  add(tablesWithoutHeaders>0,"Add table headers",
+    tablesWithoutHeaders+" table(s) do not contain th elements. Use headers for tabular data.");
+
+  return {checks,recommendations,metrics:{images:images.length,controls:controls.length,links:links.length,buttons:buttons.length}};
+}
+
+module.exports={runAccessibilityAudit};

ip-safety.js

@@ -0,0 +1,49 @@
+"use strict";
+
+const net=require("net");
+
+function isPrivateIp(address){
+  const family=net.isIP(address);
+  if(!family) return true;
+
+  if(family===4){
+    const p=address.split(".").map(Number);
+    return (
+      p[0]===0 ||
+      p[0]===10 ||
+      p[0]===127 ||
+      (p[0]===100 && p[1]>=64 && p[1]<=127) ||
+      (p[0]===169 && p[1]===254) ||
+      (p[0]===172 && p[1]>=16 && p[1]<=31) ||
+      (p[0]===192 && (
+        p[1]===0 ||
+        p[1]===2 ||
+        (p[1]===88 && p[2]===99) ||
+        p[1]===168
+      )) ||
+      (p[0]===198 && (p[1]===18 || p[1]===19 || p[1]===51 && p[2]===100)) ||
+      (p[0]===203 && p[1]===0 && p[2]===113) ||
+      p[0]>=224
+    );
+  }
+
+  const ip=address.toLowerCase();
+  if(ip.startsWith("::")) return true;
+
+  const segments=ip.split(":");
+  const first=parseInt(segments[0]||"0",16);
+  const second=parseInt(segments[1]||"0",16);
+
+  // Only global-unicast IPv6 space (2000::/3) is eligible.
+  if(first<0x2000 || first>0x3fff) return true;
+
+  // Special-purpose and documentation ranges.
+  if(first===0x2001 && second<=0x01ff) return true;
+  if(ip.startsWith("2001:db8:")) return true;
+  if(first===0x2002) return true; // 6to4
+  if(first===0x3fff && second<=0x0fff) return true; // documentation
+
+  return false;
+}
+
+module.exports={isPrivateIp};

lighthouse.js

@@ -0,0 +1,129 @@
+"use strict";
+
+const https=require("https");
+
+const API_HOST="www.googleapis.com";
+const API_PATH="/pagespeedonline/v5/runPagespeed";
+const MAX_RESPONSE_BYTES=8*1024*1024;
+
+function clampScore(value){
+  return typeof value==="number"&&Number.isFinite(value)
+    ? Math.max(0,Math.min(100,Math.round(value*100)))
+    : null;
+}
+
+function parseLighthouseResponse(payload){
+  const lighthouse=payload&&payload.lighthouseResult;
+  if(!lighthouse) throw new Error("Google PageSpeed did not return a Lighthouse report");
+
+  const categories=lighthouse.categories||{};
+  const scores={
+    performance:clampScore(categories.performance&&categories.performance.score),
+    accessibility:clampScore(categories.accessibility&&categories.accessibility.score),
+    bestPractices:clampScore(categories["best-practices"]&&categories["best-practices"].score),
+    seo:clampScore(categories.seo&&categories.seo.score)
+  };
+  const audits=lighthouse.audits||{};
+  const metricDefinitions=[
+    ["first-contentful-paint","First Contentful Paint","fcp"],
+    ["largest-contentful-paint","Largest Contentful Paint","lcp"],
+    ["speed-index","Speed Index","speedIndex"],
+    ["total-blocking-time","Total Blocking Time","tbt"],
+    ["cumulative-layout-shift","Cumulative Layout Shift","cls"],
+    ["interactive","Time to Interactive","tti"]
+  ];
+  const metrics={};
+  for(const [id,label,key] of metricDefinitions){
+    const audit=audits[id];
+    if(audit&&audit.displayValue){
+      metrics[key]={label,value:audit.displayValue,numericValue:typeof audit.numericValue==="number"?audit.numericValue:null};
+    }
+  }
+
+  const opportunities=[];
+  const refs=categories.performance&&categories.performance.auditRefs||[];
+  for(const ref of refs){
+    if(!ref||ref.weight===0) continue;
+    const audit=audits[ref.id];
+    if(!audit||audit.score===null||audit.score>=1) continue;
+    const details=audit.details||{};
+    if(!["opportunity","table","list","debugdata"].includes(details.type)&&!audit.displayValue) continue;
+    opportunities.push({
+      id:ref.id,
+      title:audit.title||ref.id,
+      description:String(audit.description||"").replace(/\s+/g," ").slice(0,500),
+      displayValue:audit.displayValue||"",
+      score:clampScore(audit.score),
+      savingsMs:typeof details.overallSavingsMs==="number"?Math.round(details.overallSavingsMs):null,
+      savingsBytes:typeof details.overallSavingsBytes==="number"?Math.round(details.overallSavingsBytes):null
+    });
+  }
+  opportunities.sort((a,b)=>(b.savingsMs||0)-(a.savingsMs||0)||(b.savingsBytes||0)-(a.savingsBytes||0));
+
+  return {
+    provider:"Google PageSpeed Insights / Lighthouse",
+    reportType:"lab",
+    strategy:lighthouse.configSettings&&lighthouse.configSettings.emulatedFormFactor||"unknown",
+    lighthouseVersion:lighthouse.lighthouseVersion||"",
+    fetchTime:lighthouse.fetchTime||"",
+    finalUrl:lighthouse.finalUrl||payload.id||"",
+    scores,
+    metrics,
+    opportunities:opportunities.slice(0,8),
+    warnings:(lighthouse.runWarnings||[]).map(item=>String(item).slice(0,400)).slice(0,10)
+  };
+}
+
+function requestPageSpeed(targetUrl,{strategy="mobile",locale="en",apiKey="",timeoutMs=65000}={}){
+  return new Promise((resolve,reject)=>{
+    const params=new URLSearchParams();
+    params.set("url",targetUrl);
+    params.set("strategy",strategy==="desktop"?"desktop":"mobile");
+    params.set("locale",locale==="ru"?"ru":"en");
+    for(const category of ["performance","accessibility","best-practices","seo"]){
+      params.append("category",category);
+    }
+    if(apiKey) params.set("key",apiKey);
+
+    const request=https.get({
+      hostname:API_HOST,
+      path:API_PATH+"?"+params.toString(),
+      headers:{
+        "Accept":"application/json",
+        "User-Agent":"WebCheck/0.5 Lighthouse diagnostics"
+      }
+    },response=>{
+      const chunks=[];
+      let bytes=0;
+      response.on("data",chunk=>{
+        bytes+=chunk.length;
+        if(bytes>MAX_RESPONSE_BYTES){
+          request.destroy(new Error("PageSpeed response exceeded the size limit"));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      response.on("end",()=>{
+        const body=Buffer.concat(chunks).toString("utf8");
+        let data;
+        try{data=JSON.parse(body);}
+        catch{reject(new Error("PageSpeed returned an invalid JSON response"));return;}
+        if(response.statusCode<200||response.statusCode>=300){
+          const message=data.error&&data.error.message||"PageSpeed API request failed";
+          const error=new Error(response.statusCode===429
+            ? "PageSpeed API quota reached. Try later or configure PAGESPEED_API_KEY."
+            : message);
+          error.statusCode=response.statusCode;
+          reject(error);
+          return;
+        }
+        try{resolve(parseLighthouseResponse(data));}
+        catch(error){reject(error);}
+      });
+    });
+    request.setTimeout(timeoutMs,()=>request.destroy(new Error("Lighthouse request timed out")));
+    request.on("error",reject);
+  });
+}
+
+module.exports={requestPageSpeed,parseLighthouseResponse};

link-audit.js

@@ -0,0 +1,64 @@
+"use strict";
+
+const {parseAttributes}=require("./seo-audit");
+
+function extractInternalLinks(html,baseUrl,limit=8){
+  const base=new URL(baseUrl);
+  const matches=[...String(html).matchAll(/<a\b[^>]*>/gi)];
+  const seen=new Set();
+  const links=[];
+  for(const match of matches){
+    const attrs=parseAttributes(match[0]);
+    const href=String(attrs.href||"").trim();
+    if(!href||/^(?:#|mailto:|tel:|javascript:|data:)/i.test(href)) continue;
+    try{
+      const candidate=new URL(href,base);
+      if(!["http:","https:"].includes(candidate.protocol)||candidate.origin!==base.origin) continue;
+      candidate.hash="";
+      const normalized=candidate.toString();
+      if(seen.has(normalized)) continue;
+      seen.add(normalized);
+      links.push(normalized);
+      if(links.length>=limit) break;
+    }catch{
+      // Invalid link URLs are ignored by the crawler.
+    }
+  }
+  return links;
+}
+
+async function crawlInternalLinks(html,baseUrl,probe,{limit=8,concurrency=3}={}){
+  const urls=extractInternalLinks(html,baseUrl,limit);
+  const results=new Array(urls.length);
+  let cursor=0;
+  const workers=Array.from({length:Math.min(concurrency,urls.length)},async()=>{
+    while(true){
+      const index=cursor++;
+      if(index>=urls.length) return;
+      try{
+        const result=await probe(urls[index]);
+        results[index]={
+          url:urls[index],
+          statusCode:Number(result.statusCode)||0,
+          statusText:result.statusText||"",
+          redirectTo:result.redirectTo||"",
+          status:result.status==="blocked"?"blocked":
+            Number(result.statusCode)>=400?"broken":
+            Number(result.statusCode)>=300?"redirect":"ok"
+        };
+      }catch(error){
+        results[index]={url:urls[index],statusCode:0,statusText:error.message||"Request failed",status:"unverified"};
+      }
+    }
+  });
+  await Promise.all(workers);
+  return {
+    checked:results.length,
+    broken:results.filter(item=>item.status==="broken").length,
+    redirects:results.filter(item=>item.status==="redirect").length,
+    unverified:results.filter(item=>item.status==="blocked"||item.status==="unverified").length,
+    results
+  };
+}
+
+module.exports={extractInternalLinks,crawlInternalLinks};