From df60437077523616e8383f11e65a498ca644606b Mon Sep 17 00:00:00 2001 From: Wouter van Wijk Date: Fri, 13 Nov 2026 10:03:28 +0100 Subject: [PATCH 1/4] Validate the app against real sites; fix what it found Add a site-validation harness: an integration test loads ~10 real sites in the macOS app and exports each inspection package; reference.py measures the same sites twice with independent tools (curl, openssl, Playwright WebKit, dig, Team Cymru); compare.py reports only the differences, plus cross-page consistency checks on the real exports. App fixes found by the runs, each with a test: - Certificate host match reads the SAN, not only the CN. - A Set-Cookie without SameSite stays "not set" when WebKit reports None. - A null MX (RFC 7505) is no mail server. - Sovereignty CMS evidence shows where it was found, not raw source. - Sovereignty re-runs when detections change (cookie-only trackers). - Social widgets count as trackers in Sovereignty, as on Privacy. - Exported detections list their cookies (evidence is capped at 12). - Score thresholds live in one place (score_band.dart) for verdict text and card colour. Triage: docs/audits/2026-11-site-validatie.md Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 +- docs/audits/2026-11-site-validatie.md | 50 ++ integration_test/site_validation_test.dart | 138 +++++ lib/services/derive/cookies.dart | 8 +- lib/services/derive/cookies_view.dart | 12 +- lib/services/derive/score_band.dart | 35 ++ lib/services/derive/security_audit.dart | 89 +++- lib/services/detection/engine.dart | 4 + lib/services/html_service.dart | 34 +- lib/services/sovereignty_service.dart | 38 +- lib/widgets/probe_views.dart | 27 +- lib/widgets/score_band_color.dart | 10 + lib/widgets/sovereignty_view.dart | 13 +- pubspec.lock | 51 +- pubspec.yaml | 2 + test/derive/cookies_test.dart | 29 + test/derive/fixtures.dart | 6 +- test/derive/privacy_consistency_test.dart | 167 ++++++ test/derive/score_band_test.dart | 57 ++ test/derive/security_audit_test.dart | 46 +- test/fixtures/certs/rtvutrecht.pem | 34 ++ test/services/sovereignty_service_test.dart | 11 + test/services/sovereignty_state_test.dart | 47 ++ validation/README.md | 66 +++ validation/compare.py | 560 ++++++++++++++++++++ validation/reference.py | 350 ++++++++++++ validation/run.sh | 34 ++ validation/sites.yaml | 56 ++ 28 files changed, 1901 insertions(+), 77 deletions(-) create mode 100644 docs/audits/2026-11-site-validatie.md create mode 100644 integration_test/site_validation_test.dart create mode 100644 lib/services/derive/score_band.dart create mode 100644 lib/widgets/score_band_color.dart create mode 100644 test/derive/privacy_consistency_test.dart create mode 100644 test/derive/score_band_test.dart create mode 100644 test/fixtures/certs/rtvutrecht.pem create mode 100644 validation/README.md create mode 100644 validation/compare.py create mode 100644 validation/reference.py create mode 100755 validation/run.sh create mode 100644 validation/sites.yaml diff --git a/.gitignore b/.gitignore index c5a792ba..f682b56f 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,6 @@ android/.kotlin/sessions/ # Android signing secrets — never commit these android/key.properties *.jks -*.keystore \ No newline at end of file +*.keystore +# Site validation runs (validation/README.md) +validation/runs/ diff --git a/docs/audits/2026-11-site-validatie.md b/docs/audits/2026-11-site-validatie.md new file mode 100644 index 00000000..0718232b --- /dev/null +++ b/docs/audits/2026-11-site-validatie.md @@ -0,0 +1,50 @@ +# Site-validatie: triage (13 nov 2026) + +Tien echte sites zijn door de app gehaald (`validation/run.sh`, zie `validation/README.md`). Elke export is vergeleken met een onafhankelijke meting: curl, openssl, Playwright WebKit, dig en Team Cymru, telkens twee keer. De eerste run gaf ~40 afwijkingen. Per afwijking: oordeel, oorzaak en wat ermee is gedaan. + +Oordelen: **app-bug** · **harness** (de referentie of vergelijker zat fout) · **site** (de site gedraagt zich zo) · **platform** (een grens van WKWebView of iOS/macOS). + +## App-bugs (gefixt, met test) + +| # | Bevinding | Site(s) | Oorzaak | Fix | Test | +|---|---|---|---|---|---| +| A1 | Certificaat "CN differs from host" terwijl de SAN-lijst de host bevat | rtvutrecht | Alleen de Common Name werd vergeleken (zie ook audit nov 2026) | SAN (OID 2.5.29.17) uit de PEM gelezen; label zegt nu of CN of SAN matchte | `test/derive/security_audit_test.dart` met het echte certificaat (`test/fixtures/certs/rtvutrecht.pem`) | +| A2 | Dezelfde cookie zonder SameSite heette "None" (zwak, −5 in de cookiescore) als de browser hem ook zag, en "not set (Lax by default)" als alleen de server hem zag | kvk, wikipedia | WebKit meldt een ontbrekende SameSite als `None`; bij het samenvoegen won die waarde | Header zonder SameSite + browser `None` blijft "not set"; een echte browserwaarde (bv. Lax via JS) blijft staan | `test/derive/cookies_test.dart` (groep SameSite) | +| A3 | Null-MX (`0 .`, RFC 7505) verscheen als lege mailserver en werd als host opgezocht | example.com | DNS-parser nam de lege naam op | Null-MX wordt overgeslagen | `test/services/sovereignty_service_test.dart` | +| A4 | De CMS-check in Sovereignty toonde ruwe bronfragmenten als evidence | github | `detail` (bronfragment) in plaats van `title` | `title` ("HTML line 414: …") | — (alleen tekst) | +| A5 | Een tracker die alleen via een later binnengekomen cookie werd gevonden, stond wel op Services maar niet in de privacypijler van Sovereignty | rijksoverheid | Sovereignty rekende alleen opnieuw als het aantal resources veranderde | De detectienamen horen nu bij de invoersleutel | `test/services/sovereignty_state_test.dart` | +| A6 | Privacy-tab ("7 trackers follow you") en Sovereignty ("Number of trackers: 6") telden verschillend | allbirds, techcrunch | Privacy telt social widgets als tracker, Sovereignty niet | Besluit: social widgets zijn trackers op beide pagina's. Sovereignty-check heet nu "Trackers in the site's jurisdiction"; de oude widgets-check is "Payments & e-commerce" (geen dubbeltelling) | `test/derive/privacy_consistency_test.dart` | +| A7 | In de export ontbrak de koppeling cookie → service als de evidence-lijst (max. 12) vol zat | allbirds (`_gcl_au`) | Alleen evidence werd geëxporteerd | `cookies` per detectie staat nu in `analysis/detections.json` | `test/derive/privacy_consistency_test.dart` | +| A8 | Kleurdrempels van de scorekaarten stonden los in de widgets (80/50, 75/45) | — | Dubbele drempels in model en UI | `lib/services/derive/score_band.dart` is de enige bron; `bandColor()` voor de kleur | `test/derive/score_band_test.dart` (grenzen + geen losse drempels in de widgets) | + +## Open (niet gefixt) + +| # | Bevinding | Oordeel | Voorstel | +|---|---|---|---| +| O1 | bol.com: ook de WebView van de app kreeg Akamai's blokpagina ("IP address is blocked"), terwijl Playwright vanaf hetzelfde IP de echte site kreeg. De app analyseert de blokpagina als de site, en de divergentiemelding zegt ten onrechte "the browser loaded the page". | app-bug | De browser-status van het hoofddocument meten (`PerformanceNavigationTiming.responseStatus`) en een blok-/challengepagina markeren in plaats van analyseren. Dit is review-punt 2 uit het plan van 6 nov. | +| O2 | De app ziet soms minder cookies dan Playwright: kvk 6 van 7 (`IDHUB_JSESSIONID`), rijksoverheid wisselend 1 of 5 (Piwik `_pk_*`, `stg_*`). In een eerdere run met 25 s wachttijd klopte rijksoverheid wel. | site/platform | Meerdere app-runs per site draaien om het als volatiel te kunnen labelen; mogelijk ITP van WKWebView of steekproefsgewijze surveyscripts | +| O3 | Paginagewicht wijkt fors af op advertentiesites (techcrunch 25 MB tegen 6 MB) | site | Alleen als signaal (±50%); geen fout | + +## Harness-correcties (de app had gelijk) + +- HSTS en Referrer-Policy: de app volgt de drempels van Mozilla Observatory (HSTS ≥ 6 maanden; `no-referrer-when-downgrade` lekt). De referentie gebruikt nu dezelfde norm. +- robots.txt: curl volgde de consent-redirect van ad.nl. Daarnaast telde de regex lege `Disallow:`-regels mee omdat `\s` ook een newline matcht. +- NS/MX: gemeten op de CNAME of op een gedelegeerde www-zone in plaats van op het domein zelf. +- Hosting achter een CDN: de app noemt bewust de CDN in plaats van de verborgen origin. De check is nu "CDN = AS-houder". +- Browserdata: de eerste runs exporteerden direct na "settled". Met 20 s extra wachttijd (`VSV_LINGER`, standaard in `run.sh`) komen cookies, storage en hosts overeen. + +## Consistentiechecks (nieuw) + +Pagina's die hetzelfde beoordelen moeten hetzelfde zeggen. Dat wordt nu op twee niveaus getest: + +- **Offline (Dart)**: `test/derive/privacy_consistency_test.dart` controleert vijf dingen. Elke trackingcookie uit de Cookie Lens is een Services-detectie. Cookie en service zijn het eens over het soort tracking. De eigen cookie van de site is nergens een tracker. Privacy en Sovereignty tellen dezelfde trackers. De consent-tool is dezelfde. Daarnaast: `test/derive/score_band_test.dart` en de bestaande `cross_page_consistency_test.dart` (HSTS, nosniff, CDN). +- **Op echte exports**: `compare.py` heeft het domein `consistency` met dezelfde invarianten, plus de controle dat de Sovereignty-grade bij de score hoort. + +## Eindstand + +| Run | ok | volatiel | ❌ | +|---|---|---|---| +| Eerste (13 nov, 08:34), voor triage | 336 | 6 | 78 (incl. gaten en fantomen) | +| Laatste (13 nov, 09:32), na fixes | 418 | 7 | 16 | + +De resterende 16: tien komen door bol.com (O1). De andere zes zijn browservariatie op advertentie- en surveysites (O2, O3). Consistentie is op alle tien sites groen. diff --git a/integration_test/site_validation_test.dart b/integration_test/site_validation_test.dart new file mode 100644 index 00000000..5d616e83 --- /dev/null +++ b/integration_test/site_validation_test.dart @@ -0,0 +1,138 @@ +// Loads real sites in the real app and writes each inspection package, for +// the validation harness in validation/ (see validation/README.md). +// +// flutter test integration_test/site_validation_test.dart -d macos \ +// --dart-define=VSV_SITES=example=https://example.com,github=https://github.com \ +// --dart-define=VSV_OUT=/abs/path/validation/runs/ +// +// Writes //app.zip and //app-meta.json. +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/widgets.dart'; +import 'package:flutter_inappwebview/flutter_inappwebview.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:view_source_vibe/main.dart' as app; +import 'package:view_source_vibe/models/inspection/observed.dart'; +import 'package:view_source_vibe/services/html_service.dart'; +import 'package:view_source_vibe/services/inspection_export_service.dart'; + +const _sites = String.fromEnvironment('VSV_SITES'); +const _out = String.fromEnvironment('VSV_OUT'); +const _timeout = Duration(seconds: 90); + +/// Extra seconds to keep the page open after it settled (to test whether +/// late scripts change what the app reports). +const _linger = int.fromEnvironment('VSV_LINGER'); + +/// Session-restore keys of AppStateService; removed so the app does not +/// reopen the last page on start. Settings stay untouched. +const _restoreKeys = [ + 'pending_url', + 'last_url', + 'last_file', + 'last_file_name', + 'last_file_path', + 'last_file_is_url', + 'input_text', +]; + +/// What is still unfinished, by name; empty when the page is ready. +List _pending(HtmlService s) => [ + if (s.isLoading) 'loading', + if (!s.store.current.browserSettled) 'browserSettled', + if (!s.detectionsView.complete) 'detections', + if (s.generalView is Pending) 'general', + if (s.securityView is Pending) 'security', + if (s.cookiesView is Pending) 'cookies', + if (s.sovereigntyState is Pending) 'sovereignty', + ]; + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; + + testWidgets('site validation run', (tester) async { + if (_sites.isEmpty || _out.isEmpty) { + fail('Pass --dart-define=VSV_SITES=slug=url,... and VSV_OUT=/abs/dir'); + } + final sites = [ + for (final entry in _sites.split(',')) + if (entry.contains('=')) + ( + slug: entry.substring(0, entry.indexOf('=')), + url: entry.substring(entry.indexOf('=') + 1), + ), + ]; + + final prefs = await SharedPreferences.getInstance(); + for (final key in _restoreKeys) { + await prefs.remove(key); + } + + app.main(); + for (var i = 0; i < 100 && find.byType(app.MyApp).evaluate().isEmpty; i++) { + await tester.pump(const Duration(milliseconds: 100)); + } + await tester.pump(const Duration(seconds: 2)); + final html = tester.element(find.byType(app.MyApp)).read(); + // Sovereignty only runs while a view watches it; act as that view. + html.watchSovereignty(true); + + for (final site in sites) { + final dir = Directory('$_out/${site.slug}')..createSync(recursive: true); + final started = DateTime.now().toUtc(); + final clock = Stopwatch()..start(); + String? error; + var pending = []; + + try { + // Cold, pre-consent measurement: no cookies or cache from earlier + // sites or earlier runs. + await CookieManager.instance().deleteAllCookies(); + await InAppWebViewController.clearAllCache(); + + await html.loadUrl(site.url); + // Give the load a moment to start before polling for "done". + await tester.pump(const Duration(seconds: 2)); + while (clock.elapsed < _timeout) { + pending = _pending(html); + if (pending.isEmpty) break; + await tester.pump(const Duration(milliseconds: 500)); + } + pending = _pending(html); + if (_linger > 0) { + await tester.pump(Duration(seconds: _linger)); + // Collect once more (as when the page reports new resources) so + // the late state is in the snapshot. + html.onBrowserResourcesChanged(); + await tester.pump(const Duration(seconds: 8)); + } + + final bytes = InspectionExportService().buildPackageBytes(html); + File('${dir.path}/app.zip').writeAsBytesSync(bytes); + } catch (e) { + error = '$e'; + } + + final meta = { + 'slug': site.slug, + 'url': site.url, + 'startedAt': started.toIso8601String(), + 'finishedAt': DateTime.now().toUtc().toIso8601String(), + 'waitedMs': clock.elapsedMilliseconds, + 'timedOut': pending.isNotEmpty, + 'pendingAtExport': pending, + if (error != null) 'error': error, + }; + File('${dir.path}/app-meta.json') + .writeAsStringSync(const JsonEncoder.withIndent(' ').convert(meta)); + debugPrint('VSV ${site.slug}: ${clock.elapsedMilliseconds} ms' + '${pending.isEmpty ? '' : ', still pending: $pending'}' + '${error == null ? '' : ', error: $error'}'); + } + }, timeout: const Timeout(Duration(minutes: 30))); +} diff --git a/lib/services/derive/cookies.dart b/lib/services/derive/cookies.dart index f986eea2..ecb29624 100644 --- a/lib/services/derive/cookies.dart +++ b/lib/services/derive/cookies.dart @@ -64,7 +64,13 @@ List mergeCookies( path: existing.path ?? bc.path, requestHost: existing.requestHost, expires: existing.expires ?? bc.expires, - sameSite: existing.sameSite ?? bc.sameSite, + // WebKit reports a cookie without SameSite as "None"; when the server + // set it without the attribute, keep "not set" so the cookie reads the + // same whether or not the browser also saw it. + sameSite: existing.sameSite ?? + (existing.source == 'Server' && bc.sameSite?.toLowerCase() == 'none' + ? null + : bc.sameSite), secure: _anyTrue(existing.secure, bc.secure), httpOnly: _anyTrue(existing.httpOnly, bc.httpOnly), category: existing.category == CookieCategory.unknown diff --git a/lib/services/derive/cookies_view.dart b/lib/services/derive/cookies_view.dart index 56c4e1d6..b489778f 100644 --- a/lib/services/derive/cookies_view.dart +++ b/lib/services/derive/cookies_view.dart @@ -11,6 +11,7 @@ import 'cookies.dart'; import 'header_insights.dart' show InsightTone; import 'observed_text.dart'; import 'probe_general.dart' show probeGate; +import 'score_band.dart'; /// One cookie as the Cookies tab shows it. Flags are "yes", "no" or /// "unknown": a flag the platform did not expose is never "no". @@ -266,11 +267,12 @@ Observed deriveCookiesView(PageInspection p, {DateTime? now}) { } final baseVerdict = score == null ? 'Not judged: no cookies seen and a source is unknown.' - : score >= 75 - ? 'Low visible tracking pressure from cookies.' - : score >= 45 - ? 'Moderate cookie tracking signals detected.' - : 'Heavy tracking or weak cookie boundaries detected.'; + : switch (cookieScale.band(score)!) { + ScoreBand.good => 'Low visible tracking pressure from cookies.', + ScoreBand.fair => 'Moderate cookie tracking signals detected.', + ScoreBand.poor => + 'Heavy tracking or weak cookie boundaries detected.', + }; final verdict = score != null && !complete ? '$baseVerdict (partial: not all cookie sources could be read)' diff --git a/lib/services/derive/score_band.dart b/lib/services/derive/score_band.dart new file mode 100644 index 00000000..e3e9ced5 --- /dev/null +++ b/lib/services/derive/score_band.dart @@ -0,0 +1,35 @@ +/// One place for the thresholds that turn a 0–100 score into good / fair / +/// poor. The models use them for their verdict text and the widgets for the +/// colour, so a score never reads "Good" in a red card. +enum ScoreBand { good, fair, poor } + +class ScoreScale { + /// Lowest score that is [ScoreBand.good]. + final int good; + + /// Lowest score that is [ScoreBand.fair]. + final int fair; + + const ScoreScale({required this.good, required this.fair}); + + /// The band of [score]; null when there is no score. + ScoreBand? band(int? score) => score == null + ? null + : score >= good + ? ScoreBand.good + : score >= fair + ? ScoreBand.fair + : ScoreBand.poor; +} + +/// Security headers score (share of passing checks). +const securityScale = ScoreScale(good: 80, fair: 50); + +/// Content-Security-Policy score. +const cspScale = ScoreScale(good: 80, fair: 50); + +/// Sovereignty score ("Good" / "Fair" / "Poor"). +const sovereigntyScale = ScoreScale(good: 80, fair: 50); + +/// Cookie Privacy Lens score (low / moderate / heavy tracking pressure). +const cookieScale = ScoreScale(good: 75, fair: 45); diff --git a/lib/services/derive/security_audit.dart b/lib/services/derive/security_audit.dart index 84bc4578..3c41b504 100644 --- a/lib/services/derive/security_audit.dart +++ b/lib/services/derive/security_audit.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import '../../models/inspection/observed.dart'; import '../../models/inspection/page_inspection.dart'; import '../../models/inspection/server_snapshot.dart'; @@ -364,6 +366,74 @@ bool _hostMatches(String host, String commonName) { return false; } +/// The DNS names in the subjectAltName extension (OID 2.5.29.17) of a PEM +/// certificate; empty when there is none or the PEM cannot be read. Browsers +/// match the host against these names, not the Common Name (RFC 6125). +List subjectAltNames(String? pem) { + if (pem == null) return const []; + final b64 = pem + .split('\n') + .where((l) => !l.startsWith('-----')) + .join() + .replaceAll(RegExp(r'\s'), ''); + final List der; + try { + der = base64.decode(b64); + } on FormatException { + return const []; + } + // Header of the element at [pos]: (content start, content length). + (int, int)? header(int pos) { + if (pos + 1 >= der.length) return null; + var len = der[pos + 1]; + var start = pos + 2; + if (len & 0x80 != 0) { + final n = len & 0x7F; + if (n == 0 || n > 3 || start + n > der.length) return null; + len = 0; + for (var i = 0; i < n; i++) { + len = (len << 8) | der[start + i]; + } + start += n; + } + return start + len <= der.length ? (start, len) : null; + } + + const sanOid = [0x06, 0x03, 0x55, 0x1D, 0x11]; + for (var i = 0; i + sanOid.length < der.length; i++) { + var hit = true; + for (var j = 0; j < sanOid.length && hit; j++) { + hit = der[i + j] == sanOid[j]; + } + if (!hit) continue; + var pos = i + sanOid.length; + if (pos < der.length && der[pos] == 0x01) { + final crit = header(pos); // optional "critical" BOOLEAN + if (crit == null) return const []; + pos = crit.$1 + crit.$2; + } + if (pos >= der.length || der[pos] != 0x04) return const []; // OCTET STRING + final octets = header(pos); + if (octets == null || der[octets.$1] != 0x30) return const []; + final seq = header(octets.$1); + if (seq == null) return const []; + final names = []; + var p = seq.$1; + while (p < seq.$1 + seq.$2) { + final h = header(p); + if (h == null) break; + if (der[p] == 0x82) { + // [2] dNSName, IA5String + names.add( + ascii.decode(der.sublist(h.$1, h.$1 + h.$2), allowInvalid: true)); + } + p = h.$1 + h.$2; + } + return names; + } + return const []; +} + String _formatDate(String iso) { if (iso.isEmpty) return 'Unknown'; final d = DateTime.tryParse(iso); @@ -405,14 +475,21 @@ CertificateModel _certificate( } final subject = _fields(cert['subjectParsed']); final commonName = subject['Common Name'] ?? ''; - final matches = _hostMatches(host, commonName); + final sans = subjectAltNames(cert['pem']?.toString()); + final cnMatches = _hostMatches(host, commonName); + final sanMatches = sans.any((n) => _hostMatches(host, n)); + final matches = cnMatches || sanMatches; final hostLabel = host.isEmpty ? 'Host unknown' - : commonName.isEmpty - ? 'Common name unknown' - : matches - ? 'CN matches host' - : 'CN differs from host (SAN not checked)'; + : cnMatches + ? 'CN matches host' + : sanMatches + ? 'SAN matches host' + : sans.isNotEmpty + ? 'Host not in certificate (CN or SAN)' + : commonName.isEmpty + ? 'Common name unknown' + : 'CN differs from host (SAN not available)'; final der = cert['der']?.toString(); return CertificateModel( validityLabel: label, diff --git a/lib/services/detection/engine.dart b/lib/services/detection/engine.dart index dd923526..7ec392d5 100644 --- a/lib/services/detection/engine.dart +++ b/lib/services/detection/engine.dart @@ -65,6 +65,9 @@ class Detection { 'category': category.name, 'evidence': [for (final e in evidence) e.toMap()], 'hosts': hosts.toList(), + // The cookies this service set; the evidence list is capped and may + // not show them all. + if (cookies.isNotEmpty) 'cookies': cookies.toList(), 'inSource': inSource, if (version != null) 'version': version, }; @@ -80,6 +83,7 @@ class Detection { ..inSource = raw['inSource'] == true ..version = raw['version']?.toString(); d.hosts.addAll([for (final h in raw['hosts'] as List? ?? []) '$h']); + d.cookies.addAll([for (final c in raw['cookies'] as List? ?? []) '$c']); for (final e in raw['evidence'] as List? ?? []) { if (DetectionEvidence.fromMap(e) case final item?) d.evidence.add(item); } diff --git a/lib/services/html_service.dart b/lib/services/html_service.dart index a12a2e54..1cd45467 100644 --- a/lib/services/html_service.dart +++ b/lib/services/html_service.dart @@ -796,8 +796,10 @@ class HtmlService extends ChangeNotifier { } List? _divergences; - ({Observed browser, Observed server})? - _divergencesFor; + ({ + Observed browser, + Observed server + })? _divergencesFor; /// Where the app's own request and the browser disagree (UI: Task 10). /// Cached on the two snapshots it reads, not on the PageInspection object. @@ -869,8 +871,10 @@ class HtmlService extends ChangeNotifier { } Observed? _cookiesView; - ({Observed browser, Observed server})? - _cookiesViewFor; + ({ + Observed browser, + Observed server + })? _cookiesViewFor; /// The Cookies tab: derived cookies with flags (unknown stays unknown) /// and the privacy lens. Cached on the snapshots it reads, not on the @@ -1056,12 +1060,16 @@ class HtmlService extends ChangeNotifier { } /// What the analysis reads from the browser: whether it is Present, the - /// URL it is on and how many resources it reported. A change after the - /// analysis started makes the report stale. + /// URL it is on, how many resources it reported and which services were + /// detected (a tracker known only from a cookie that arrived later must + /// reach the privacy pillar too). A change after the analysis started + /// makes the report stale. String get _currentSovereigntyInputKey { final browser = store.current.browser; final count = SovereigntyService.browserResources(store.current).length; - return '${browser is Present}|$_sovereigntyUri|$count'; + final services = (detections.map((d) => d.name).toList()..sort()).join(','); + return '${browser is Present}|$_sovereigntyUri|$count' + '|$services'; } /// Registrable domain the per-site jurisdiction choice is stored under. @@ -4422,7 +4430,8 @@ Technical details: $message'''; // image resize options (?width=112 vs ?width=264 or none) must NOT // match: they serve different bytes (e.g. thumbnail vs full original). if (u.path.length > 3) { - final byCb = byResourceKey['$host|${u.path}|${queryWithoutCacheBusters(u)}']; + final byCb = + byResourceKey['$host|${u.path}|${queryWithoutCacheBusters(u)}']; if (byCb != null) return _sizeOf(byCb); } @@ -4873,8 +4882,7 @@ Technical details: $message'''; // Hidden sizes, like the live load (shared lookup cache per page). final urls = estimateTargetsOf(store.current.freshLoad, const {}, - limit: _maxEstimateLookups, - declaredMedia: store.current.declaredMedia); + limit: _maxEstimateLookups, declaredMedia: store.current.declaredMedia); var next = 0; Future worker() async { while (next < urls.length && !stale()) { @@ -5247,7 +5255,8 @@ Technical details: $message'''; // A load poll that changed neither the resources nor the document has // nothing to apply: skip the rebuild (the common case while polling). final sig = _browserSignature(); - if (sig == _lastAppliedBrowserSig && identical(domText, _lastAppliedDomText)) { + if (sig == _lastAppliedBrowserSig && + identical(domText, _lastAppliedDomText)) { return; } _lastAppliedBrowserSig = sig; @@ -5304,7 +5313,8 @@ class _ControllerBrowserLoader implements BrowserLoader { @override void loadUrl(Uri url) { final controller = _service.activeWebViewController; - if (controller == null) return; // still pending; the view loads it on create + if (controller == null) + return; // still pending; the view loads it on create _service._browserLoadIssuedUrl = url.toString(); controller .loadUrl(urlRequest: URLRequest(url: WebUri(url.toString()))) diff --git a/lib/services/sovereignty_service.dart b/lib/services/sovereignty_service.dart index 658dcf3f..ee3b10bd 100644 --- a/lib/services/sovereignty_service.dart +++ b/lib/services/sovereignty_service.dart @@ -13,6 +13,7 @@ import 'package:view_source_vibe/services/derive/page_weight.dart'; import 'package:view_source_vibe/services/derive/security_audit.dart' show hstsMaxAge, hstsMinSeconds; import 'package:view_source_vibe/services/detection/engine.dart'; +import 'package:view_source_vibe/services/derive/score_band.dart'; class HostingProvider { final String name; @@ -502,11 +503,11 @@ class SovereigntyReport { String? get grade { final s = score; if (s == null) return null; - return s >= 80 - ? 'Good' - : s >= 50 - ? 'Fair' - : 'Poor'; + return switch (sovereigntyScale.band(s)!) { + ScoreBand.good => 'Good', + ScoreBand.fair => 'Fair', + ScoreBand.poor => 'Poor', + }; } int count(CheckStatus status) => @@ -1050,7 +1051,9 @@ class SovereigntyService { ? 'CMS: ${cms.name} (open source)' : 'CMS: ${cms.name}${origin != null ? ' ($origin)' : ''}', status, - cms.evidence.firstOrNull?.detail ?? 'Detected in the page source', + // The title says where ("HTML line 414: "images.ctfassets.net""); + // the detail is a raw source excerpt that reads as noise here. + cms.evidence.firstOrNull?.title ?? 'Detected in the page source', weight: 1, alternatives: status == CheckStatus.fail ? alternativesFor('cms', home) @@ -1224,11 +1227,11 @@ class SovereigntyService { final privacy = html == null ? null : findPrivacyLink(html); final consent = _detectedServiceNames(detections, {DetectionCategory.consent}); + // The same trackers as the Privacy tab: every tracking category, + // social widgets included (they follow visitors across sites). final trackers = _detectedServiceNames(detections, { - DetectionCategory.advertising, - DetectionCategory.behavior, - DetectionCategory.analytics, - DetectionCategory.marketing, + for (final c in DetectionCategory.values) + if (c.isTracking) c }).toSet().toList(); final fonts = { ..._detectedServiceNames(detections, {DetectionCategory.fonts}), @@ -1272,15 +1275,14 @@ class SovereigntyService { trackers.isEmpty ? 'None detected' : trackers.join(', '), ), _servicesCheck( - 'Analytics, ads & marketing in the site\'s jurisdiction', + 'Trackers in the site\'s jurisdiction', trackers, 2, home, ), _servicesCheck( - 'Widgets, embeds & payments in the site\'s jurisdiction', - _detectedServiceNames(detections, - {DetectionCategory.social, DetectionCategory.ecommerce}), + 'Payments & e-commerce in the site\'s jurisdiction', + _detectedServiceNames(detections, {DetectionCategory.ecommerce}), 1, home, ), @@ -1603,7 +1605,13 @@ class SovereigntyService { final rtype = u16(pos); final rdlen = u16(pos + 8); final rdata = pos + 10; - if (rtype == type && rdata + rdlen <= d.length) { + // A null MX (RFC 7505: exchange ".") means the domain takes no mail; + // it is not a mail server. + final nullMx = type == 15 && + rtype == 15 && + rdata + rdlen <= d.length && + readName(rdata + 2).$1.isEmpty; + if (rtype == type && rdata + rdlen <= d.length && !nullMx) { out.add(switch (type) { 15 => readName(rdata + 2).$1, 16 => _txt(d.sublist(rdata, rdata + rdlen)), diff --git a/lib/widgets/probe_views.dart b/lib/widgets/probe_views.dart index fe9b9078..f0be7f0a 100644 --- a/lib/widgets/probe_views.dart +++ b/lib/widgets/probe_views.dart @@ -14,6 +14,8 @@ import 'package:view_source_vibe/services/derive/probe_general.dart'; import 'package:view_source_vibe/services/derive/security_audit.dart'; import 'package:view_source_vibe/services/html_service.dart'; import 'package:view_source_vibe/utils/format_utils.dart'; +import 'package:view_source_vibe/services/derive/score_band.dart'; +import 'package:view_source_vibe/widgets/score_band_color.dart'; // --------------------------------------------------------------------------- // Shared pieces of the migrated probe tabs @@ -1133,14 +1135,7 @@ class ProbeSecurityView extends StatelessWidget { final audit = state.value; final percentage = audit.percentage!; final score = percentage / 100; - final Color scoreColor; - if (percentage >= 80) { - scoreColor = Colors.green; - } else if (percentage >= 50) { - scoreColor = Colors.orange; - } else { - scoreColor = Colors.red; - } + final scoreColor = bandColor(securityScale.band(percentage)); return _card( context, @@ -1262,13 +1257,7 @@ class ProbeSecurityView extends StatelessWidget { } final csp = state.value; final score = csp.score; - final color = score == null - ? Colors.grey - : score >= 80 - ? Colors.green - : score >= 50 - ? Colors.orange - : Colors.red; + final color = bandColor(cspScale.band(score)); return _card( context, @@ -1708,13 +1697,7 @@ class ProbeCookiesView extends StatelessWidget { Widget _buildCookiePrivacyLens(BuildContext context, CookiesModel m) { final score = m.score; - final color = score == null - ? Colors.grey - : score >= 75 - ? Colors.green - : score >= 45 - ? Colors.orange - : Colors.red; + final color = bandColor(cookieScale.band(score)); return _card( context, diff --git a/lib/widgets/score_band_color.dart b/lib/widgets/score_band_color.dart new file mode 100644 index 00000000..d1a80906 --- /dev/null +++ b/lib/widgets/score_band_color.dart @@ -0,0 +1,10 @@ +import 'package:flutter/material.dart'; +import 'package:view_source_vibe/services/derive/score_band.dart'; + +/// The colour of a score card: the same band as the model's verdict text. +Color bandColor(ScoreBand? band) => switch (band) { + ScoreBand.good => Colors.green, + ScoreBand.fair => Colors.orange, + ScoreBand.poor => Colors.red, + null => Colors.grey, + }; diff --git a/lib/widgets/sovereignty_view.dart b/lib/widgets/sovereignty_view.dart index 3ad55526..710e8e15 100644 --- a/lib/widgets/sovereignty_view.dart +++ b/lib/widgets/sovereignty_view.dart @@ -7,6 +7,8 @@ import 'package:view_source_vibe/services/html_service.dart'; import 'package:view_source_vibe/services/jurisdiction.dart'; import 'package:view_source_vibe/services/sovereignty_service.dart'; import 'package:view_source_vibe/utils/format_utils.dart'; +import 'package:view_source_vibe/services/derive/score_band.dart'; +import 'package:view_source_vibe/widgets/score_band_color.dart'; class SovereigntyView extends StatefulWidget { const SovereigntyView({super.key}); @@ -237,13 +239,7 @@ class _SovereigntyViewState extends State { Widget _scoreCard(BuildContext context, SovereigntySummary summary, SovereigntyReport report) { final score = summary.score; - final color = score == null - ? Colors.grey - : score >= 80 - ? Colors.green - : score >= 50 - ? Colors.orange - : Colors.red; + final color = bandColor(sovereigntyScale.band(score)); final counters = { CheckStatus.pass: 'passed', CheckStatus.warn: 'warnings', @@ -279,8 +275,7 @@ class _SovereigntyViewState extends State { ], ), if (score != null) - Text( - 'based on ${(summary.coverage * 100).round()}% of the checks', + Text('based on ${(summary.coverage * 100).round()}% of the checks', style: const TextStyle(fontSize: 12, color: Colors.grey)), const SizedBox(height: 4), Wrap(spacing: 12, children: [ diff --git a/pubspec.lock b/pubspec.lock index 1cb500c6..ef81141c 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -334,6 +334,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" flutter_fancy_tree_view2: dependency: "direct main" description: @@ -450,6 +455,11 @@ packages: description: flutter source: sdk version: "0.0.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -522,6 +532,11 @@ packages: url: "https://pub.dev" source: hosted version: "4.12.0" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" io: dependency: transitive description: @@ -610,6 +625,14 @@ packages: url: "https://pub.dev" source: hosted version: "6.1.0" + listen: + dependency: transitive + description: + name: listen + sha256: "47501a08016a43fcad79252439d723f50f14f88fa7bfd8a177e0a417e5c9e1f2" + url: "https://pub.dev" + source: hosted + version: "1.0.1" logging: dependency: transitive description: @@ -794,6 +817,14 @@ packages: url: "https://pub.dev" source: hosted version: "6.5.4" + process: + dependency: transitive + description: + name: process + sha256: "4242ba3508d37e01808bdf71ad1d5bb93a8d671bf2e7450e6b1b353fb0808891" + url: "https://pub.dev" + source: hosted + version: "5.0.6" provider: dependency: "direct main" description: @@ -967,6 +998,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.1" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -979,10 +1018,10 @@ packages: dependency: transitive description: name: test_api - sha256: "2a122cbe059f8b610d3a5415f42e255b6c17b1f21eee1d960f31080237fb4f11" + sha256: "0a10344e901e5b2e63819567951cb6a06673ed6b84f40462188ff5a0c41f371f" url: "https://pub.dev" source: hosted - version: "0.7.12" + version: "0.7.14" typed_data: dependency: transitive description: @@ -1135,6 +1174,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "28b82ec894fed45dd71c23ba62d1af973ed97dd59a4f5790a4d38b0b13e5657e" + url: "https://pub.dev" + source: hosted + version: "3.2.0" win32: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 617c83c6..5b49fbb8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -64,6 +64,8 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter + integration_test: + sdk: flutter flutter_lints: ^6.0.0 mockito: ^5.8.1 build_runner: ^2.16.1 diff --git a/test/derive/cookies_test.dart b/test/derive/cookies_test.dart index f05cfaf6..c7cfcc9e 100644 --- a/test/derive/cookies_test.dart +++ b/test/derive/cookies_test.dart @@ -43,6 +43,35 @@ void main() { expect(c.where((x) => x.domain == 'other.test'), hasLength(1)); }); + // Validation run 2026-11-13 (kvk.nl, nl.wikipedia.org): WebKit reports a + // cookie whose Set-Cookie has no SameSite as "None". Merged, that made the + // same cookie "None" (weak) when the browser saw it and "not set" when + // only the server did. + group('SameSite of a merged cookie', () { + List merged(String header, String browser) => deriveCookies( + inspection( + server: serverSnap(setCookies: [header]), + browser: browserSnap(cookies: [ + cookie('a', sameSite: b(browser)), + ])), + now: now) + .map((c) => c.sameSite) + .toList(); + + test('header without SameSite + browser "None" stays not set', () { + expect(merged('a=1', 'None'), [null]); + }); + + test('header without SameSite + browser "Lax" keeps the browser value', + () { + expect(merged('a=1', 'Lax'), ['Lax']); + }); + + test('the header value wins', () { + expect(merged('a=1; SameSite=Strict', 'None'), ['Strict']); + }); + }); + test('unknown browser flags stay null, never false', () { final c = deriveCookies( inspection(browser: browserSnap(cookies: [cookie('x')])), diff --git a/test/derive/fixtures.dart b/test/derive/fixtures.dart index ae8fc543..1c8384ae 100644 --- a/test/derive/fixtures.dart +++ b/test/derive/fixtures.dart @@ -41,7 +41,9 @@ BrowserCookie cookie(String name, Observed secure = const Unavailable(UnavailableReason.notExposedToPage), Observed httpOnly = - const Unavailable(UnavailableReason.notExposedToPage)}) => + const Unavailable(UnavailableReason.notExposedToPage), + Observed sameSite = + const Unavailable(UnavailableReason.notReportedByPlatform)}) => BrowserCookie( name: name, value: value, @@ -50,7 +52,7 @@ BrowserCookie cookie(String name, expires: expires, isSecure: secure, isHttpOnly: httpOnly, - sameSite: const Unavailable(UnavailableReason.notReportedByPlatform)); + sameSite: sameSite); BrowserSnapshot browserSnap({ String url = 'https://a.test/page', diff --git a/test/derive/privacy_consistency_test.dart b/test/derive/privacy_consistency_test.dart new file mode 100644 index 00000000..2174d756 --- /dev/null +++ b/test/derive/privacy_consistency_test.dart @@ -0,0 +1,167 @@ +// The same trackers must be told the same way on every page that shows +// them: the Cookie Privacy Lens (Security tab), Services/Privacy +// (detections) and the Sovereignty privacy pillar. +import 'package:flutter_test/flutter_test.dart'; +import 'package:view_source_vibe/models/detection_evidence.dart'; +import 'package:view_source_vibe/models/inspection/observed.dart'; +import 'package:view_source_vibe/models/inspection/page_inspection.dart'; +import 'package:view_source_vibe/services/derive/cookies_view.dart'; +import 'package:view_source_vibe/services/derive/detections.dart'; +import 'package:view_source_vibe/services/detection/catalog.dart'; +import 'package:view_source_vibe/services/detection/engine.dart'; +import 'package:view_source_vibe/services/jurisdiction.dart'; +import 'package:view_source_vibe/services/sovereignty_service.dart'; + +import '../fixtures/inspection/builders.dart'; + +const _url = 'https://news.example.com/'; +const _html = 'News' + '' + '' + '' + 'News'; + +/// A news page with trackers seen three ways: in the source, as requests +/// and as first-party cookies (catalogued, Open Cookie Database only, and +/// the site's own). +PageInspection _page() { + final h = hop(_url, 200, headers: { + 'content-type': ['text/html; charset=utf-8'] + }, setCookies: [ + '_ga=GA1.1.1.1; Domain=.news.example.com; Path=/; Secure; SameSite=Lax', + '_fbp=fb.1.1.1; Domain=.news.example.com; Path=/; Secure; SameSite=Lax', + '_gcl_au=1.1.1.1; Domain=.news.example.com; Path=/; Secure', + '_hjSessionUser_123=x; Path=/; Secure', + '_pk_id.1.abcd=x; Path=/; Secure', + 'cto_bundle=x; Path=/; Secure', + 'sessionid=x; Path=/; Secure; HttpOnly', + ]); + return urlInspection( + url: _url, + serverResult: + server(serverSnapshot(url: _url, hops: [h], body: _html)), + browserResult: browser(browserSnapshot( + url: _url, + dom: _html, + resources: [ + resource('https://connect.facebook.net/en_US/fbevents.js', 'script'), + resource('https://www.google-analytics.com/g/collect?v=2', 'beacon'), + resource('https://static.hotjar.com/c/hotjar-1.js', 'script'), + ], + cookies: const [], + )), + ); +} + +CookiesModel _lens(PageInspection p) => + (deriveCookiesView(p) as Present).value; + +List _privacy(List d) => + SovereigntyService.privacyChecks(const [], d, _html, eu); + +/// The names in a Sovereignty check's evidence ("A, B" or "None detected"). +Set _evidenceNames(SovereigntyCheck c) => c.evidence == + 'None detected' + ? {} + : c.evidence.split(', ').map((s) => s.trim()).toSet(); + +const _trackerCookieCategories = {'advertising', 'analytics', 'social'}; + +void main() { + final page = _page(); + final lens = _lens(page); + final detections = deriveDetections(page); + final privacy = _privacy(detections); + + test('the fixture has tracker cookies, tracker detections and consent', () { + expect(lens.rows.where((r) => _trackerCookieCategories.contains(r.category)), + hasLength(greaterThanOrEqualTo(4))); + expect(detections.where((d) => d.category.isTracking), + hasLength(greaterThanOrEqualTo(3))); + expect(detections.any((d) => d.category == DetectionCategory.consent), + isTrue); + }); + + test('every tracking cookie in the Cookie Lens is a Services detection', () { + for (final row in lens.rows) { + if (!_trackerCookieCategories.contains(row.category) || + row.provider == null) { + continue; + } + final owner = detections.where((d) => d.cookies.contains(row.name)); + expect(owner, isNotEmpty, + reason: '${row.name} (${row.category}, ${row.provider}) is in the ' + 'Cookie Lens but no service on Services/Privacy claims it'); + } + }); + + test('a tracking cookie and its service agree on the kind of tracking', () { + for (final d in detections) { + for (final name in d.cookies) { + final row = lens.rows.firstWhere((r) => r.name == name); + if (!_trackerCookieCategories.contains(row.category)) continue; + expect(d.category.isTracking, isTrue, + reason: '$name is ${row.category} in the Cookie Lens but ' + '${d.name} is ${d.category.label} on Services'); + } + } + }); + + test('the site\'s own cookie is not a tracker anywhere', () { + final own = lens.rows.firstWhere((r) => r.name == 'sessionid'); + expect(_trackerCookieCategories, isNot(contains(own.category))); + expect(detections.any((d) => d.cookies.contains('sessionid')), isFalse); + }); + + test('Sovereignty counts the trackers that the Privacy tab lists', () { + final listed = { + for (final d in detections) + if (d.category.isTracking) d.name + }; + final check = privacy.firstWhere((c) => c.title == 'Number of trackers'); + expect(_evidenceNames(check), listed); + }); + + test('the Privacy tab and Sovereignty give the same tracker count', () { + final privacyTab = detections.where((d) => d.category.isTracking).length; + final check = privacy.firstWhere((c) => c.title == 'Number of trackers'); + // Social widgets are trackers on both pages (decided 2026-11-13). + expect(_evidenceNames(check).length, privacyTab); + }); + + test('a social widget is judged once in Sovereignty, as a tracker', () { + final social = detections + .where((d) => d.category == DetectionCategory.social) + .map((d) => d.name) + .toList(); + expect(social, isNotEmpty); + final mentions = privacy + .where((c) => social.any((n) => c.evidence.contains(n))) + .map((c) => c.title) + .toSet(); + expect(mentions, isNot(contains(startsWith('Payments')))); + }); + + // Validation run 2026-11-13 (allbirds.com): Google Ads had 12 request + // evidence items, so its _gcl_au cookie was not in the export. + test('the export names every cookie a service set, past the evidence cap', + () { + final d = Detection(entityNamed('Google Ads')!)..cookies.add('_gcl_au'); + for (var i = 0; i < 20; i++) { + d.add(DetectionEvidence(EvidenceKind.request, 'Request $i')); + } + final map = d.toMap(); + expect(map['cookies'], ['_gcl_au']); + expect(Detection.fromMap(map)!.cookies, {'_gcl_au'}); + }); + + test('a detected consent tool is the one Sovereignty names', () { + final consent = { + for (final d in detections) + if (d.category == DetectionCategory.consent) d.name + }; + final check = privacy.firstWhere((c) => c.title == 'Consent management'); + expect(check.status, CheckStatus.pass); + expect(_evidenceNames(check), consent); + }); +} diff --git a/test/derive/score_band_test.dart b/test/derive/score_band_test.dart new file mode 100644 index 00000000..7cda8107 --- /dev/null +++ b/test/derive/score_band_test.dart @@ -0,0 +1,57 @@ +// A score's colour (widgets) and its verdict text (models) come from one +// scale, so they cannot disagree at a boundary. +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:view_source_vibe/services/derive/score_band.dart'; +import 'package:view_source_vibe/widgets/score_band_color.dart'; + +void main() { + group('boundaries', () { + final cases = { + 'security': (securityScale, 80, 50), + 'csp': (cspScale, 80, 50), + 'sovereignty': (sovereigntyScale, 80, 50), + 'cookies': (cookieScale, 75, 45), + }; + for (final MapEntry(key: name, value: (scale, good, fair)) + in cases.entries) { + test(name, () { + expect(scale.band(100), ScoreBand.good); + expect(scale.band(good), ScoreBand.good); + expect(scale.band(good - 1), ScoreBand.fair); + expect(scale.band(fair), ScoreBand.fair); + expect(scale.band(fair - 1), ScoreBand.poor); + expect(scale.band(0), ScoreBand.poor); + expect(scale.band(null), isNull); + }); + } + }); + + test('each band has its own colour; no score is grey', () { + expect(bandColor(ScoreBand.good), Colors.green); + expect(bandColor(ScoreBand.fair), Colors.orange); + expect(bandColor(ScoreBand.poor), Colors.red); + expect(bandColor(null), Colors.grey); + }); + + test('score cards take their colour from the shared scales', () { + // A hard-coded "score >= 80" in a widget would drift from the model's + // verdict the next time a threshold changes. + final hardCoded = RegExp(r'\b(score|percentage)\s*>=\s*\d{2}\b'); + for (final path in [ + 'lib/widgets/probe_views.dart', + 'lib/widgets/sovereignty_view.dart', + 'lib/services/sovereignty_service.dart', + 'lib/services/derive/cookies_view.dart', + 'lib/services/derive/security_audit.dart', + ]) { + final hits = hardCoded + .allMatches(File(path).readAsStringSync()) + .map((m) => m.group(0)) + .toList(); + expect(hits, isEmpty, reason: '$path has its own thresholds: $hits'); + } + }); +} diff --git a/test/derive/security_audit_test.dart b/test/derive/security_audit_test.dart index 4fde9355..327dbf57 100644 --- a/test/derive/security_audit_test.dart +++ b/test/derive/security_audit_test.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + import 'package:flutter_test/flutter_test.dart'; import 'package:view_source_vibe/models/inspection/observed.dart'; import 'package:view_source_vibe/models/inspection/page_inspection.dart'; @@ -182,6 +184,46 @@ void main() { CertTone.bad); }); + // Real certificate of www.rtvutrecht.nl (validation run 2026-11-13): + // CN=rtvutrecht.nl, SAN covers www.rtvutrecht.nl. + final realPem = + File('test/fixtures/certs/rtvutrecht.pem').readAsStringSync(); + + test('reads the DNS names of the subjectAltName extension', () { + expect(subjectAltNames(realPem), + ['rtvutrecht.nl', 'www.rtvutrecht.nl', 'utrecht.regiogroei.cloud']); + expect(subjectAltNames(null), isEmpty); + expect(subjectAltNames('-----BEGIN-----'), isEmpty); + }); + + test('a host covered by the SAN matches, even when the CN differs', () { + final c = certOf( + page({}, + url: 'https://www.rtvutrecht.nl/', + certificate: server({ + ...cert, + 'subjectParsed': {'Common Name': 'rtvutrecht.nl'}, + 'pem': realPem, + })), + now: DateTime.utc(2026, 6, 1)); + expect(c.hostMatches, isTrue); + expect(c.hostLabel, 'SAN matches host'); + }); + + test('a host in neither CN nor SAN differs', () { + final c = certOf( + page({}, + url: 'https://other.test/', + certificate: server({ + ...cert, + 'subjectParsed': {'Common Name': 'rtvutrecht.nl'}, + 'pem': realPem, + })), + now: DateTime.utc(2026, 6, 1)); + expect(c.hostMatches, isFalse); + expect(c.hostLabel, 'Host not in certificate (CN or SAN)'); + }); + test('no common name is unknown, not "differs"', () { final c = certOf( page({}, certificate: server({...cert, 'subjectParsed': {}})), @@ -446,14 +488,14 @@ void main() { .certificate as Present) .value; - test('a CN mismatch says SAN was not checked', () { + test('a CN mismatch without SAN data says SAN is not available', () { final c = certOf({ 'subjectParsed': {'Common Name': 'Some Org'}, 'issuerParsed': {}, 'endValidity': '2027-01-01T00:00:00Z', }); expect(c.hostMatches, isFalse); - expect(c.hostLabel, 'CN differs from host (SAN not checked)'); + expect(c.hostLabel, 'CN differs from host (SAN not available)'); }); }); } diff --git a/test/fixtures/certs/rtvutrecht.pem b/test/fixtures/certs/rtvutrecht.pem new file mode 100644 index 00000000..264e31ba --- /dev/null +++ b/test/fixtures/certs/rtvutrecht.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF6DCCBNCgAwIBAgIQBDD4obv9zyZJH4k39ugs7jANBgkqhkiG9w0BAQsFADA8 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24g +UlNBIDIwNDggTTA0MB4XDTI2MDEzMDAwMDAwMFoXDTI3MDIyNzIzNTk1OVowGDEW +MBQGA1UEAxMNcnR2dXRyZWNodC5ubDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAMXfQVaUlkWhYhfDXuY3o7IKqiiDg3DCadv+u15kv+r7U6fRGFUiRlFw +PbzOcK8CKH3uami+3rPKz1NpkkwwCnSOlcWk3FNTV1a2oe1vNlS8f9KLB72Sm3S9 +gVnI3379gpo5FsUa8RH/UA479xc9nunklutJ82VntO4QPaXGWK1fSMnMrMj7I0en ++bG0wVwCUyXchrZzYhiCYv7d//+MUR0yf/eYXVMniMBvEtvg+5g9lRrv3gv23+MF +/GPEvvyB3hnQFvNu/iMjbPD7iI/UKe4jMlgM3pCUszxwTqJm9x8gU4vH1ZQQ6fBS +EClzjWT2FIKCgZ89HiOoWvf7DcVEUQsCAwEAAaOCAwgwggMEMB8GA1UdIwQYMBaA +FB9SkmFWglR/gWbYHT0KqjJch90IMB0GA1UdDgQWBBR8h5EIpvfGOOYZtDzhX/4Y +maw2SDBFBgNVHREEPjA8gg1ydHZ1dHJlY2h0Lm5sghF3d3cucnR2dXRyZWNodC5u +bIIYdXRyZWNodC5yZWdpb2dyb2VpLmNsb3VkMBMGA1UdIAQMMAowCAYGZ4EMAQIB +MA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATA7BgNVHR8ENDAy +MDCgLqAshipodHRwOi8vY3JsLnIybTA0LmFtYXpvbnRydXN0LmNvbS9yMm0wNC5j +cmwwdQYIKwYBBQUHAQEEaTBnMC0GCCsGAQUFBzABhiFodHRwOi8vb2NzcC5yMm0w +NC5hbWF6b250cnVzdC5jb20wNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQucjJtMDQu +YW1hem9udHJ1c3QuY29tL3IybTA0LmNlcjAMBgNVHRMBAf8EAjAAMIIBfQYKKwYB +BAHWeQIEAgSCAW0EggFpAWcAdQBMY9yY5Zwdq4j2Hoo93q6Pq0SjN3tfm5TD+6Gc +/MG+JgAAAZwM8IwpAAAEAwBGMEQCICtALFjX0ZqtI7Bl4JJLdgDsX3s/2pIuswQr +bCA/CCtvAiA6ajNJprCWEe5mRGjI6MQlLMO9eQKQeFmM7JstoWXo7wB2AByfaCzp ++vBFaVD4G5aKh93bMhDYTObIsuOCUkrEz1mfAAABnAzwi4wAAAQDAEcwRQIhAPPq +/iZKJ2lU/DNjU2fHHrKUtSjX48zHWCX8UMOeN9I8AiArK314MDGOAdVAoCmeNRIo +88rqk/l7u2bc75aS4V2WBAB2AGBMmq96f3dfAdQG/JINyJnrCxx9+MlSG/r6F3c7 +l4vJAAABnAzwjEQAAAQDAEcwRQIgZyiBJrL/+qW5wixP3xTxls/TdM9yU9TzDh0K +eboKudoCIQDbDM+ndAxR31VIZ3ZiYrTbdJ5WhoPsCEBh9BsqandUlTANBgkqhkiG +9w0BAQsFAAOCAQEALPdHhsA1o9wJpsLkHmlFgUYL9ZUhCi5sIYJ2Xt7KUAZq2qS+ +f7jdMzdX1x3xX99D8jq6kFv6PyoqxSBd/cGLl1DlHZvAc2BnmeDlFVwp+wdL3n4l +scJrhOPZ8KJafwRZapyDpgihtnH/9TI8jdZpnFQmCUcnuCcGiUwF3o2Hu6k0VLPp +uDwvu/kT7KEYmhDf1soaj3RQ8Fy67EHFzokYmDDo4NAyVyHOvLgKwhCEJ8ETLZOK +KHWDxEhn4hQXGfgVcxxn07eEcU3EmZbJ18vKet/1m3IC1NnHT8u4HCCMwtMjoNzW +5qAJqbTk0E/jwWrQAtrYv7PwDKu3GflbMzI5hw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/test/services/sovereignty_service_test.dart b/test/services/sovereignty_service_test.dart index 554e341d..6eaf9913 100644 --- a/test/services/sovereignty_service_test.dart +++ b/test/services/sovereignty_service_test.dart @@ -140,6 +140,17 @@ void main() { ['ns-311.awsdns-38.com', 'ns-823.awsdns-38.com']); }); + test('a null MX (RFC 7505, "0 .") is no mail server', () { + // example.com MX: preference 0, exchange = root. + final data = Uint8List.fromList([ + 0, 0, 0x81, 0x80, 0, 1, 0, 1, 0, 0, 0, 0, + 7, 101, 120, 97, 109, 112, 108, 101, 3, 99, 111, 109, 0, 0, 15, 0, 1, + 0xC0, 0x0C, 0, 15, 0, 1, 0, 0, 0x0E, 0x10, 0, 3, // answer header + 0, 0, 0, // preference 0, exchange "." + ]); + expect(SovereigntyService.parseDnsRecords(data, 15), isEmpty); + }); + test('tolerates garbage', () { expect(SovereigntyService.parseDnsRecords(Uint8List(3), 2), isEmpty); }); diff --git a/test/services/sovereignty_state_test.dart b/test/services/sovereignty_state_test.dart index c4b8d966..a410bd18 100644 --- a/test/services/sovereignty_state_test.dart +++ b/test/services/sovereignty_state_test.dart @@ -1,4 +1,5 @@ import 'package:flutter_test/flutter_test.dart'; +import 'package:view_source_vibe/models/inspection/browser_snapshot.dart'; import 'package:view_source_vibe/models/inspection/observed.dart'; import 'package:view_source_vibe/models/inspection/page_inspection.dart'; import 'package:view_source_vibe/models/inspection/server_snapshot.dart'; @@ -188,6 +189,52 @@ void main() { expect(calls, hasLength(3)); }); + // Validation run 2026-11-13 (rijksoverheid.nl): a tracker known only + // from a cookie that arrived later was on Services but not in the + // Sovereignty privacy pillar. + test('a tracker found later (by cookie only) re-runs the analysis', + () async { + final p = fullPage(); + seedStore(store, p.copyWith(browser: const Pending())); + service.watchSovereignty(true); + await pumpEventQueue(); + final gen = store.current.generation; + final res = [resource('https://cdn.test/a.js', 'script')]; + store.applyBrowser( + gen, + browserSnapshot( + url: 'https://example.com/', + dom: '', + resources: res, + cookies: const [])); + await pumpEventQueue(); + final before = calls.length; + + // Same resources; a Google Analytics cookie appears. + store.applyBrowser( + gen, + browserSnapshot( + url: 'https://example.com/', + dom: '', + resources: res, + cookies: [ + BrowserCookie( + name: '_ga', + value: 'GA1.1.1.1', + domain: browser('.example.com'), + path: browser('/'), + expires: const Unavailable( + UnavailableReason.notReportedByPlatform), + isSecure: browser(true), + isHttpOnly: browser(false), + sameSite: browser('Lax'), + ), + ])); + await pumpEventQueue(); + expect(calls.length, before + 1, + reason: 'the detections changed, so the privacy pillar must too'); + }); + test('a changed legal home re-runs the analysis', () async { final settings = AppSettings(); service.setAppSettings(settings); diff --git a/validation/README.md b/validation/README.md new file mode 100644 index 00000000..8359300c --- /dev/null +++ b/validation/README.md @@ -0,0 +1,66 @@ +# Site validation + +Checks whether what the app reports about real sites is true. The app loads +each site in `sites.yaml` and exports its inspection package; independent +tools measure the same site twice; a comparer lists only the differences. + +```bash +validation/run.sh # all sites, ~10–15 min +validation/run.sh example kvk # some slugs +``` + +Needs: macOS, Flutter, [uv](https://docs.astral.sh/uv/), and once +`uv run --with playwright playwright install webkit`. + +## What happens + +1. **App pass** — `integration_test/site_validation_test.dart` starts the real + app (debug, macOS), clears cookies and cache, loads each URL, waits until + the page is settled and every analysis is done (max 90 s), and writes + `runs///app.zip` (the normal export) plus `app-meta.json`. +2. **Reference R1 and R2** — `reference.py` measures every site without the + app's code: curl (redirects, headers, Set-Cookie, robots.txt, with the + app's own User-Agent), openssl (certificate), Playwright WebKit (cookies, + storage, requests, rendered DOM → BeautifulSoup metadata), dig and Team + Cymru whois (hosting AS, NS, MX). Two runs show what varies by itself. +3. **Compare** — `compare.py` writes `runs//report.md` (site × domain + matrix, then every deviation with both values and the file in the zip) and + `summary.json` (to diff runs before and after a fix). + +Neither side clicks a consent banner: both measure the pre-consent page. + +## Outcomes + +| | meaning | +|---|---| +| ✅ ok | app matches the reference | +| 〰️ volatile | R1 and R2 differ and the app is within them | +| ⚠️ gap | app says unavailable where the reference has a value — check the reason; a real platform limit is correct behaviour | +| 👻 phantom | app reports something the reference does not see | +| ❌ wrong | app value differs | + +The comparer checks *inputs* (headers, cookies, hosts, records) exactly and +scores only indirectly; the score formulas are audited in +`docs/audits/2026-11-berekeningen.md`. + +## Triage + +Classify each ❌/👻/⚠️ as **app bug**, **reference limit**, **site variation** +or **platform limit**. Only app bugs become work: + +1. Copy `debug/browser-payload.json` from the zip to + `test/fixtures/browser/.json` (and headers to a fixture in + `test/fixtures/inspection/` if the server side is involved). +2. Write a failing offline test in `test/derive/` or `test/collectors/`. +3. Fix, `flutter test`, rerun `validation/run.sh ` and compare + `summary.json` with the previous run. + +## Adding a site or check + +- Site: add an entry to `sites.yaml` with `why`, and set + `expect_detections` / `forbid_detections` by checking the site once by + hand (Wappalyzer/BuiltWith in a normal browser). Names must match the + app's detection names. +- Check: measure the value in `reference.py`, add a line in + `compare.py:check_site` with `compare_exact`, `compare_set` or + `compare_num`. diff --git a/validation/compare.py b/validation/compare.py new file mode 100644 index 00000000..62b0949b --- /dev/null +++ b/validation/compare.py @@ -0,0 +1,560 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["pyyaml"] +# /// +"""Compares the app's inspection packages with the reference measurements. + + uv run validation/compare.py validation/runs/ + +Writes /report.md and /summary.json. Outcomes per check: + ok app matches the reference + volatile the two reference runs differ and the app is within them + wrong app value differs from the reference + phantom app reports something the reference does not see + gap app says unavailable/failed where the reference has a value +""" + +from __future__ import annotations + +import argparse +import json +import re +import zipfile +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +HERE = Path(__file__).parent +SECURITY_HEADERS = [ + "strict-transport-security", + "content-security-policy", + "x-frame-options", + "x-content-type-options", + "referrer-policy", + "permissions-policy", +] +DOMAINS = ["app", "redirects", "headers", "security", "certificate", "cookies", + "browser", "metadata", "robots", "sovereignty", "detections", "consistency"] +RANK = {"ok": 0, "volatile": 1, "gap": 2, "phantom": 3, "wrong": 4} +ICON = {"ok": "✅", "volatile": "〰️", "gap": "⚠️", "phantom": "👻", "wrong": "❌"} + + +@dataclass +class Unavail: + """An app value that is not present: status + reason from the export.""" + status: str + reason: str = "" + + def __str__(self) -> str: + return f"<{self.status}: {self.reason}>" + + +@dataclass +class Result: + domain: str + check: str + outcome: str + app: object + ref: object + evidence: str + note: str = "" + + +@dataclass +class Site: + slug: str + results: list[Result] = field(default_factory=list) + + def add(self, domain, check, outcome, app, ref, evidence, note=""): + self.results.append(Result(domain, check, outcome, app, ref, evidence, note)) + + +# ---------------------------------------------------------------- app side + +class App: + def __init__(self, zip_path: Path): + self.zip = zipfile.ZipFile(zip_path) + + def json(self, name: str): + try: + return json.loads(self.zip.read(name)) + except KeyError: + return None + + def value(self, name: str, *keys): + """The present value at keys, or Unavail with the export's reason.""" + node = self.json(name) + for k in (None, *keys): + if isinstance(node, dict) and "status" in node and "value" not in node \ + and node.get("status") != "present": + return Unavail(node["status"], node.get("detail") or node.get("reason", "")) + if isinstance(node, dict) and node.get("status") == "present" and "value" in node: + node = node["value"] + if k is None: + continue + if not isinstance(node, dict) or k not in node: + return Unavail("missing", f"{name}:{'.'.join(map(str, keys))}") + node = node[k] + if isinstance(node, dict) and node.get("status") == "present" and "value" in node: + node = node["value"] + elif isinstance(node, dict) and node.get("status") in ("unavailable", "failed", "pending"): + return Unavail(node["status"], node.get("detail") or node.get("reason", "")) + return node + + +# ---------------------------------------------------------------- comparing + +def compare_exact(app, refs: list): + refs = [r for r in refs if r is not None] + if not refs: + return "ok" if isinstance(app, Unavail) or app in (None, [], False) else "phantom" + if isinstance(app, Unavail): + return "gap" + if all(r == refs[0] for r in refs): + return "ok" if app == refs[0] else "wrong" + return "volatile" if app in refs else "wrong" + + +def compare_set(app, refs: list[set]): + refs = [r for r in refs if r is not None] + if isinstance(app, Unavail): + return ("gap" if any(refs) else "ok"), "" + app = set(app) + union = set().union(*refs) if refs else set() + inter = set.intersection(*refs) if refs else set() + missing, extra = inter - app, app - union + note = [] + if missing: + note.append(f"missing in app: {sorted(missing)}") + if extra: + note.append(f"only in app: {sorted(extra)}") + if missing: + return "wrong", "; ".join(note) + if extra: + return "phantom", "; ".join(note) + return ("volatile" if union != inter else "ok"), "" + + +def compare_num(app, refs: list, rel: float): + """Within rel of the reference range (both runs), else wrong.""" + refs = [r for r in refs if r is not None] + if not refs: + return "ok" + if isinstance(app, Unavail): + return "gap" + lo, hi = min(refs) * (1 - rel), max(refs) * (1 + rel) + if not lo <= app <= hi: + return "wrong" + return "volatile" if min(refs) != max(refs) and not min(refs) <= app <= max(refs) else "ok" + + +def jaccard(a: set, b: set) -> float: + return 1.0 if not a and not b else len(a & b) / len(a | b) + + +def last_headers(ref) -> dict: + return ref["hops"][-1]["headers"] if ref and ref["hops"] else {} + + +# Mozilla HTTP Observatory: HSTS counts from six months; these referrer +# policies do not leak the full URL cross-origin. +HSTS_MIN = 15552000 +GOOD_REFERRER = {"no-referrer", "same-origin", "strict-origin", + "strict-origin-when-cross-origin", "origin", "origin-when-cross-origin"} +KNOWN_REFERRER = GOOD_REFERRER | {"no-referrer-when-downgrade", "unsafe-url"} + + +def referrer_good(value: str) -> bool: + """The last recognised token wins (fallback lists).""" + tokens = [t.strip() for t in value.replace(",", " ").split() if t.strip() in KNOWN_REFERRER] + return bool(tokens) and tokens[-1] in GOOD_REFERRER + + +def sec_checks(headers: dict, https: bool) -> dict[str, bool]: + """Independent pass/fail for the app's header checks (not CSP quality).""" + h = {k: " ".join(v).lower() for k, v in headers.items()} + csp = h.get("content-security-policy", "") + hsts = re.search(r"max-age=(\d+)", h.get("strict-transport-security", "")) + out = { + "Frames": "x-frame-options" in h or "frame-ancestors" in csp, + "No Sniff": h.get("x-content-type-options", "").strip() == "nosniff", + "Referrer": referrer_good(h.get("referrer-policy", "")), + "Permissions": "permissions-policy" in h, + } + if https: + out["HSTS"] = bool(hsts and int(hsts.group(1)) >= HSTS_MIN) + return out + + +def check_site(site: Site, app: App, refs: list[dict], meta: dict, conf: dict): + r1 = refs[0] if refs else None + + # Completeness of the app run itself. + pending = meta.get("pendingAtExport", []) + site.add("app", "finished before export", + "wrong" if pending or meta.get("error") else "ok", + pending or meta.get("error") or "complete", "complete", "app-meta.json", + f"waited {meta.get('waitedMs')} ms") + + # Redirects and status. + hops = app.value("network/probe.json", "hops") + app_chain = hops if isinstance(hops, Unavail) else [(h["url"], h["statusCode"]) for h in hops] + ref_chains = [[(h["url"], h["statusCode"]) for h in r["hops"]] for r in refs] + site.add("redirects", "chain (url, status)", compare_exact(app_chain, ref_chains), + app_chain, ref_chains[0] if ref_chains else None, "network/probe.json") + + # Header presence on the final hop. + hh = app.value("network/headers.json", "hops") + app_final = {} if isinstance(hh, Unavail) or not hh else hh[-1]["headers"] + for name in SECURITY_HEADERS + ["content-encoding"]: + a = hh if isinstance(hh, Unavail) else name in app_final + site.add("headers", name, compare_exact(a, [name in last_headers(r) for r in refs]), + a, name in last_headers(r1), "network/headers.json") + + # Security scorecard inputs. + checks = app.value("analysis/security-scorecard.json", "audit", "checks") + https = (r1 or {}).get("finalUrl", "").startswith("https") + ref_checks = [sec_checks(last_headers(r), https) for r in refs] + for label in ["HSTS", "Frames", "No Sniff", "Referrer", "Permissions"]: + if not any(label in rc for rc in ref_checks): + continue + if isinstance(checks, Unavail): + a = checks + else: + match = [c for c in checks if c["label"] == label] + a = match[0]["passing"] if match else Unavail("missing", label) + site.add("security", f"check {label}", compare_exact(a, [rc.get(label) for rc in ref_checks]), + a, ref_checks[0].get(label) if ref_checks else None, + "analysis/security-scorecard.json") + + # Certificate. + cert = app.value("analysis/certificate.json") + rc = [r.get("certificate") for r in refs] + if any(rc) or not isinstance(cert, Unavail): + def a(f): + return cert if isinstance(cert, Unavail) else f(cert) + site.add("certificate", "issuer CN", + compare_exact(a(lambda c: c["issuer"].get("Common Name")), + [c and c["issuerCN"] for c in rc]), + a(lambda c: c["issuer"].get("Common Name")), rc[0] and rc[0]["issuerCN"], + "analysis/certificate.json") + site.add("certificate", "valid until (date)", + compare_exact(a(lambda c: (c.get("validUntil") or "")[:10]), + [c and c["notAfter"][:10] for c in rc]), + a(lambda c: c.get("validUntil")), rc[0] and rc[0]["notAfter"], + "analysis/certificate.json", "app shows local time; date compared") + site.add("certificate", "host matches (ref: SAN)", + compare_exact(a(lambda c: "matches host" in c.get("host", "").lower()), + [c and c["hostMatches"] for c in rc]), + a(lambda c: c.get("host")), rc[0] and rc[0]["hostMatches"], + "analysis/certificate.json") + + # Cookies set by the server (browser cookies need the browser reference). + cookies = app.value("network/cookies.json", "cookies") + ref_server = [{c["name"]: c for h in r["hops"] for c in h["setCookies"]} for r in refs] + if isinstance(cookies, Unavail): + app_server = cookies + else: + app_server = {c["name"]: c for c in cookies if "Server" in str(c.get("source", ""))} + outcome, note = compare_set( + app_server if isinstance(app_server, Unavail) else set(app_server), + [set(r) for r in ref_server]) + site.add("cookies", "Set-Cookie names", outcome, + app_server if isinstance(app_server, Unavail) else sorted(app_server), + sorted(ref_server[0]) if ref_server else None, "network/cookies.json", note) + if not isinstance(app_server, Unavail) and ref_server: + for name in sorted(set(app_server) & set(ref_server[0])): + ac, rcook = app_server[name], ref_server[0][name] + browser_ss = {c["name"]: (c.get("sameSite") or "").lower() + for c in (refs[0].get("browser") or {}).get("cookies", [])} + for flag, av, rv in [ + ("Secure", ac.get("secure") == "yes", rcook["secure"]), + ("HttpOnly", ac.get("httpOnly") == "yes", rcook["httpOnly"]), + ("SameSite", "(not set)" if str(ac.get("sameSite", "")).lower().startswith("not set") + else str(ac.get("sameSite", "")).lower(), + (rcook["sameSite"] or "(not set)").lower()), + ]: + ok = av == rv or (flag == "SameSite" and rv == "(not set)" + and av not in ("none", "(not set)") + and browser_ss.get(name) == av) + # The other reference run got another header: the site varies. + other = [r.get(name) for r in ref_server[1:] if name in r] + key = {"Secure": "secure", "HttpOnly": "httpOnly", "SameSite": "sameSite"}[flag] + varies = any((o[key] or "(not set)").lower() == av if flag == "SameSite" + else o[key] == av for o in other) + outcome = "ok" if ok else "volatile" if varies else "wrong" + site.add("cookies", f"{name} {flag}", outcome, av, rv, + "network/cookies.json", + f"reference browser: {browser_ss.get(name)}" if flag == "SameSite" else "") + + # robots.txt and sitemaps. + rb = app.value("analysis/robots-sitemap.json") + rr = [r["robots"] for r in refs] + a_present = rb if isinstance(rb, Unavail) else str(rb.get("robots", "")).endswith("200") + site.add("robots", "robots.txt present", compare_exact(a_present, [r["present"] for r in rr]), + a_present, rr[0]["present"] if rr else None, "analysis/robots-sitemap.json", + "" if isinstance(rb, Unavail) else f"app fetched {rb.get('robotsUrl')}") + if not isinstance(rb, Unavail) and rr and rr[0]["present"]: + outcome, note = compare_set(set(rb.get("sitemaps") or []), [set(r["sitemaps"]) for r in rr]) + site.add("robots", "sitemaps", outcome, len(rb.get("sitemaps") or []), + len(rr[0]["sitemaps"]), "analysis/robots-sitemap.json", note) + m = re.match(r"(\d+)", str(rb.get("disallow", ""))) + site.add("robots", "disallow count", + compare_exact(int(m.group(1)) if m else Unavail("missing", "disallow"), + [r["disallowCount"] for r in rr]), + rb.get("disallow"), rr[0]["disallowCount"], "analysis/robots-sitemap.json") + + # Browser: cookies, storage, resources, page weight. + brs = [r.get("browser") for r in refs if r.get("browser")] + if brs: + all_names = [{c["name"] for c in b["cookies"]} | set(srv) + for b, srv in zip(brs, ref_server)] + app_names = cookies if isinstance(cookies, Unavail) else {c["name"] for c in cookies} + outcome, note = compare_set(app_names, all_names) + site.add("browser", "cookie names (browser + server)", outcome, + app_names if isinstance(app_names, Unavail) else len(app_names), + len(all_names[0]), "network/cookies.json", note) + for area in ("local", "session"): + a = app.value("network/browser-probe.json", "storage", area, "count") + refs_n = [b["storage"][area] for b in brs] + # Ad and A/B scripts add or drop a few keys per load. + close = (not isinstance(a, Unavail) + and all(abs(a - n) <= max(2, 0.2 * n) for n in refs_n)) + site.add("browser", f"{area}Storage keys (±2 or 20%)", + "ok" if close else compare_exact(a, refs_n), a, + brs[0]["storage"][area], "network/browser-probe.json") + a = app.value("network/browser-probe.json", "resourceCount") + site.add("browser", "resource count (±20%)", + compare_num(a, [b["resourceEntryCount"] for b in brs], 0.2), a, + brs[0]["resourceEntryCount"], "network/browser-probe.json", + f"ref requests incl. document: {brs[0]['requestCount']}") + a = app.value("analysis/page-weight.json", "transfer") + lower = app.value("analysis/page-weight.json", "transferLowerBound") is True + outcome = compare_num(a, [b["transferBytes"] for b in brs], 0.5) + if outcome == "wrong" and lower and a <= max(b["transferBytes"] for b in brs): + outcome = "ok" # the app says it is a lower bound, and it is one + site.add("browser", "transfer bytes (±50%)", outcome, a, + brs[0]["transferBytes"], "analysis/page-weight.json", + "app: lower bound" if lower else "") + tl = app.value("network/timeline.json") + if isinstance(tl, Unavail): + site.add("browser", "request hosts", "gap", tl, len(brs[0]["hosts"]), + "network/timeline.json") + else: + app_hosts = {re.sub(r"^https?://([^/:]+).*", r"\1", e["url"]) for e in tl} + ref_hosts = set().union(*[set(b["hosts"]) for b in brs]) + j = jaccard(app_hosts, ref_hosts) + site.add("browser", "request hosts (Jaccard ≥ 0.6)", "ok" if j >= 0.6 else "wrong", + len(app_hosts), len(ref_hosts), "network/timeline.json", + f"J={j:.2f}; only app: {sorted(app_hosts - ref_hosts)[:8]}; " + f"only ref: {sorted(ref_hosts - app_hosts)[:8]}") + + # Metadata from the rendered DOM. + md = app.value("analysis/metadata.json") + for key in ("title", "description", "canonical"): + a = md if isinstance(md, Unavail) else md.get(key) + site.add("metadata", key, compare_exact(a, [b["metadata"][key] for b in brs]), + a, brs[0]["metadata"][key], "analysis/metadata.json") + if not isinstance(md, Unavail): + outcome, note = compare_set(set((md.get("openGraph") or {}).items()), + [set(b["metadata"]["openGraph"].items()) for b in brs]) + site.add("metadata", "og:* tags", outcome, len(md.get("openGraph") or {}), + len(brs[0]["metadata"]["openGraph"]), "analysis/metadata.json", note) + sd = app.value("analysis/structured-data.json") + if isinstance(sd, Unavail): + types = sd + else: + types, stack = set(), list(sd) + while stack: + it = stack.pop() + if isinstance(it, dict): + t = it.get("@type") + types.update(t if isinstance(t, list) else [t] if t else []) + stack.extend(it.get("@graph", [])) + elif isinstance(it, list): + stack.extend(it) + outcome, note = compare_set(types, [set(b["metadata"]["jsonLdTypes"]) for b in brs]) + site.add("metadata", "JSON-LD @types", outcome, + types if isinstance(types, Unavail) else sorted(types), + brs[0]["metadata"]["jsonLdTypes"], "analysis/structured-data.json", note) + + # Sovereignty inputs (the score formula itself is audited separately). + infra = [r.get("infrastructure") for r in refs if r.get("infrastructure")] + sov = app.value("analysis/sovereignty.json") + if infra: + if isinstance(sov, Unavail): + site.add("sovereignty", "report", "gap", sov, "measured", "analysis/sovereignty.json") + else: + host_check = next((c for c in sov.get("checks", []) + if c.get("title", "").startswith("Hosting company")), None) + evidence = (host_check or {}).get("evidence", "") + cdn = re.search(r"served through ([^;(]+?)(?: \(|;)", evidence) + if cdn: + # The app names the CDN instead of the hidden origin: the + # main host's AS holder must be that CDN. + word = cdn.group(1).split()[0].lower() + holder = (infra[0]["asName"] or "").lower() + site.add("sovereignty", "CDN = AS holder of main host", + "ok" if word in holder else "wrong", cdn.group(1), + infra[0]["asName"], "analysis/sovereignty.json", evidence) + else: + m = re.search(r"AS(\d+)", evidence) + a_asn = int(m.group(1)) if m else Unavail("missing", "no AS in hosting check") + site.add("sovereignty", "hosting AS", + compare_exact(a_asn, [i["asn"] for i in infra]), + a_asn, infra[0]["asn"], "analysis/sovereignty.json", evidence) + c = (re.search(r"registered in (\w\w)", evidence) + or re.search(r"\((\w\w)\)$", (host_check or {}).get("title", ""))) + a_cc = c.group(1) if c else Unavail("missing", "no AS country in evidence") + site.add("sovereignty", "hosting AS country", + compare_exact(a_cc, [i["asCountry"] for i in infra]), a_cc, + infra[0]["asCountry"], "analysis/sovereignty.json") + for key in ("nameServers", "mailServers"): + recs = {r["record"].lower().rstrip(".") for r in sov.get(key) or []} + outcome, note = compare_set(recs, [set(i[key]) for i in infra]) + site.add("sovereignty", key, outcome, sorted(recs), infra[0][key], + "analysis/sovereignty.json", note) + + check_consistency(site, app) + + # Detections against hand-made ground truth. + det = app.json("analysis/detections.json") or {} + names = {d["name"] for d in det.get("value", [])} + for name in conf.get("expect_detections") or []: + site.add("detections", f"expect {name}", "ok" if name in names else "wrong", + name in names, True, "analysis/detections.json") + for name in conf.get("forbid_detections") or []: + site.add("detections", f"forbid {name}", "phantom" if name in names else "ok", + name in names, False, "analysis/detections.json") + + +# ---------------------------------------------------------------- consistency + +TRACKING = {"advertising", "behavior", "social", "analytics", "marketing"} + + +def check_consistency(site: Site, app: App): + """Pages of the app that judge the same thing must agree (no reference + needed): Cookie Lens <-> Services <-> Sovereignty privacy pillar.""" + det = (app.json("analysis/detections.json") or {}).get("value", []) + cookies = app.value("network/cookies.json", "cookies") + sov = app.value("analysis/sovereignty.json") + + # 1. Every tracking cookie in the Cookie Lens is claimed by a service. + if not isinstance(cookies, Unavail): + claimed = {c for d in det for c in d.get("cookies", [])} | { + e["title"].removeprefix("Cookie ") for d in det + for e in d.get("evidence", []) if e.get("kind") == "cookie"} + orphans = sorted(c["name"] for c in cookies + if c.get("category") in ("advertising", "analytics", "social") + and c.get("provider") and c["name"] not in claimed) + site.add("consistency", "tracking cookies are Services detections", + "wrong" if orphans else "ok", orphans or "all claimed", "[]", + "network/cookies.json", "Cookie Lens vs analysis/detections.json") + + if isinstance(sov, Unavail): + return + checks = {c["title"]: c for c in sov.get("checks", [])} + + def names(title): + ev = (checks.get(title) or {}).get("evidence", "") + return set() if ev in ("", "None detected") else {n.strip() for n in ev.split(", ")} + + # 2. Sovereignty counts the trackers the Privacy tab lists. + if "Number of trackers" in checks: + privacy = {d["name"] for d in det if d["category"] in TRACKING} + sv = names("Number of trackers") + site.add("consistency", "Privacy tab trackers = Sovereignty trackers", + "ok" if sv == privacy else "wrong", len(sv), len(privacy), + "analysis/sovereignty.json", + f"only Privacy: {sorted(privacy - sv)}; only Sovereignty: {sorted(sv - privacy)}" + if sv != privacy else "") + + # 3. The consent tool Services found is the one Sovereignty names. + consent = {d["name"] for d in det if d["category"] == "consent"} + if consent and "Consent management" in checks: + sv = names("Consent management") + site.add("consistency", "consent tool: Services = Sovereignty", + "ok" if sv == consent else "wrong", sorted(sv), sorted(consent), + "analysis/sovereignty.json") + + # 4. The grade is the band of the score (80 / 50, lib/services/derive/score_band.dart). + score, grade = sov.get("score"), sov.get("grade") + if score is not None: + want = "Good" if score >= 80 else "Fair" if score >= 50 else "Poor" + site.add("consistency", "Sovereignty grade matches score", + "ok" if grade == want else "wrong", f"{score} {grade}", want, + "analysis/sovereignty.json") + + +# ---------------------------------------------------------------- report + +def fmt(v) -> str: + s = str(v) if not isinstance(v, str) else v + s = s.replace("|", "\\|").replace("\n", " ") + return s if len(s) <= 160 else s[:157] + "…" + + +def report(run_dir: Path, sites: list[Site]) -> str: + domains = sorted({r.domain for s in sites for r in s.results}, + key=lambda d: DOMAINS.index(d) if d in DOMAINS else 99) + out = [f"# Validation report — {run_dir.name}", "", + "✅ ok · 〰️ volatile · ⚠️ gap · 👻 phantom · ❌ wrong", "", + "| site | " + " | ".join(domains) + " |", + "|---|" + "---|" * len(domains)] + for s in sites: + cells = [] + for d in domains: + rs = [r for r in s.results if r.domain == d] + worst = max((r.outcome for r in rs), key=RANK.get, default=None) + bad = sum(1 for r in rs if RANK[r.outcome] >= 2) + cells.append("" if worst is None else ICON[worst] + (f" {bad}" if bad else "")) + out.append(f"| {s.slug} | " + " | ".join(cells) + " |") + out += ["", "## Deviations", ""] + for s in sites: + devs = [r for r in s.results if RANK[r.outcome] >= 2] + if not devs: + continue + out += [f"### {s.slug}", "", "| | domain | check | app | reference | evidence | note |", + "|---|---|---|---|---|---|---|"] + for r in devs: + out.append(f"| {ICON[r.outcome]} | {r.domain} | {fmt(r.check)} | {fmt(r.app)} | " + f"{fmt(r.ref)} | `{s.slug}/app.zip:{r.evidence}` | {fmt(r.note)} |") + out.append("") + return "\n".join(out) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("run_dir", type=Path) + ap.add_argument("--sites", type=Path, default=HERE / "sites.yaml") + args = ap.parse_args() + conf = {s["slug"]: s for s in yaml.safe_load(args.sites.read_text())["sites"]} + + sites = [] + for site_dir in sorted(p for p in args.run_dir.iterdir() if p.is_dir()): + zip_path = site_dir / "app.zip" + if not zip_path.exists(): + continue + refs = [json.loads(p.read_text()) for p in sorted(site_dir.glob("ref-R*.json"))] + meta_path = site_dir / "app-meta.json" + meta = json.loads(meta_path.read_text()) if meta_path.exists() else {} + site = Site(site_dir.name) + check_site(site, App(zip_path), refs, meta, conf.get(site_dir.name, {})) + sites.append(site) + + (args.run_dir / "report.md").write_text(report(args.run_dir, sites)) + summary = { + s.slug: {f"{r.domain}/{r.check}": r.outcome for r in s.results} for s in sites + } + (args.run_dir / "summary.json").write_text(json.dumps(summary, indent=2)) + totals = {} + for s in sites: + for r in s.results: + totals[r.outcome] = totals.get(r.outcome, 0) + 1 + print(f"{len(sites)} sites: {totals} -> {args.run_dir / 'report.md'}") + + +if __name__ == "__main__": + main() diff --git a/validation/reference.py b/validation/reference.py new file mode 100644 index 00000000..0b9a6773 --- /dev/null +++ b/validation/reference.py @@ -0,0 +1,350 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["pyyaml", "playwright", "beautifulsoup4"] +# /// +"""Independent reference measurement for the validation harness. + +Measures every site of a run with other tools than the app (curl, openssl, +Playwright WebKit, BeautifulSoup, dig, Team Cymru whois) and writes //ref-