Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion PRIVACY.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ The application contacts:
1. **The site you inspect**: the page itself (in the in-app browser and with a separate request by the app), its redirects, `robots.txt`, `sitemap.xml` and, once the Sovereignty tab has been opened, `/.well-known/security.txt`.
2. **The hosts that page loads files from**: the in-app browser loads them like any browser. For files whose size the browser does not report, the app requests the file once more to measure it. Known tracking pixels, and third-party images that look like pings, are not requested again. If you start a "fresh load", the app loads the page and its third-party files again in an off-screen browser.
3. **RIPEstat (stat.ripe.net, RIPE NCC, Netherlands)**: the IP addresses of the hosts a page uses, to look up their network and country. Only after you open the Sovereignty tab. Once opened, the analysis also runs for the pages you open next during that session.
4. **A DNS-over-HTTPS resolver** (doh.libredns.gr, Greece; dns.google, United States, as fallback): the domain name of the site, to read its DNS records. Only after you open the Sovereignty tab, as above.
4. **A DNS-over-HTTPS resolver** (doh.libredns.gr, Greece; dns.google, United States, as fallback): the domain name of the site, to read its DNS records, and the host names on the site's own domain that the page loads (at most 20), to see whether any is an alias for a tracker. Only after you open the Sovereignty tab, as above, or the Privacy tab.
5. **Your device's DNS resolver** (usually your internet provider's or the one set in your operating system): the app looks up the IP address of every host a page uses, so this resolver sees those host names. The in-app browser also uses it for every page it loads, whether or not you open the Sovereignty tab; the app's own lookups of all hosts a page uses happen only after you open it, as above.
6. **DuckDuckGo's icon service (icons.duckduckgo.com)**: the domain names of services detected on a page, to show their logos. Logos are cached for 30 days.

Expand Down
9 changes: 9 additions & 0 deletions lib/models/inspection/page_inspection.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'observed.dart';
import 'page_layout.dart';
import 'permission_request.dart';
import 'server_snapshot.dart';
import 'technique_signals.dart';

/// [PageInspection.freshLoad] before the user asked for one.
const kFreshLoadNotMeasured = Unavailable<BrowserSnapshot>(
Expand Down Expand Up @@ -73,6 +74,10 @@ class PageInspection {
/// screen, wait for a quiet network), only when the user asks for it.
final Observed<BrowserSnapshot> freshLoad;

/// What the page called during the fresh load (fingerprinting and similar).
/// Only the fresh load is instrumented, never the live browser.
final Observed<TechniqueSignals> freshTechniques;

/// The app's own size lookups for [freshLoad] (see [resourceEstimates]).
final Map<String, Observed<int>> freshEstimates;

Expand All @@ -99,6 +104,7 @@ class PageInspection {
this.primaryDocumentPreference,
this.browserSettled = false,
this.freshLoad = kFreshLoadNotMeasured,
this.freshTechniques = kTechniquesNotMeasured,
this.freshEstimates = const {},
this.liveScreenshot = kNoScreenshot,
this.freshScreenshot = kNoScreenshot,
Expand All @@ -122,6 +128,7 @@ class PageInspection {
primaryDocumentPreference = null,
browserSettled = false,
freshLoad = kFreshLoadNotMeasured,
freshTechniques = kTechniquesNotMeasured,
freshEstimates = const {},
liveScreenshot = kNoScreenshot,
freshScreenshot = kNoScreenshot,
Expand All @@ -138,6 +145,7 @@ class PageInspection {
Object? primaryDocumentPreference = _keep,
bool? browserSettled,
Observed<BrowserSnapshot>? freshLoad,
Observed<TechniqueSignals>? freshTechniques,
Map<String, Observed<int>>? freshEstimates,
Observed<Uint8List>? liveScreenshot,
Observed<Uint8List>? freshScreenshot,
Expand All @@ -158,6 +166,7 @@ class PageInspection {
: primaryDocumentPreference as DataSource?,
browserSettled: browserSettled ?? this.browserSettled,
freshLoad: freshLoad ?? this.freshLoad,
freshTechniques: freshTechniques ?? this.freshTechniques,
freshEstimates: freshEstimates ?? this.freshEstimates,
liveScreenshot: liveScreenshot ?? this.liveScreenshot,
freshScreenshot: freshScreenshot ?? this.freshScreenshot,
Expand Down
157 changes: 157 additions & 0 deletions lib/models/inspection/technique_signals.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
import 'dart:convert';

import 'package:flutter/foundation.dart' show debugPrint;

import 'observed.dart';

/// [PageInspection.freshTechniques] before a fresh load was asked for. Not
/// measured is not clean: it is never shown as "none found".
const kTechniquesNotMeasured = Unavailable<TechniqueSignals>(
UnavailableReason.notApplicable, 'Run a fresh load to check');

/// A canvas the page read back (toDataURL / toBlob), with what was drawn on
/// its 2d context. Counts only: no pixels, no text.
class CanvasRead {
final int width, height, chars, colors, reads;
final String script;
const CanvasRead({
required this.width,
required this.height,
required this.chars,
required this.colors,
required this.reads,
required this.script,
});
}

/// A permission or similar prompt asked for by the page.
class PromptCall {
final String api;

/// Time since the page started loading.
final Duration at;

/// Whether the user had clicked or typed recently; null when unknown.
final bool? userActive;
final String script;
const PromptCall({
required this.api,
required this.at,
required this.userActive,
required this.script,
});
}

/// What the instrumentation script saw the page call during a fresh load.
class TechniqueSignals {
final List<CanvasRead> canvas;
final int fontMeasureCalls, distinctFonts;
final bool webglDebugInfo;
final int audioOffline, audioOscillators, audioRendered;
final int rtcConnections, rtcDataChannels, rtcOffers, rtcGetUserMedia;
final int enumerateDevices, battery, clipboardRead;
final List<PromptCall> prompts;

/// Signal group (`fonts`, `webgl`, `audio`, `rtc`) to the script URL that
/// made the first call; '' when unknown.
final Map<String, String> scripts;

const TechniqueSignals({
this.canvas = const [],
this.fontMeasureCalls = 0,
this.distinctFonts = 0,
this.webglDebugInfo = false,
this.audioOffline = 0,
this.audioOscillators = 0,
this.audioRendered = 0,
this.rtcConnections = 0,
this.rtcDataChannels = 0,
this.rtcOffers = 0,
this.rtcGetUserMedia = 0,
this.enumerateDevices = 0,
this.battery = 0,
this.clipboardRead = 0,
this.prompts = const [],
this.scripts = const {},
});
}

const _kReportUnreadable = 'The technique report could not be read';

int _int(Object? v) => v is num ? v.toInt() : 0;
String _str(Object? v) => v is String ? v : '';
Map<String, Object?> _map(Object? v) =>
v is Map ? v.cast<String, Object?>() : const {};

/// Reads the JSON string returned by `window.__vsv_tech_report()`. No report
/// is Unavailable (the script did not run); a malformed one is Failed. Never
/// an empty, clean-looking value.
Observed<TechniqueSignals> parseTechniqueReport(Object? raw) {
if (raw == null) {
return const Unavailable(UnavailableReason.notExposedToPage,
'The page gave no technique report');
}
try {
final decoded = raw is String ? jsonDecode(raw) : raw;
if (decoded is! Map) {
debugPrint('Technique report had an unexpected shape');
return const Failed(_kReportUnreadable);
}
final r = decoded.cast<String, Object?>();
final fonts = _map(r['fonts']);
final webgl = _map(r['webgl']);
final audio = _map(r['audio']);
final rtc = _map(r['rtc']);
final canvas = <CanvasRead>[
for (final c in (r['canvas'] is List ? r['canvas'] as List : const []))
if (c is Map)
CanvasRead(
width: _int(c['w']),
height: _int(c['h']),
chars: _int(c['chars']),
colors: _int(c['colors']),
reads: _int(c['reads']),
script: _str(c['script']),
),
];
final prompts = <PromptCall>[
for (final p in (r['prompts'] is List ? r['prompts'] as List : const []))
if (p is Map)
PromptCall(
api: _str(p['api']),
at: Duration(milliseconds: _int(p['atMs'])),
userActive:
p['userActive'] is bool ? p['userActive'] as bool : null,
script: _str(p['script']),
),
];
return Present(
TechniqueSignals(
canvas: canvas,
fontMeasureCalls: _int(fonts['measureCalls']),
distinctFonts: _int(fonts['distinctFonts']),
webglDebugInfo: webgl['debugInfo'] == true,
audioOffline: _int(audio['offline']),
audioOscillators: _int(audio['oscillators']),
audioRendered: _int(audio['rendered']),
rtcConnections: _int(rtc['connections']),
rtcDataChannels: _int(rtc['dataChannels']),
rtcOffers: _int(rtc['offers']),
rtcGetUserMedia: _int(rtc['getUserMedia']),
enumerateDevices: _int(r['enumerateDevices']),
battery: _int(r['battery']),
clipboardRead: _int(r['clipboardRead']),
prompts: prompts,
scripts: {
'fonts': _str(fonts['script']),
'webgl': _str(webgl['script']),
'audio': _str(audio['script']),
'rtc': _str(rtc['script']),
},
),
source: DataSource.browser);
} catch (e) {
debugPrint('Technique report could not be read: $e');
return const Failed(_kReportUnreadable);
}
}
116 changes: 116 additions & 0 deletions lib/services/cname_cloaking_service.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
import 'dart:async';

import '../utils/site.dart';
import 'detection/catalog.dart';
import 'sovereignty_service.dart';

/// A host on the page's own site whose DNS name is an alias for a tracker.
class CnameAlias {
final String host; // metrics.shop.test
final String target; // shop.sc.omtrdc.net
final String tracker; // catalogue entity name
const CnameAlias(this.host, this.target, this.tracker);

Map<String, Object?> toJson() =>
{'host': host, 'target': target, 'tracker': tracker};
}

class CloakingResult {
final List<CnameAlias> aliases;
final List<String> checked;

/// Hosts whose lookup failed or timed out.
final List<String> notChecked;
const CloakingResult({
this.aliases = const [],
this.checked = const [],
this.notChecked = const [],
});
}

/// CNAME targets of a host: null when the lookup failed, empty when it has none.
typedef CnameLookup = Future<List<String>?> Function(String host);

/// [findCloakedTrackers] without its tuning knobs (a seam for tests).
typedef CloakingFinder = Future<CloakingResult> Function(
Iterable<String> resourceHosts,
{required String pageHost,
required CnameLookup lookup});

/// The app's own DNS-over-HTTPS lookup of CNAME records.
Future<List<String>?> dohCnameLookup(String host) =>
SovereigntyService.dnsQuery(host, 5);

/// Categories whose services exist to follow visitors. Marketing and CRM
/// entities also own hosting domains (force.com), so an alias for those says
/// nothing about tracking.
const Set<DetectionCategory> kCloakingTrackerCategories = {
DetectionCategory.advertising,
DetectionCategory.behavior,
DetectionCategory.analytics,
};

const int kCloakingMaxHosts = 20;
const int kCloakingParallel = 4;

/// Hosts of [resourceHosts] on the page's own site (same registrable domain,
/// not the page host itself), at most [max]; each is looked up once. A lookup
/// that fails, throws or exceeds [lookupTimeout] lists that host as not
/// checked and does not stop the others.
Future<CloakingResult> findCloakedTrackers(
Iterable<String> resourceHosts, {
required String pageHost,
required CnameLookup lookup,
int max = kCloakingMaxHosts,
Duration lookupTimeout = const Duration(seconds: 12),
}) async {
final page = pageHost.toLowerCase();
final hosts = <String>[];
for (final raw in resourceHosts) {
final h = raw.toLowerCase();
if (h.isEmpty || h == page || !sameSite(h, page) || hosts.contains(h)) {
continue;
}
hosts.add(h);
if (hosts.length == max) break;
}

Future<List<String>?> safeLookup(String h) async {
try {
return await lookup(h).timeout(lookupTimeout);
} catch (_) {
return null;
}
}

final results = <String, List<String>?>{};
for (var i = 0; i < hosts.length; i += kCloakingParallel) {
final batch = hosts.skip(i).take(kCloakingParallel).toList();
final answers = await Future.wait(batch.map(safeLookup));
for (var j = 0; j < batch.length; j++) {
results[batch[j]] = answers[j];
}
}

final aliases = <CnameAlias>[];
final checked = <String>[];
final notChecked = <String>[];
for (final h in hosts) {
final targets = results[h];
if (targets == null) {
notChecked.add(h);
continue;
}
checked.add(h);
for (final t in targets) {
if (sameSite(t, page)) continue;
final e = entityForHost(t);
if (e != null && kCloakingTrackerCategories.contains(e.category)) {
aliases.add(CnameAlias(h, t, e.name));
break;
}
}
}
return CloakingResult(
aliases: aliases, checked: checked, notChecked: notChecked);
}
22 changes: 20 additions & 2 deletions lib/services/collectors/fresh_load_measurer.dart
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import 'package:flutter_inappwebview/flutter_inappwebview.dart';
import '../derive/network_quiet.dart';
import 'collector_world.dart';
import 'frame_relay.dart';
import 'vsv_techniques_js.dart';

/// Loads a page once more, apart from the live browser, the way Lighthouse
/// does: empty cache, phone-sized screen, waiting until the network is
Expand All @@ -24,7 +25,11 @@ class FreshLoadRaw {

/// Small JPEG; null when the platform gave none.
final Uint8List? screenshot;
const FreshLoadRaw(this.payload, {this.screenshot});

/// What `window.__vsv_tech_report()` returned (a JSON string); null when
/// the script did not run or the call failed.
final Object? techniques;
const FreshLoadRaw(this.payload, {this.screenshot, this.techniques});
}

/// Thumbnail settings for screenshots of a load (small JPEG).
Expand Down Expand Up @@ -89,6 +94,12 @@ class HeadlessFreshLoadMeasurer implements FreshLoadMeasurer {
),
initialUserScripts: UnmodifiableListView<UserScript>([
collectorUserScript(defaultTargetPlatform),
// Page world: it has to see the page's own calls. Fresh load only.
UserScript(
source: vsvTechniquesJs,
injectionTime: UserScriptInjectionTime.AT_DOCUMENT_START,
forMainFrameOnly: true,
),
]),
onWebViewCreated: (controller) {
controller.addJavaScriptHandler(
Expand Down Expand Up @@ -140,6 +151,13 @@ class HeadlessFreshLoadMeasurer implements FreshLoadMeasurer {
} catch (_) {}
await waitQuiet(clock.elapsed + _scrollSettleBudget);

Object? techniques;
try {
techniques = await controller.evaluateJavascript(
source: 'window.__vsv_tech_report && window.__vsv_tech_report()');
} catch (_) {
// No report is said as unavailable, never as clean.
}
final payload = await controller.evaluateJavascript(
source: 'window.__vsv_collect()', contentWorld: world);
Uint8List? shot;
Expand All @@ -149,7 +167,7 @@ class HeadlessFreshLoadMeasurer implements FreshLoadMeasurer {
} catch (_) {
// No screenshot is fine: the sizes are what was measured.
}
return FreshLoadRaw(payload, screenshot: shot);
return FreshLoadRaw(payload, screenshot: shot, techniques: techniques);
} finally {
await view.dispose();
}
Expand Down
Loading
Loading