Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion cli/benches/quality_bench.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use criterion::{Criterion, criterion_group, criterion_main};
use do_wdr_lib::bias_scorer::score_result;
use do_wdr_lib::quality::score_content;
use std::hint::black_box;

Expand Down Expand Up @@ -53,5 +54,39 @@ One more subscribe button.
});
}

criterion_group!(benches, bench_quality_scoring);
fn bench_bias_scoring(c: &mut Criterion) {
let content = r#"
# Sample Page

This is a sample page with some content.
It has multiple lines.

Cookie Policy: We use cookies.
All rights reserved (c) 2026.
Privacy Policy is available here.

```rust
fn main() {
println!("Hello, world!");
}
```

Subscribe to our newsletter for more updates.
Follow us on Twitter.
Click here to learn more.
buy now cheap discount free trial best price
"#
.repeat(10);

c.bench_function("score_result", |b| {
b.iter(|| {
score_result(
black_box("https://github.com/rust-lang/rust"),
black_box(&content),
);
});
});
}

criterion_group!(benches, bench_quality_scoring, bench_bias_scoring);
criterion_main!(benches);
20 changes: 17 additions & 3 deletions cli/src/bias_scorer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,21 @@

use url::Url;

/// Check if a string contains another string case-insensitively (ASCII-only)
fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool {
if needle.is_empty() {
return true;
}
let haystack_bytes = haystack.as_bytes();
let needle_bytes = needle.as_bytes();
if haystack_bytes.len() < needle_bytes.len() {
return false;
}
haystack_bytes
.windows(needle_bytes.len())
.any(|window| window.eq_ignore_ascii_case(needle_bytes))
}

/// Score a result based on domain trust and content quality
pub fn score_result(url: &str, content: &str) -> f64 {
let mut score: f64 = 0.5;
Expand Down Expand Up @@ -53,11 +68,10 @@ pub fn score_result(url: &str, content: &str) -> f64 {
score += 0.05;
}

// SEO spam detection
// SEO spam detection using zero-allocation case-insensitive ASCII substring search
let spam_terms = ["buy now", "cheap", "discount", "free trial", "best price"];
let lower_content = content.to_lowercase();
for term in spam_terms {
if lower_content.contains(term) {
if contains_ignore_ascii_case(content, term) {
score -= 0.1;
}
}
Expand Down
Loading