From cc8cb7948b021a3d210551b6b13c502a47277f7e Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 02:11:37 +0300 Subject: [PATCH 01/18] look: a place in a file - file:line, file:A-B, file::name, 'file name' A model addresses code the way tracebacks and test runners do: src/eval.rs:62, tests/test_x.py:120-160, src/eval.rs::matches_greater, tests/t.py::TestA::test_b[case], 'pad src/display.rs'. look answered all of these with 'nothing found'; in a trial (Qwen 3.5 4B and 27B, the fix tasks) that was 37 of 89 look calls, and the small model with the short tool list spent its step limit on them. - file:N - the function around the line, as look shows a function; outside any: the lines around - file:A-B (also 'file A-B', L120-L154) - those lines, up to 200, and the functions they are in - file::name, file:name, 'file name', 'name file' - the name in that file; a test id loses its case - not in that file: said so, with the outline of the file - 'file | grep ...' is the file - several words of which some are names: those names, not 'nothing found' Co-Authored-By: Claude Fable 5.2 --- crates/gitnext-cli/src/cmds.rs | 1 + crates/gitnext-cli/src/mcp.rs | 6 +- crates/gitnext-ops/src/agents.rs | 1 + crates/gitnext-ops/src/look.rs | 169 +++++++++++++++++++++++++++---- crates/gitnext-ops/tests/look.rs | 100 ++++++++++++++++++ 5 files changed, 255 insertions(+), 22 deletions(-) create mode 100644 crates/gitnext-ops/tests/look.rs diff --git a/crates/gitnext-cli/src/cmds.rs b/crates/gitnext-cli/src/cmds.rs index 8174a4b..7919aea 100644 --- a/crates/gitnext-cli/src/cmds.rs +++ b/crates/gitnext-cli/src/cmds.rs @@ -166,6 +166,7 @@ pub enum More { /// External libraries: which the code calls (by package, with declared versions) and which are only declared Libs, /// One call for any target: symbol → code + callers + callees + tests + impact; file/folder → outline; + /// `file:120-160` → those lines, `file:62` / `file::name` → the function there; /// error text → where; words → where they appear. Several targets at once. Look { #[arg(required = true)] diff --git a/crates/gitnext-cli/src/mcp.rs b/crates/gitnext-cli/src/mcp.rs index fd93a75..d6a9789 100644 --- a/crates/gitnext-cli/src/mcp.rs +++ b/crates/gitnext-cli/src/mcp.rs @@ -58,7 +58,7 @@ fn tools() -> Value { "inputSchema": { "type": "object", "properties": props, "required": req } }) }; json!([ - tool("look", "Use this first. One call answers whatever you need next. what = a function/class/method name → its code, who calls it, what it calls, the tests that reach it with the command to run them, and what a change would affect; a file or folder → every signature with line numbers; an error/traceback → the code where it happened; words → where they appear. Pass several targets at once to save steps.", + tool("look", "Use this first. One call answers whatever you need next. what = a function/class/method name → its code, who calls it, what it calls, the tests that reach it with the command to run them, and what a change would affect; a file or folder → every signature with line numbers; `file:120-160` → those lines, `file:62` or `file::name` → the function there; an error/traceback → the code where it happened; words → where they appear. Pass several targets at once to save steps.", json!({ "what": { "type": "string" }, "targets": { "type": "array", "items": { "type": "string" }, "description": "several targets in one call" } }), &[]), tool("overview", "Start here in an unfamiliar repo: languages, areas with purposes, entry points (main, routes, CLI commands), area dependencies, recent activity, hotspots, health — one screen.", json!({}), &[]), @@ -125,8 +125,8 @@ fn lite_tools() -> Value { let str_ = |d: &str| json!({ "type": "string", "description": d }); json!([ { "name": "look", - "description": "Look at code without reading whole files. what = a file or folder (lists functions/classes with lines), a function or class name (shows its code), an error message (shows where it happened), or words (finds them).", - "inputSchema": { "type": "object", "properties": { "what": str_("file, folder, name, error text or words") }, "required": ["what"] } }, + "description": "Look at code without reading whole files. what = a file or folder (lists functions/classes with lines), a function or class name (shows its code), lines of a file (`src/app.py:120-160`), an error message (shows where it happened), or words (finds them).", + "inputSchema": { "type": "object", "properties": { "what": str_("file, folder, name, file:lines, error text or words") }, "required": ["what"] } }, { "name": "replace", "description": "Replace one whole function or class with new code. No need to read the file first; indentation is fixed for you.", "inputSchema": { "type": "object", "properties": { "symbol": str_("function or class name, e.g. parse_date or Task.reopen"), "code": str_("the complete new definition") }, "required": ["symbol", "code"] } }, diff --git a/crates/gitnext-ops/src/agents.rs b/crates/gitnext-ops/src/agents.rs index 8727300..056a0a3 100644 --- a/crates/gitnext-ops/src/agents.rs +++ b/crates/gitnext-ops/src/agents.rs @@ -78,6 +78,7 @@ pub fn instructions_lite() -> String { Use `mcp__gitnext__look` before reading files. It takes one string `what`:\n\ - a file or folder → its functions and classes with line numbers: {\"what\": \"utils.py\"}\n\ - a function or class → its code: {\"what\": \"parse_date\"} or {\"what\": \"Task.reopen\"}\n\ + - lines of a file: {\"what\": \"utils.py:120-160\"}\n\ - an error or traceback → the code where it happened\n\ - words → where they appear: {\"what\": \"retry delay\"}\n\ To change one function: `mcp__gitnext__replace` {\"symbol\": \"parse_date\", \"code\": \"\"}.\n\ diff --git a/crates/gitnext-ops/src/look.rs b/crates/gitnext-ops/src/look.rs index 07cf1a4..e87c4dd 100644 --- a/crates/gitnext-ops/src/look.rs +++ b/crates/gitnext-ops/src/look.rs @@ -1,6 +1,7 @@ //! `look`: one call that answers whatever the agent needs next, decided by the //! shape of the target — a file or folder (outline), a symbol (everything about -//! it at once), an error (where it happened) or words (where they appear). +//! it at once), a place in a file (`src/eval.rs:62`, `tests/t.py:120-154`, +//! `src/eval.rs::matches_greater`), an error (where it happened) or words (where they appear). //! //! Every agent step re-sends the whole system prompt, so one answer that saves a //! follow-up call is worth far more than a shorter answer. For a symbol, `look` @@ -30,13 +31,18 @@ pub fn look(ws: &Workspace, g: &Graph, what: &str, lite: bool) -> String { return s; } } + // a shell habit: `tests/test_x.py | grep -A 20 name` — the file is what was asked for + let what = match what.split_once(" | ") { + Some((a, _)) if file_of(g, a.trim()).is_some() => a.trim(), + _ => what, + }; + // a place in a file, the way tracebacks, test runners and models write it + if let Some(s) = place(ws, g, what, lite) { + return s; + } // a file or folder (exact, or a unique file name like `utils.py`) let p = what.trim_start_matches("./").trim_end_matches('/'); - let file = g.files.iter().find(|f| f.path == p).map(|f| f.path.clone()).or_else(|| { - let v: Vec<&str> = g.files.iter().filter(|f| f.path.ends_with(&format!("/{p}"))).map(|f| f.path.as_str()).collect(); - (v.len() == 1).then(|| v[0].to_string()) - }); - if let Some(f) = file { + if let Some(f) = file_of(g, what) { if g.files.iter().any(|x| x.path == f && x.parsed) { let mut s = render::outline(g, root, &f, Opts { tests: true, ..opts }); s.push_str("next: look at several names from this list in one call\n"); @@ -71,21 +77,129 @@ pub fn look(ws: &Workspace, g: &Graph, what: &str, lite: bool) -> String { } } if !ids.is_empty() { - let first = g.syms[ids[0]].sym.qualified.clone(); - if ids.iter().all(|&i| g.syms[i].sym.qualified == first) { - return symbol(ws, g, &ids, lite); + return one_or_choice(ws, g, name, &ids, lite); + } + words_hits(g, root, what) +} + +/// The symbol, or — when the ids are different things with one name — the list to pick from. +fn one_or_choice(ws: &Workspace, g: &Graph, name: &str, ids: &[usize], lite: bool) -> String { + let first = &g.syms[ids[0]].sym.qualified; + if ids.iter().all(|&i| &g.syms[i].sym.qualified == first) { + return symbol(ws, g, ids, lite); + } + let mut s = format!("{} things are called `{name}` — look at one by its full name (or several at once):\n", ids.len()); + for &i in ids.iter().take(12) { + let r = &g.syms[i]; + // a name that would only bring this list back is pinned by its line: `main@78` + let short = g.short_name(i); + let label = if g.find(short).iter().all(|&x| g.syms[x].sym.qualified == r.sym.qualified) && short != name { short.to_string() } else { format!("{}@{}", r.sym.name, r.sym.start_line) }; + let _ = writeln!(s, " {label} ({}) {}:{}", r.sym.kind.as_str(), r.file, r.sym.start_line); + } + s +} + +/// The repo file `p` names: its path, or a file name / path tail that only one file has (`utils.py`). +fn file_of(g: &Graph, p: &str) -> Option { + let p = p.trim_start_matches("./").trim_end_matches('/'); + if p.is_empty() { + return None; + } + g.files.iter().find(|f| f.path == p).map(|f| f.path.clone()).or_else(|| { + let v: Vec<&str> = g.files.iter().filter(|f| f.path.ends_with(&format!("/{p}"))).map(|f| f.path.as_str()).collect(); + (v.len() == 1).then(|| v[0].to_string()) + }) +} + +/// Lines of a file the way they get written: `62`, `L62`, `120-154`, `L120-L154`. +fn line_range(s: &str) -> Option<(usize, usize)> { + let num = |x: &str| x.trim().trim_start_matches(['L', 'l']).parse::().ok().filter(|&n| n > 0); + match s.split_once('-') { + Some((a, b)) => Some((num(a)?, num(b)?)).filter(|(a, b)| a <= b), + None => num(s).map(|n| (n, n)), + } +} + +/// Most lines one answer shows; a longer range says how to ask for the rest. +const RANGE_LINES: usize = 200; + +/// A place in a file: `src/eval.rs:62` (the function around the line), `tests/t.py:120-154` +/// (those lines), `src/eval.rs::matches_greater`, `tests/t.py::TestA::test_b[case]`, +/// `eval.rs matches_greater`, `pad src/display.rs`. None: `what` is not written that way. +fn place(ws: &Workspace, g: &Graph, what: &str, lite: bool) -> Option { + let words: Vec<&str> = what.split_whitespace().collect(); + // written as a path (`eval.rs`, `src/eval`): a bare word is a name, even if a file is called that + let path = |p: &str| if p.contains(['.', '/']) { file_of(g, p) } else { None }; + let (file, rest) = match words.as_slice() { + [one] => { + let (a, b) = one.split_once(':')?; + (path(a)?, b.trim_start_matches(':')) } - let mut s = format!("{} things are called `{name}` — look at one by its full name (or several at once):\n", ids.len()); - for &i in ids.iter().take(12) { - let r = &g.syms[i]; - // a name that would only bring this list back is pinned by its line: `main@78` - let short = g.short_name(i); - let label = if g.find(short).iter().all(|&x| g.syms[x].sym.qualified == r.sym.qualified) && short != name { short.to_string() } else { format!("{}@{}", r.sym.name, r.sym.start_line) }; - let _ = writeln!(s, " {label} ({}) {}:{}", r.sym.kind.as_str(), r.file, r.sym.start_line); + [a, b] => match (path(a), path(b)) { + (Some(f), None) => (f, *b), + (None, Some(f)) => (f, *a), + _ => return None, + }, + _ => return None, + }; + let rest = rest.trim_end_matches("()").trim_end_matches(':'); + if rest.is_empty() { + return None; + } + let inside = |a: usize, b: usize| -> Vec { + (0..g.syms.len()) + .filter(|&i| { + let s = &g.syms[i].sym; + g.syms[i].file == file && (s.kind.is_callable() || s.kind.is_type()) && s.start_line as usize <= b && s.end_line as usize >= a + }) + .collect() + }; + if let Some((a, b)) = line_range(rest) { + // one line: the function it is in, with everything `look` says about a function + if a == b { + if let Some(i) = inside(a, a).into_iter().min_by_key(|&i| g.syms[i].sym.end_line - g.syms[i].sym.start_line) { + return Some(symbol(ws, g, &[i], lite)); + } } - return s; + let text = std::fs::read_to_string(ws.root.join(&file)).ok()?; + let lines: Vec<&str> = text.lines().collect(); + // one line outside any function: the lines around it + let (a, b) = if a == b { (a.saturating_sub(10).max(1), a + 20) } else { (a, b) }; + if a > lines.len() { + return Some(format!("{file} has {} lines\n", lines.len())); + } + let end = b.min(lines.len()).min(a + RANGE_LINES - 1); + let fence = gitnext_parse::lang_for(std::path::Path::new(&file)).map(|l| l.fence()).unwrap_or(""); + let mut s = format!("{file}:{a}-{end} (of {} lines)\n```{fence}\n", lines.len()); + for l in &lines[a - 1..end] { + let _ = writeln!(s, "{l}"); + } + s.push_str("```\n"); + if end < b.min(lines.len()) { + let _ = writeln!(s, "… cut at {RANGE_LINES} lines: ask for {file}:{}-{} for the rest", end + 1, b.min(lines.len())); + } + let mut owners = inside(a, end); + owners.sort_by_key(|&i| g.syms[i].sym.start_line); + if !owners.is_empty() { + let list: Vec = owners.iter().take(8).map(|&i| format!("{} ({}-{})", g.short_name(i), g.syms[i].sym.start_line, g.syms[i].sym.end_line)).collect(); + let _ = writeln!(s, "these lines are in: {}{}", list.join(", "), if owners.len() > 8 { " …" } else { "" }); + let _ = writeln!(s, "{}", if lite { "to change one: replace(symbol=, code=)" } else { "next: look at one by name for its callers and tests" }); + } + return Some(s); } - words_hits(g, root, what) + // a name in that file; a test id keeps its class and loses its case: `TestA::test_b[case]` + let name = rest.split('[').next().unwrap_or(rest).replace('.', "::"); + let last = name.rsplit("::").next().unwrap_or(&name); + let here = |v: Vec| -> Vec { v.into_iter().filter(|&i| g.syms[i].file == file).collect() }; + let mut ids = here(g.find(&name)); + if ids.is_empty() { + ids = here(g.named(last)); + } + if ids.is_empty() { + let outline = render::outline(g, &ws.root, &file, Opts { tests: true, ..Opts::default() }); + return Some(format!("no `{rest}` in {file} — what is there:\n{outline}")); + } + Some(one_or_choice(ws, g, last, &ids, lite)) } /// Several targets in one call: each answered as by [`look`], separated. @@ -238,7 +352,24 @@ fn words_hits(g: &Graph, root: &std::path::Path, what: &str) -> String { s.push('\n'); } if s.is_empty() { - s = format!("nothing found for `{what}`. Try a file name, a function name or other words; look . shows the whole project\n"); + // several words and no line has them all: the ones that are names are still an answer + let mut seen = std::collections::HashSet::new(); + let names: Vec = what + .split(|c: char| c.is_whitespace() || c == ',') + .filter(|w| w.len() > 2) + .flat_map(|w| g.find(w).into_iter().take(2)) + .filter(|&i| (g.syms[i].sym.kind.is_callable() || g.syms[i].sym.kind.is_type()) && seen.insert(i)) + .take(10) + .collect(); + if what.split_whitespace().count() > 1 && !names.is_empty() { + s.push_str("nothing has all these words; the ones that are names — look at one:\n"); + for i in names { + let r = &g.syms[i]; + let _ = writeln!(s, " {} ({}) {}:{}", g.short_name(i), r.sym.kind.as_str(), r.file, r.sym.start_line); + } + return s; + } + s = format!("nothing found for `{what}`. Try a function name, a file name, lines of a file (`src/app.py:120-160`) or other words; look . shows the whole project\n"); } s } diff --git a/crates/gitnext-ops/tests/look.rs b/crates/gitnext-ops/tests/look.rs new file mode 100644 index 0000000..2a7117d --- /dev/null +++ b/crates/gitnext-ops/tests/look.rs @@ -0,0 +1,100 @@ +//! `gitnext look`: a place in a file, written the way tracebacks, test runners and models write it. + +use gitnext_ops::gitnext_index::Index; +use gitnext_ops::look::look; +use gitnext_ops::Workspace; + +const EVAL: &str = "pub fn matches(a: u32, b: u32) -> bool {\n matches_greater(a, b) || a == b\n}\n\nfn matches_greater(a: u32, b: u32) -> bool {\n if a == b {\n return false;\n }\n a > b\n}\n\npub const LIMIT: u32 = 9;\n"; +const DISPLAY: &str = "pub fn pad(s: &str) -> String {\n format!(\"{s:>8}\")\n}\n\npub fn show(n: u32) -> String {\n pad(&n.to_string())\n}\n"; +const TESTS: &str = "#[test]\nfn test_greater_than() {\n assert!(demo::eval::matches(2, 1));\n}\n\n#[test]\nfn test_equal() {\n assert!(demo::eval::matches(1, 1));\n}\n"; + +fn project(name: &str) -> (Workspace, std::path::PathBuf) { + let dir = std::env::temp_dir().join(format!("gitnext-look-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for (p, c) in [ + ("Cargo.toml", "[package]\nname = \"demo\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"), + ("src/lib.rs", "pub mod display;\npub mod eval;\n"), + ("src/eval.rs", EVAL), + ("src/display.rs", DISPLAY), + ("tests/test_req.rs", TESTS), + ] { + let f = dir.join(p); + std::fs::create_dir_all(f.parent().unwrap()).unwrap(); + std::fs::write(f, c).unwrap(); + } + (Workspace::open(&dir).unwrap(), dir) +} + +#[test] +fn a_name_in_a_file_is_found_however_it_is_written() { + let (ws, dir) = project("name"); + let mut idx = Index::open(&ws.root).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + let by_name = look(&ws, &g, "matches_greater", false); + assert!(by_name.starts_with("src/eval.rs:5-10 matches_greater"), "{by_name}"); + for what in [ + "src/eval.rs::matches_greater", + "eval.rs::matches_greater", + "src/eval.rs:matches_greater", + "src/eval.rs matches_greater", + "matches_greater src/eval.rs", + "matches_greater() eval.rs", + "src/eval.rs:7", + "src/eval.rs:L7", + "./src/eval.rs:5", + ] { + assert_eq!(look(&ws, &g, what, false), by_name, "`{what}`"); + } + // a test id of a runner: the file, the test, its case + let t = look(&ws, &g, "tests/test_req.rs::test_greater_than[case-1]", false); + assert!(t.starts_with("tests/test_req.rs:1-4 test_greater_than"), "{t}"); + // not in that file: said so, with what the file has + let miss = look(&ws, &g, "src/display.rs::matches_greater", false); + assert!(miss.starts_with("no `matches_greater` in src/display.rs — what is there:") && miss.contains("pad") && miss.contains("show"), "{miss}"); + // a file followed by a shell pipe is the file + let piped = look(&ws, &g, "src/display.rs | grep -n pad", false); + assert_eq!(piped, look(&ws, &g, "src/display.rs", false)); + // names with `::` that are not files are names as before + let q = look(&ws, &g, "eval::matches_greater", false); + assert_eq!(q, by_name); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn lines_of_a_file() { + let (ws, dir) = project("lines"); + let mut idx = Index::open(&ws.root).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + let r = look(&ws, &g, "src/eval.rs:5-7", false); + assert!(r.starts_with("src/eval.rs:5-7 (of 12 lines)\n```rust\nfn matches_greater(a: u32, b: u32) -> bool {\n if a == b {\n return false;\n```\n"), "{r}"); + assert!(r.contains("these lines are in: matches_greater (5-10)"), "{r}"); + // the same written with a space, and past the end of the file + assert_eq!(look(&ws, &g, "eval.rs 5-7", false), r); + let tail = look(&ws, &g, "src/eval.rs:9-400", false); + assert!(tail.starts_with("src/eval.rs:9-12 (of 12 lines)") && tail.contains("LIMIT"), "{tail}"); + assert_eq!(look(&ws, &g, "src/eval.rs:300-400", false), "src/eval.rs has 12 lines\n"); + // two functions in the range: both named; the small-model wording says how to change one + let both = look(&ws, &g, "src/eval.rs:1-10", true); + assert!(both.contains("these lines are in: matches (1-3), matches_greater (5-10)") && both.contains("replace(symbol="), "{both}"); + // a line outside any function: the lines around it + let around = look(&ws, &g, "src/eval.rs:12", false); + assert!(around.starts_with("src/eval.rs:2-12 (of 12 lines)") && around.contains("LIMIT"), "{around}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn words_that_are_names() { + let (ws, dir) = project("words"); + let mut idx = Index::open(&ws.root).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + // no line has all of the words; two of them are functions + let r = look(&ws, &g, "pad formatter width align show", false); + assert!(r.starts_with("nothing has all these words; the ones that are names"), "{r}"); + assert!(r.contains("pad (function) src/display.rs:1") && r.contains("show (function) src/display.rs:5"), "{r}"); + let none = look(&ws, &g, "zzzz qqqq", false); + assert!(none.starts_with("nothing found for `zzzz qqqq`") && none.contains("src/app.py:120-160"), "{none}"); + let _ = std::fs::remove_dir_all(&dir); +} From 8087e48b9247f31d3713c7e5efb33d1c7935960e Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 02:11:37 +0300 Subject: [PATCH 02/18] eval: a tool call cut by the token limit is asked for again; changelog llama.cpp answers 500 when the model's tool call is not whole JSON - a long replace cut at 1024 tokens. The run ended there as an error; now the same step is asked again with room to finish (counted as cut_calls). The look tool of the short list says it reads lines of a file. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 9 +++++++++ eval/agent.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cad0493..70d8984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- **`gitnext look` takes a place in a file**, written the way tracebacks, test runners and models write it: + `src/eval.rs:62` (the function around the line), `tests/test_x.py:120-160` (those lines), + `src/eval.rs::matches_greater`, `tests/test_x.py::TestA::test_b[case]`, `eval.rs matches_greater`. + Words of which only some are names list those names instead of "nothing found". In a trial with + Qwen 3.5 4B and 27B, 37 of 89 `look` calls were written one of these ways and found nothing; all are answered now. +- eval: a tool call cut by the token limit (a long `replace`) is asked for again instead of ending the run. + ## 0.3.0 — 2026-11-12 Measured on ten open-source projects in five languages; the numbers and how they are obtained are in diff --git a/eval/agent.py b/eval/agent.py index 073d9fb..32cc71d 100644 --- a/eval/agent.py +++ b/eval/agent.py @@ -54,6 +54,7 @@ import re import subprocess import time +import urllib.error import urllib.request from pathlib import Path @@ -91,7 +92,7 @@ def tool(name, desc, props, req): ] # mode `lite`: these and nothing else — what gitnext's lite MCP profile gives a small model (plus `finish`) LITE_TOOLS = [ - tool("look", "Look at code. what = a function or class name (shows its code, who calls it, its tests), a file or folder (lists what is in it), an error message, or words to find.", {"what": S}, ["what"]), + tool("look", "Look at code. what = a function or class name (shows its code, who calls it, its tests), a file or folder (lists what is in it), lines of a file (`src/app.py:120-160`), an error message, or words to find.", {"what": S}, ["what"]), tool("replace", "Replace one whole function or class with new code. No need to read the file first.", {"symbol": S, "code": S}, ["symbol", "code"]), tool("add_after", "Add new code (a new function) right after an existing function or class.", {"symbol": S, "code": S}, ["symbol", "code"]), tool("rename", "Rename a function or class everywhere: definition, calls, imports, tests.", {"old": S, "new": S}, ["old", "new"]), @@ -334,7 +335,15 @@ def locate(err: str) -> str: # where gitnext thinks an error comes from used["timeout"] = True break compress(msgs, CTX_CHARS) - r = chat(host, msgs, tools, 1024) + try: + r = chat(host, msgs, tools, 1024) + except urllib.error.HTTPError as e: + # llama.cpp answers 500 when the model's tool call is not whole JSON: a long `replace` cut + # by the token limit. The same step again with room to finish it; any other error is an error + if e.code != 500 or "tool call" not in e.read().decode(errors="replace"): + raise + used["cut_calls"] = used.get("cut_calls", 0) + 1 + r = chat(host, msgs, tools, 4096) u = r.get("usage", {}) used["tokens_in"] += u.get("prompt_tokens", 0) used["tokens_out"] += u.get("completion_tokens", 0) From 1024c47822cd391f5d4d4306d10397d2979c5209 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 02:14:25 +0300 Subject: [PATCH 03/18] workflows: actions on Node 24; the Intel Mac binary is built on the arm runner The v0.3.0 release has no gitnext-x86_64-apple-darwin.tar.gz: its job asked for a macos-13 runner, and those are retired, so it never started. Both Mac targets now build on macos-latest (cargo --target), with a cache key per target. Action versions that run on Node 24 (the Node 20 ones print a deprecation warning): checkout v7, setup-go v7, setup-node v7, setup-java v6, setup-python v7, cache v6, upload-artifact v7, action-gh-release v3. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/ci.yml | 2 +- .github/workflows/eval.yml | 16 ++++++++-------- .github/workflows/release.yml | 9 ++++++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f713419..9b17aa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: components: clippy diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index dea6558..2622067 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -13,16 +13,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: { go-version: "stable" } - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: { node-version: "22" } - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v6 with: { distribution: temurin, java-version: "21", cache: maven } - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: { python-version: "3.12" } - name: build gitnext run: cargo build --release -p gitnext-cli @@ -32,7 +32,7 @@ jobs: - name: truth cache key id: truth run: echo "key=eval-truth-v$(python3 -c "import sys; sys.path.insert(0, 'eval'); import run; print(run.CACHE_VERSION)")-${{ hashFiles('eval/repos.json') }}" >> "$GITHUB_OUTPUT" - - uses: actions/cache/restore@v4 + - uses: actions/cache/restore@v6 with: path: eval/.work/truth.json key: ${{ steps.truth.outputs.key }}-${{ github.run_id }} @@ -43,7 +43,7 @@ jobs: - name: eval timeout-minutes: 100 run: GITNEXT=$PWD/target/release/gitnext python3 eval/run.py --seed 7 --n 8 --mutants 4 --check - - uses: actions/cache/save@v4 + - uses: actions/cache/save@v6 if: always() with: path: eval/.work/truth.json @@ -51,6 +51,6 @@ jobs: - name: results if: always() run: cat eval/results/latest.md >> "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: { name: eval-results, path: eval/results/latest.* } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfaffa4..383a686 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,15 +18,18 @@ jobs: matrix: include: - { os: macos-latest, target: aarch64-apple-darwin } - - { os: macos-13, target: x86_64-apple-darwin } + # Intel Macs: built on the arm runner (the macos-13 runners are retired; a job on them never starts) + - { os: macos-latest, target: x86_64-apple-darwin } - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } - { os: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.target }} - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.target }} - name: build run: cargo build --release --locked --target ${{ matrix.target }} -p gitnext-cli - name: package @@ -35,7 +38,7 @@ jobs: cd target/${{ matrix.target }}/release tar -czf gitnext-${{ matrix.target }}.tar.gz gitnext (command -v sha256sum >/dev/null && sha256sum gitnext-${{ matrix.target }}.tar.gz || shasum -a 256 gitnext-${{ matrix.target }}.tar.gz) > gitnext-${{ matrix.target }}.tar.gz.sha256 - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: | target/${{ matrix.target }}/release/gitnext-${{ matrix.target }}.tar.gz From 6d66dcc8a5e19255abf9d0d0222159f24b7e1b3f Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 02:54:56 +0300 Subject: [PATCH 04/18] dead, edit, callees: what a check of the commands against each other and the source found eval/consistency.py asks, on the ten projects of the eval, what can be asked without a compiler: callers <-> callees <-> impact, read and symbols against the file, search, edit replace / delete round trips, hierarchy against class signatures, and dead by deleting what it names. Results: eval/results/consistency.md; COVERAGE.md has a level for it. What it found, fixed here: - dead knew calls only. 160 of 225 'likely dead' symbols were named elsewhere in the code (a value, a type, a base or metaclass, an import, a callback, a Go init); deleting them broke the build of at least 6 projects of 8. The name must now be written nowhere outside the symbol; .d.ts declarations and Go init are not dead. 64 are left; sqlparse still loses 5 methods reached by names built at run time. - edit replace moved a symbol replaced with its own code when a body line was indented less than the first, or tabs stood under spaces (11 of 40 edits in the Java projects): code that already stands where it belongs is not touched, the rest keeps its place relative to the first line. - edit delete refused, and read cut the comment in half, for a Javadoc whose lines do not start with '*': a block comment that ends above the item is taken whole. - callees listed as sure a method called on an object of unknown type that callers lists as 'maybe' (5 pairs): callees has 'maybe' too, look counts them. Also: the Kaggle trial (Qwen 3.5 27B and 4B, the fix tasks) in HISTORY.md. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 10 + crates/gitnext-cli/src/main.rs | 1 + crates/gitnext-index/src/graph.rs | 7 +- crates/gitnext-ops/src/edit.rs | 40 ++- crates/gitnext-ops/src/insight.rs | 55 +++- crates/gitnext-ops/src/look.rs | 5 +- crates/gitnext-ops/src/render.rs | 19 +- crates/gitnext-ops/tests/dead.rs | 72 +++++ crates/gitnext-ops/tests/edit.rs | 90 ++++++ eval/consistency.py | 366 +++++++++++++++++++++++++ eval/results/COVERAGE.md | 47 +++- eval/results/HISTORY.md | 21 ++ eval/results/consistency.json | 439 ++++++++++++++++++++++++++++++ eval/results/consistency.md | 30 ++ 14 files changed, 1166 insertions(+), 36 deletions(-) create mode 100644 crates/gitnext-ops/tests/dead.rs create mode 100644 crates/gitnext-ops/tests/edit.rs create mode 100644 eval/consistency.py create mode 100644 eval/results/consistency.json create mode 100644 eval/results/consistency.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d8984..95c11f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,17 @@ `src/eval.rs::matches_greater`, `tests/test_x.py::TestA::test_b[case]`, `eval.rs matches_greater`. Words of which only some are names list those names instead of "nothing found". In a trial with Qwen 3.5 4B and 27B, 37 of 89 `look` calls were written one of these ways and found nothing; all are answered now. +- **`gitnext dead` names only what nothing refers to.** It knew calls only: a function used as a value, a class + used as a type, a base or a metaclass, an imported name, a Go `init` were "likely dead" — 160 of 225 on ten + projects, and deleting them broke the build of six. Now the name must be written nowhere outside the symbol. +- **`gitnext edit replace`** keeps every line where it was when the code comes back as it was read (a line indented + less than the first, tabs under spaces); **`edit delete`** and `read` take the whole doc comment (a Javadoc + whose lines do not start with `*` was cut in half, and the delete refused). +- **`callees`** lists a method called on an object of unknown type as "maybe", as `callers` does; `look` counts + such calls instead of naming them. +- `eval/consistency.py`: do the commands agree with each other and with the source (`eval/results/consistency.md`). - eval: a tool call cut by the token limit (a long `replace`) is asked for again instead of ending the run. +- Release: the Intel Mac binary is built again (its runner was retired); `CITATION.cff`. ## 0.3.0 — 2026-11-12 diff --git a/crates/gitnext-cli/src/main.rs b/crates/gitnext-cli/src/main.rs index 860fb3d..48f763e 100644 --- a/crates/gitnext-cli/src/main.rs +++ b/crates/gitnext-cli/src/main.rs @@ -779,6 +779,7 @@ fn callees_json(g: &Graph, i: usize) -> serde_json::Value { let c = g.callees(i); serde_json::json!({ "resolved": c.resolved.into_iter().map(|(x, l)| (g.syms[x].sym.qualified.clone(), l)).collect::>(), + "maybe": c.maybe.into_iter().map(|(x, l)| (g.syms[x].sym.qualified.clone(), l)).collect::>(), "ambiguous": c.ambiguous.into_iter().map(|(n, v)| (n, v.into_iter().map(|x| g.syms[x].sym.qualified.clone()).collect::>())).collect::>(), "external": c.external, }) diff --git a/crates/gitnext-index/src/graph.rs b/crates/gitnext-index/src/graph.rs index a79b4f2..273b240 100644 --- a/crates/gitnext-index/src/graph.rs +++ b/crates/gitnext-index/src/graph.rs @@ -84,8 +84,11 @@ pub struct Hierarchy { /// Outgoing calls of a function, split by how confidently they resolved. #[derive(Debug, Default)] pub struct Callees { - /// Exactly one candidate in the repo. + /// Exactly one candidate in the repo, and the reference names it for sure. pub resolved: Vec<(usize, u32)>, + /// One candidate, reached only by its name on a receiver of unknown type (`x.close()`): + /// the same pairs that `callers` lists as "maybe". + pub maybe: Vec<(usize, u32)>, /// Several candidates (e.g. `.as_str()` on an unknown receiver). pub ambiguous: Vec<(String, Vec)>, /// Not defined in the repo (std, external crates, closures…). @@ -1093,7 +1096,7 @@ impl Graph { } [i] => { if seen.insert(i) { - out.resolved.push((i, e.line)); + if self.is_guess(src, i) { out.maybe.push((i, e.line)) } else { out.resolved.push((i, e.line)) } } } ref many => { diff --git a/crates/gitnext-ops/src/edit.rs b/crates/gitnext-ops/src/edit.rs index 1c3d10b..75034cb 100644 --- a/crates/gitnext-ops/src/edit.rs +++ b/crates/gitnext-ops/src/edit.rs @@ -37,6 +37,17 @@ pub fn doc_start(lines: &[String], start: usize) -> usize { let mut a = start; while a > 1 { let prev = lines[a - 2].trim_start(); + // the end of a block comment: the whole comment belongs to the item, whatever its lines + // start with (a Javadoc without a `*` on every line) + if prev.trim_end().ends_with("*/") && !prev.contains("/*") { + match (1..a - 1).rev().take(400).find(|&k| lines[k - 1].contains("/*")) { + Some(open) if lines[open - 1].trim_start().starts_with("/*") => { + a = open; + continue; + } + _ => break, // the comment begins after code on its line: not this item's + } + } let is_doc = prev.starts_with("///") || prev.starts_with("//!") || prev.starts_with("#[") || prev.starts_with('@') || prev.starts_with("/**") || prev.starts_with("* ") || prev == "*" || prev.starts_with("*/") || (prev.starts_with("//") && !prev.is_empty()); @@ -54,18 +65,31 @@ fn starts_with_doc(code: &str) -> bool { first.starts_with("///") || first.starts_with("//") || first.starts_with("#[") || first.starts_with('@') || first.starts_with("/*") } -/// Re-indent `code` to `indent` if it was sent flush-left (common for agents). +/// Move `code` so that its first line stands at `indent` (agents send code flush-left); the other +/// lines keep their place relative to the first — also one that is indented less than it, so that +/// a symbol replaced with its own code is the same text. fn reindent(code: &str, indent: &str) -> Vec { let lines: Vec<&str> = code.trim_end_matches('\n').lines().collect(); - let min = lines - .iter() - .filter(|l| !l.trim().is_empty()) - .map(|l| l.len() - l.trim_start().len()) - .min() - .unwrap_or(0); + let own = |l: &str| l.len() - l.trim_start().len(); + let first_line = lines.iter().find(|l| !l.trim().is_empty()).copied().unwrap_or(""); + let first = own(first_line); + // already where it belongs (the code as `read` showed it): not a character is touched — a file + // that mixes tabs and spaces has no "less indented" to compute + if &first_line[..first] == indent { + return lines.iter().map(|l| l.to_string()).collect(); + } lines .iter() - .map(|l| if l.trim().is_empty() { String::new() } else { format!("{indent}{}", &l[min.min(l.len() - l.trim_start().len())..]) }) + .map(|l| { + if l.trim().is_empty() { + String::new() + } else if own(l) >= first { + format!("{indent}{}", &l[first..]) + } else { + let keep = indent.len().saturating_sub(first - own(l)); + format!("{}{}", indent.get(..keep).unwrap_or(indent), l.trim_start()) + } + }) .collect() } diff --git a/crates/gitnext-ops/src/insight.rs b/crates/gitnext-ops/src/insight.rs index 3a7af8c..ef7696f 100644 --- a/crates/gitnext-ops/src/insight.rs +++ b/crates/gitnext-ops/src/insight.rs @@ -209,13 +209,14 @@ pub struct Dead { } fn is_entry_name(name: &str) -> bool { - matches!(name, "main" | "new" | "default" | "drop" | "fmt" | "from" | "setup" | "teardown") + // `init`: Go runs every `func init()` of a package by itself + matches!(name, "main" | "init" | "new" | "default" | "drop" | "fmt" | "from" | "setup" | "teardown") || (name.starts_with("__") && name.ends_with("__")) || name.starts_with('~') } -/// Symbols nothing in the repo references (approximate: dynamic dispatch, macros, -/// reflection and external users are invisible to the graph). +/// Symbols nothing in the repo refers to: no call in the graph, and the name is written nowhere +/// outside the symbol itself (approximate: reflection, generated names and external users are invisible). pub fn dead(g: &Graph, root: &Path) -> Dead { let entries: HashSet = entrypoints(g, root).into_iter().map(|e| e.symbol).collect(); let referenced: HashSet<&str> = g @@ -227,6 +228,7 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { .collect(); let mut likely = Vec::new(); let mut public_unused = Vec::new(); + let mut candidates = Vec::new(); for (i, r) in g.syms.iter().enumerate() { let s = &r.sym; if s.test || s.trait_impl.is_some() || s.decl || !(s.kind.is_callable() || s.kind.is_type()) || is_entry_name(&s.name) || entries.contains(&i) { @@ -243,6 +245,9 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { if s.signature.starts_with('@') { continue; // decorated Python callables are usually registered somewhere } + if r.file.ends_with(".d.ts") { + continue; // declarations for the compiler (`interface ImportMeta`), merged with ones outside the repo + } if referenced.contains(s.qualified.as_str()) { continue; } @@ -258,6 +263,17 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { } } } + candidates.push(i); + } + // The graph knows calls. A name is also used as a value (`return _BaseCommand`), a type + // (`-> _SourceVarsDict`), a base or metaclass, an import, a callback (`map(helper)`): if the name + // is written anywhere outside the symbol itself, something refers to it. + let named = named_elsewhere(g, root, &candidates); + for i in candidates { + if named.contains(&i) { + continue; + } + let (r, s) = (&g.syms[i], &g.syms[i].sym); let line = format!("{} {} {}:{}", s.kind.as_str(), g.short_name(i), r.file, s.start_line); if s.is_public() && !s.name.starts_with('_') { public_unused.push(line); @@ -268,6 +284,39 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { Dead { likely, public_unused } } +/// Of `candidates`, the symbols whose name is written in the code outside their own lines +/// (comment lines aside). One pass over the files. +fn named_elsewhere(g: &Graph, root: &Path, candidates: &[usize]) -> HashSet { + let mut by_name: HashMap<&str, Vec> = HashMap::new(); + for &i in candidates { + by_name.entry(g.syms[i].sym.name.as_str()).or_default().push(i); + } + let mut named = HashSet::new(); + if by_name.is_empty() { + return named; + } + for f in g.files.iter().filter(|f| f.parsed && f.lines < 50_000) { + let Ok(text) = std::fs::read_to_string(root.join(&f.path)) else { continue }; + for (n, line) in text.lines().enumerate() { + let t = line.trim_start(); + if t.starts_with('#') || t.starts_with("//") || t.starts_with('*') || t.starts_with("/*") { + continue; + } + let ln = n as u32 + 1; + for word in line.split(|c: char| !(c.is_alphanumeric() || c == '_')) { + let Some(ids) = by_name.get(word) else { continue }; + for &i in ids { + let r = &g.syms[i]; + if !(r.file == f.path && r.sym.start_line <= ln && ln <= r.sym.end_line) { + named.insert(i); + } + } + } + } + } + named +} + pub fn render_dead(d: &Dead) -> String { let mut s = format!("unreferenced, not public ({}): likely dead\n", d.likely.len()); for l in &d.likely { diff --git a/crates/gitnext-ops/src/look.rs b/crates/gitnext-ops/src/look.rs index e87c4dd..2784421 100644 --- a/crates/gitnext-ops/src/look.rs +++ b/crates/gitnext-ops/src/look.rs @@ -250,7 +250,10 @@ fn symbol(ws: &Workspace, g: &Graph, ids: &[usize], lite: bool) -> String { let c = g.callees(ids[0]); if !c.resolved.is_empty() { let list: Vec = c.resolved.iter().take(12).map(|(t, _)| g.short_name(*t).to_string()).collect(); - let _ = writeln!(s, "calls: {}{}", list.join(", "), if c.resolved.len() > 12 { format!(" +{}", c.resolved.len() - 12) } else { String::new() }); + let more = if c.resolved.len() > 12 { format!(" +{}", c.resolved.len() - 12) } else { String::new() }; + // guesses (a method called on an object of unknown type) are counted, not named — as in "called by" + let maybe = if c.maybe.is_empty() || lite { String::new() } else { format!(" · maybe {} more via calls on objects of unknown type", c.maybe.len()) }; + let _ = writeln!(s, "calls: {}{more}{maybe}", list.join(", ")); } if let Some(l) = crate::libs::describe(root, g, ids[0]) { let _ = writeln!(s, "libraries: {l}"); diff --git a/crates/gitnext-ops/src/render.rs b/crates/gitnext-ops/src/render.rs index 89bed84..7158a11 100644 --- a/crates/gitnext-ops/src/render.rs +++ b/crates/gitnext-ops/src/render.rs @@ -202,15 +202,9 @@ pub fn read(g: &Graph, root: &Path, ids: &[usize]) -> String { let lines = cache.entry(r.file.as_str()).or_insert_with(|| { std::fs::read_to_string(root.join(&r.file)).map(|t| t.lines().map(str::to_string).collect()).unwrap_or_default() }); - let (mut a, b) = (r.sym.start_line as usize, (r.sym.end_line as usize).min(lines.len())); - // pull in doc comments / attributes / decorators directly above - while a > 1 { - let prev = lines[a - 2].trim_start(); - let is_doc = prev.starts_with("///") || prev.starts_with("//!") || prev.starts_with("#[") || prev.starts_with('@') - || prev.starts_with("/**") || prev.starts_with("* ") || prev == "*" || prev.starts_with("*/") - || (prev.starts_with("//") && !lines[a - 2].trim().is_empty()); - if is_doc { a -= 1 } else { break } - } + let b = (r.sym.end_line as usize).min(lines.len()); + // pull in doc comments / attributes / decorators directly above — the lines an edit of the symbol takes with it + let a = if r.sym.start_line as usize <= lines.len() { crate::edit::doc_start(lines, r.sym.start_line as usize) } else { r.sym.start_line as usize }; if a == 0 || b < a { continue; } @@ -274,6 +268,10 @@ pub fn show(g: &Graph, i: usize, o: Opts) -> String { let v: Vec = c.resolved.iter().map(|(x, _)| g.short_name(*x).to_string()).collect(); parts.push(format!("calls ({}): {}", v.len(), v.join(", "))); } + if !c.maybe.is_empty() { + let v: Vec = c.maybe.iter().map(|(x, _)| g.short_name(*x).to_string()).collect(); + parts.push(format!("maybe calls ({}): {}", v.len(), v.join(", "))); + } if !c.ambiguous.is_empty() { let v: Vec = c.ambiguous.iter().map(|(n, v)| format!("{n}?{}", v.len())).collect(); parts.push(format!("ambiguous: {}", v.join(", "))); @@ -355,6 +353,9 @@ pub fn callees(g: &Graph, i: usize, o: Opts) -> String { for (x, line) in &c.resolved { let _ = writeln!(s, "{} L{line} → {}", g.short_name(*x), loc(g, *x)); } + for (x, line) in &c.maybe { + let _ = writeln!(s, "{} L{line} → {} (maybe: by name, on an object of unknown type)", g.short_name(*x), loc(g, *x)); + } for (name, cands) in &c.ambiguous { let v: Vec<&str> = cands.iter().take(6).map(|&x| g.short_name(x)).collect(); let _ = writeln!(s, "{name} ambiguous: {}", v.join(" | ")); diff --git a/crates/gitnext-ops/tests/dead.rs b/crates/gitnext-ops/tests/dead.rs new file mode 100644 index 0000000..63ee122 --- /dev/null +++ b/crates/gitnext-ops/tests/dead.rs @@ -0,0 +1,72 @@ +//! `gitnext dead`: only what nothing refers to — a call, or the name written anywhere else. + +use gitnext_ops::gitnext_index::Index; +use gitnext_ops::insight::dead; + +const CORE: &str = "\ +import typing as t + + +class _Meta(type): + pass + + +class _Base(metaclass=_Meta): + pass + + +class _Vars(t.TypedDict): + name: str + + +def _really_dead(x): + return x + 1 + + +def _only_in_a_comment(x): + return x + + +def _callback(x): + return x * 2 + + +def _value(): + return 1 + + +def source_vars() -> _Vars: + # _only_in_a_comment is mentioned here and nowhere else + return {\"name\": str(list(map(_callback, [1])))} + + +def __getattr__(name): + if name == \"Base\": + return _Base + return _value +"; + +#[test] +fn a_name_used_as_a_value_or_a_type_is_not_dead() { + let dir = std::env::temp_dir().join(format!("gitnext-dead-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for (p, c) in [("pkg/__init__.py", "from .core import source_vars\n"), ("pkg/core.py", CORE)] { + let f = dir.join(p); + std::fs::create_dir_all(f.parent().unwrap()).unwrap(); + std::fs::write(f, c).unwrap(); + } + let mut idx = Index::open(&dir).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + let d = dead(&g, &dir); + let all = d.likely.join("\n") + &d.public_unused.join("\n"); + // never called, but referred to: a metaclass, a base returned as a value, a return type, a callback + for used in ["_Meta", "_Base", "_Vars", "_callback", "_value"] { + assert!(!all.contains(used), "`{used}` is used, not dead:\n{all}"); + } + // nothing names these outside themselves (a comment does not count) + for unused in ["_really_dead", "_only_in_a_comment"] { + assert!(d.likely.iter().any(|l| l.contains(unused)), "`{unused}` is dead:\n{all}"); + } + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/gitnext-ops/tests/edit.rs b/crates/gitnext-ops/tests/edit.rs new file mode 100644 index 0000000..eba5255 --- /dev/null +++ b/crates/gitnext-ops/tests/edit.rs @@ -0,0 +1,90 @@ +//! `gitnext edit`: a symbol replaced with its own code is the same file; a deleted symbol takes its +//! whole doc comment with it. + +use gitnext_ops::edit::{edit_symbol, EditKind}; +use gitnext_ops::gitnext_index::Index; +use gitnext_ops::render; + +const JAVA: &str = "\ +package demo; + +public class Range { + /** + Maps a slot to the start of its name. + */ + private static int nameStart(int index) { + return index * 4; + } + + /** + * Reads by index. + * @return the element + */ + private static Object readByIndex(Object[] items, String token) { + try { + int index = Integer.parseInt(token); + return items[index]; + } catch (NumberFormatException e) { +\t\t\treturn null; + } + } + + int last; /* not a doc + of anything */ + static int width() { + return 4; + } +} +"; + +fn project() -> (std::path::PathBuf, Index) { + let dir = std::env::temp_dir().join(format!("gitnext-edit-{}-{:?}", std::process::id(), std::thread::current().id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("src")).unwrap(); + std::fs::write(dir.join("src/Range.java"), JAVA).unwrap(); + let mut idx = Index::open(&dir).unwrap(); + idx.refresh().unwrap(); + (dir, idx) +} + +#[test] +fn replace_with_its_own_code_changes_nothing() { + let (dir, idx) = project(); + let g = idx.graph().unwrap(); + let i = g.find("readByIndex")[0]; + // the code as `read` gives it: the doc comment, and two lines indented less than the first + let shown = render::read(&g, &dir, &[i]); + let code: Vec<&str> = shown.lines().skip(2).collect(); + let code = code[..code.len() - 1].join("\n"); + assert!(code.starts_with(" /**\n * Reads by index.") && code.contains("\n } catch"), "{shown}"); + edit_symbol(&dir, &g, i, EditKind::Replace, &code, false).unwrap(); + assert_eq!(std::fs::read_to_string(dir.join("src/Range.java")).unwrap(), JAVA); + // sent flush-left, as an agent sends it: every line that began with the four spaces is back in place + let flush: String = code.lines().map(|l| l.strip_prefix(" ").unwrap_or(l)).collect::>().join("\n"); + edit_symbol(&dir, &g, i, EditKind::Replace, &flush, false).unwrap(); + let again = std::fs::read_to_string(dir.join("src/Range.java")).unwrap(); + assert!(again.contains(" private static Object readByIndex(") && again.contains("\n int index = Integer.parseInt(token);\n"), "{again}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn delete_takes_the_whole_doc_comment() { + let (dir, idx) = project(); + let g = idx.graph().unwrap(); + let i = g.find("nameStart")[0]; + // a Javadoc whose lines do not start with `*` is still the doc of the method + assert!(render::read(&g, &dir, &[i]).starts_with("src/Range.java:4-9 "), "{}", render::read(&g, &dir, &[i])); + edit_symbol(&dir, &g, i, EditKind::Delete, "", false).unwrap(); + let left = std::fs::read_to_string(dir.join("src/Range.java")).unwrap(); + assert_eq!(left, JAVA.replace(" /**\n Maps a slot to the start of its name.\n */\n private static int nameStart(int index) {\n return index * 4;\n }\n\n", "")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_comment_that_ends_a_line_of_code_is_not_a_doc() { + let (dir, idx) = project(); + let g = idx.graph().unwrap(); + let i = g.find("width")[0]; + assert!(render::read(&g, &dir, &[i]).starts_with("src/Range.java:26-28 "), "{}", render::read(&g, &dir, &[i])); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/eval/consistency.py b/eval/consistency.py new file mode 100644 index 0000000..2c3ecbc --- /dev/null +++ b/eval/consistency.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Do gitnext's commands agree with each other and with the source? + +The commands that eval/run.py does not measure against a compiler (see results/COVERAGE.md, level 3) +are checked here for what can be checked without one: + + symmetry if B is among the callers of A, then A is among the callees of B and B is in the impact of A; + if A is among the callees of B, then B is among the callers of A — and as sure there as here + read the code `read` prints is exactly the lines of the file it names, and the name is in them + symbols every symbol's lines exist, its name is in them, a method lies inside its parent + search a symbol is found by its own name + edit `edit replace` with the symbol's own code leaves the file as it was; `edit delete` removes + the symbol's lines and nothing else + hierarchy the bases written in a class's own signature (Python, Java, TypeScript) are in `extends` / + `implements`, and the class is among the subtypes / implementors of each of them + dead what `dead` calls likely dead: no other line of the code names it; and with all of them + deleted the project still compiles and no test starts failing (Python, Go, Rust, TypeScript) + + python3 eval/consistency.py # all projects of eval/repos.json that are set up + python3 eval/consistency.py --repos click,semver --n 40 --no-dead-run + +Writes eval/results/consistency.md and .json. Exit 1 if anything disagrees. +""" +from __future__ import annotations + +import argparse +import json +import random +import re +import subprocess +import time +from pathlib import Path + +import run as ev + +HERE = Path(__file__).resolve().parent + + +def gj(args: list[str], root: Path): + """A gitnext command's JSON answer, None if it failed.""" + code, out = ev.gx(["--json", *args], root) + try: + return json.loads(out) if code == 0 else None + except ValueError: + return None + + +def lines_of(root: Path, file: str, cache: dict) -> list[str]: + if file not in cache: + try: + cache[file] = (root / file).read_text(errors="replace").split("\n") + except OSError: + cache[file] = [] + return cache[file] + + +def check_symmetry(root: Path, sample: list[dict]) -> dict: + res = {"functions": 0, "caller_pairs": 0, "not_in_callees": [], "not_in_impact": [], "callee_pairs": 0, "not_in_callers": [], "sure_here_maybe_there": []} + callers_memo: dict[str, tuple[set, set]] = {} + + def callers_of(q: str) -> tuple[set, set]: # sure, maybe + if q not in callers_memo: + d = gj(["callers", q], root) or [] + callers_memo[q] = ({c[0]["qualified"] for x in d for c in x.get("callers", [])}, {c[0]["qualified"] for x in d for c in x.get("maybe", [])}) + return callers_memo[q] + + callers = lambda q: callers_of(q)[0] # noqa: E731 + + for s in sample: + a = s["qualified"] + res["functions"] += 1 + imp = {n["symbol"]["qualified"] for x in (gj(["--tests", "impact", a], root) or []) for n in x.get("impact", [])} + for b in sorted(callers(a)): + res["caller_pairs"] += 1 + callees = {c[0] for x in (gj(["callees", b], root) or []) for c in x.get("resolved", [])} + if a not in callees: + res["not_in_callees"].append(f"{b} calls {a}") + if b not in imp: + res["not_in_impact"].append(f"{b} not in impact of {a}") + for c in sorted({c[0] for x in (gj(["callees", a], root) or []) for c in x.get("resolved", [])}): + if c == a: + continue # a function that calls itself is not listed among its own callers + res["callee_pairs"] += 1 + sure, maybe = callers_of(c) + if a in maybe and a not in sure: + res["sure_here_maybe_there"].append(f"{a} calls {c}: sure in callees, a guess in callers") + elif a not in sure: + res["not_in_callers"].append(f"{a} calls {c}") + return res + + +def check_read(root: Path, sample: list[dict]) -> dict: + res = {"symbols": 0, "wrong": []} + cache: dict = {} + for s in sample: + d = gj(["read", s["qualified"]], root) + if not d: + res["wrong"].append(f"{s['qualified']}: no answer") + continue + res["symbols"] += 1 + src = d[0]["source"] + m = re.match(r"(\S+):(\d+)-(\d+) [^\n]*\n```[^\n]*\n(.*)\n```\n$", src, re.S) + if not m: + res["wrong"].append(f"{s['qualified']}: unexpected shape") + continue + file, a, b, code = m.group(1), int(m.group(2)), int(m.group(3)), m.group(4) + real = "\n".join(lines_of(root, file, cache)[a - 1:b]) + if code != real: + res["wrong"].append(f"{s['qualified']}: the code differs from {file}:{a}-{b}") + elif s["name"] not in code: + res["wrong"].append(f"{s['qualified']}: its name is not in {file}:{a}-{b}") + elif not (a <= s["start_line"] and s["end_line"] <= b): + res["wrong"].append(f"{s['qualified']}: read {a}-{b} does not cover the symbol {s['start_line']}-{s['end_line']}") + return res + + +def check_symbols(root: Path, syms: list[dict]) -> dict: + res = {"symbols": 0, "wrong": []} + cache: dict = {} + by_q: dict[str, list[dict]] = {} + for s in syms: + by_q.setdefault(s["qualified"], []).append(s) + for s in syms: + if s["kind"] == "module" or "$anon" in s["name"]: # an anonymous class has a made-up name + continue + res["symbols"] += 1 + L = lines_of(root, s["file"], cache) + a, b = s["start_line"], s["end_line"] + if not (1 <= a <= b <= len(L)): + res["wrong"].append(f"{s['qualified']}: lines {a}-{b} of a file with {len(L)}") + elif s["name"] not in "\n".join(L[a - 1:b]): + res["wrong"].append(f"{s['qualified']}: the name is not in {s['file']}:{a}-{b}") + else: + # a method lies inside the type it belongs to, when that type is in the same file (not an `impl` elsewhere) + ps = [p for p in by_q.get(s.get("parent") or "", []) if p["file"] == s["file"] and p["kind"] != "module"] + if s["kind"] == "method" and ps and not any(p["start_line"] <= a and b <= p["end_line"] for p in ps) \ + and not s["file"].endswith((".rs", ".go", ".c", ".cc", ".cpp", ".h", ".hpp")): + res["wrong"].append(f"{s['qualified']}: {a}-{b} is outside its parent {ps[0]['start_line']}-{ps[0]['end_line']}") + return res + + +def check_search(root: Path, sample: list[dict]) -> dict: + res = {"symbols": 0, "not_found": []} + for s in sample: + d = gj(["--tests", "search", s["name"]], root) or [] + if len(d) >= 20 and not any(x.get("qualified") == s["qualified"] for x in d): + continue # `toString`, `fmt`: more namesakes than one answer holds — not a miss + res["symbols"] += 1 + if not any(x.get("qualified") == s["qualified"] for x in d): + res["not_found"].append(f"{s['qualified']} by `{s['name']}` ({len(d)} hits)") + return res + + +def check_edit(root: Path, sample: list[dict]) -> dict: + res = {"replaced": 0, "deleted": 0, "wrong": []} + clean = lambda: ev.restore(root) # noqa: E731 + cache: dict = {} + for s in sample: + q = s["qualified"] + d = gj(["read", q], root) + m = d and re.match(r"(\S+):(\d+)-(\d+) [^\n]*\n```[^\n]*\n(.*)\n```\n$", d[0]["source"], re.S) + if not m: + continue + file, a, b, code = m.group(1), int(m.group(2)), int(m.group(3)), m.group(4) + before = (root / file).read_text(errors="replace") + p = subprocess.run([ev.GITNEXT, "edit", "replace", q], input=code + "\n", cwd=root, capture_output=True, text=True) + res["replaced"] += 1 + if (root / file).read_text(errors="replace") != before: + diff = ev.sh(f"git diff --stat -- {file}", root)[1].strip().splitlines() + res["wrong"].append(f"replace {q} with its own code changed {file}: {diff[-1].strip() if diff else p.stdout[:80]}") + clean() + ev.gx(["index"], root) + p = subprocess.run([ev.GITNEXT, "edit", "delete", q], cwd=root, capture_output=True, text=True) + res["deleted"] += 1 + after = (root / file).read_text(errors="replace").split("\n") + old = lines_of(root, file, cache) if file in cache else before.split("\n") + # what is left must be the old file without one block of lines: the symbol, and at most + # three blank or comment lines around it (a diff by common prefix slides over repeated lines) + if after == old: + res["wrong"].append(f"delete {q}: nothing was removed ({(p.stdout + p.stderr).strip()[:100]})") + else: + blocks = [(lo, hi) for lo in range(max(1, a - 3), a + 1) for hi in range(b, min(len(old), b + 3) + 1) if old[:lo - 1] + old[hi:] == after] + if not blocks: + res["wrong"].append(f"delete {q}: what is left is not {file} without the lines around {a}-{b}") + elif not any(all(not x.strip() or x.lstrip().startswith(("#", "//", "*", "/*", "@")) for x in old[lo - 1:a - 1] + old[b:hi]) for lo, hi in blocks): + res["wrong"].append(f"delete {q}: removed code next to the symbol {a}-{b}") + clean() + ev.gx(["index"], root) + return res + + +def no_generics(sig: str) -> str: + while re.search(r"<[^<>]*>", sig): + sig = re.sub(r"<[^<>]*>", "", sig) + return sig + + +BASES = { + ".py": lambda sig: [b.strip().split("[")[0].split(".")[-1] for b in (re.search(r"class \w+\((.*)\)", sig) or [None, ""])[1].split(",") + if b.strip() and "=" not in b], + ".java": lambda sig: re.findall(r"[\w.]+", " ".join(re.findall(r"(?:extends|implements)\s+([\w.,\s]+?)(?=\s+(?:extends|implements|permits)\b|\s*\{|$)", no_generics(sig)))), + ".ts": lambda sig: re.findall(r"[\w.]+", " ".join(re.findall(r"(?:extends|implements)\s+([\w.,\s]+?)(?=\s+(?:extends|implements)\b|\s*\{|$)", no_generics(sig)))), +} + + +def check_hierarchy(root: Path, syms: list[dict]) -> dict: + res = {"types": 0, "bases": 0, "missing_base": [], "missing_subtype": []} + types = [s for s in syms if s["kind"] in ("class", "interface", "struct", "trait", "enum")] + by_name: dict[str, list[dict]] = {} + for t in types: + by_name.setdefault(t["name"], []).append(t) + for t in types: + ext = Path(t["file"]).suffix + ext = ".ts" if ext in (".tsx", ".mts") else ext + if ext not in BASES or len(by_name[t["name"]]) != 1: + continue + bases = [b.split(".")[-1] for b in BASES[ext](t.get("signature") or "")] + bases = [b for b in bases if len(by_name.get(b, [])) == 1 and b != t["name"]] # a base that is one type of this repo + if not bases: + continue + res["types"] += 1 + h = (gj(["--tests", "hierarchy", t["qualified"]], root) or [{}])[0] + up = {x.split("::")[-1].split(".")[-1] for x in h.get("extends", []) + h.get("implements", [])} + for b in bases: + res["bases"] += 1 + if b not in up: + res["missing_base"].append(f"{t['qualified']}: `{b}` is in its signature, not in extends / implements") + hb = (gj(["--tests", "hierarchy", by_name[b][0]["qualified"]], root) or [{}])[0] + down = {x["qualified"] for x in hb.get("subtypes", []) + hb.get("implementors", [])} + if t["qualified"] not in down: + res["missing_subtype"].append(f"{by_name[b][0]['qualified']}: `{t['name']}` extends it, not among its subtypes") + return res + + +def dead_entries(root: Path) -> dict[str, list[tuple[str, str, str, int]]]: + d = gj(["dead"], root) or {} + out = {} + for group in ("likely", "public_unused"): + out[group] = [] + for e in d.get(group, []): + m = re.match(r"(\w+) (\S+)\s+(\S+):(\d+)$", e) + if m: + out[group].append((m.group(1), m.group(2), m.group(3), int(m.group(4)))) + return out + + +def check_dead(root: Path, repo: dict, syms: list[dict], run: bool) -> dict: + ent = dead_entries(root) + res = {"likely": len(ent["likely"]), "public_unused": len(ent["public_unused"]), "named_elsewhere": [], "deleted": 0, "broke": None} + files = sorted({s["file"] for s in syms}) + text = {f: (root / f).read_text(errors="replace").split("\n") for f in files if (root / f).is_file()} + span = {(s["file"], s["start_line"]): s["end_line"] for s in syms} + for kind, name, file, line in ent["likely"]: + short = name.split("::")[-1] + end = span.get((file, line), line) + pat = re.compile(rf"(? 3 else ''}") + L = ev.LANG.get(repo["lang"]) + if not run or not L or not ent["likely"] or repo["lang"] not in ("python", "go", "rust", "typescript"): + return res + compiled = lambda: set(map(str, L["errors"](ev.sh(L["compile"], root, timeout=1800)[1], root))) if L.get("compile") and L.get("errors") else set() # noqa: E731 + before_c = compiled() + before_t = set(L["failed"](ev.sh(L["tests"], root, timeout=1800)[1], root)) + # bottom-up inside each file, so the lines of the ones still to delete do not move + for kind, name, file, line in sorted(ent["likely"], key=lambda e: (e[2], -e[3])): + p = subprocess.run([ev.GITNEXT, "edit", "delete", f"{name.split('::')[-1]}@{line}"], cwd=root, capture_output=True, text=True) + res["deleted"] += p.returncode == 0 + after_c = compiled() + out = ev.sh(L["tests"], root, timeout=1800)[1] + after_t = set(L["failed"](out, root)) + # pytest: a test module that no longer imports is an ERROR line without a test name + after_t |= {f"{m} (does not import)" for m in re.findall(r"^ERROR (\S+\.py)(?: - .*)?$", out, re.M)} + res["broke"] = {"compile_errors": sorted(after_c - before_c)[:8], "tests": sorted(after_t - before_t)[:8], "tests_ran": bool(L["ran"](out, root))} + ev.restore(root) + ev.gx(["index"], root) + return res + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--repos", default="") + ap.add_argument("--n", type=int, default=30, help="functions per project for symmetry / read / search") + ap.add_argument("--n-edit", type=int, default=10) + ap.add_argument("--seed", type=int, default=5) + ap.add_argument("--no-dead-run", action="store_true", help="skip deleting the dead code and running the tests") + ap.add_argument("--out", default=str(HERE / "results" / "consistency.json")) + args = ap.parse_args() + repos = [r for r in json.loads((HERE / "repos.json").read_text()) if not args.repos or r["name"] in args.repos.split(",")] + all_res = {} + for repo in repos: + t0 = time.time() + try: + root = ev.prepare(repo) + except RuntimeError as e: + print(f"{repo['name']}: skipped ({str(e)[:80]})", flush=True) + continue + ev.gx(["init"], root, timeout=1800) + syms = ev.symbols(root) + count: dict[str, int] = {} + for s in syms: + count[s["qualified"]] = count.get(s["qualified"], 0) + 1 + funcs = [s for s in syms if s["kind"] in ("function", "method") and count[s["qualified"]] == 1 and not s.get("test") + and not s["file"].startswith(("examples/", "example/", "docs/", "_examples/"))] + rng = random.Random(args.seed) + rng.shuffle(funcs) + sample = funcs[:args.n] + r = { + "symmetry": check_symmetry(root, sample), + "read": check_read(root, sample), + "symbols": check_symbols(root, syms), + "search": check_search(root, sample), + "edit": check_edit(root, sample[:args.n_edit]), + "hierarchy": check_hierarchy(root, syms), + "dead": check_dead(root, repo, syms, not args.no_dead_run), + } + ev.restore(root) + r["seconds"] = round(time.time() - t0) + all_res[repo["name"]] = r + sy, d = r["symmetry"], r["dead"] + print(f"{repo['name']:10} symmetry {sy['caller_pairs']}+{sy['callee_pairs']} pairs, off {len(sy['not_in_callees'])}/{len(sy['not_in_impact'])}/{len(sy['not_in_callers'])}/{len(sy['sure_here_maybe_there'])}" + f" · read off {len(r['read']['wrong'])}/{r['read']['symbols']} · symbols off {len(r['symbols']['wrong'])}/{r['symbols']['symbols']}" + f" · search off {len(r['search']['not_found'])} · edit off {len(r['edit']['wrong'])}/{r['edit']['replaced'] + r['edit']['deleted']}" + f" · hierarchy off {len(r['hierarchy']['missing_base'])}+{len(r['hierarchy']['missing_subtype'])}/{r['hierarchy']['bases']}" + f" · dead {d['likely']}, named elsewhere {len(d['named_elsewhere'])}, broke {d['broke']} ({r['seconds']}s)", flush=True) + Path(args.out).write_text(json.dumps(all_res, ensure_ascii=False, indent=1)) + write_md(all_res, Path(args.out).with_suffix(".md"), args) + bad = sum(len(r["symmetry"]["not_in_callees"]) + len(r["symmetry"]["not_in_impact"]) + len(r["symmetry"]["not_in_callers"]) + len(r["symmetry"]["sure_here_maybe_there"]) + len(r["read"]["wrong"]) + + len(r["symbols"]["wrong"]) + len(r["search"]["not_found"]) + len(r["edit"]["wrong"]) + len(r["hierarchy"]["missing_base"]) + + len(r["hierarchy"]["missing_subtype"]) + len(r["dead"]["named_elsewhere"]) + + (len(r["dead"]["broke"]["compile_errors"]) + len(r["dead"]["broke"]["tests"]) if r["dead"]["broke"] else 0) for r in all_res.values()) + raise SystemExit(1 if bad else 0) + + +def write_md(all_res: dict, path: Path, args) -> None: + md = ["# Do the commands agree with each other and with the source?", "", + f"`eval/consistency.py`, {args.n} functions per project (seed {args.seed}). What each column checks is at the top of the script.", + "A cell is `disagreements / checked`.", "", + "| project | callers→callees | callers→impact | callees→callers | callees sure, callers maybe | read | symbols | search | edit | hierarchy | dead: named elsewhere | dead deleted: new compile errors / failing tests |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"] + for name, r in all_res.items(): + sy, d, h = r["symmetry"], r["dead"], r["hierarchy"] + broke = "—" if not d["broke"] else f"{len(d['broke']['compile_errors'])} / {len(d['broke']['tests'])} ({d['deleted']} deleted)" + md.append(f"| {name} | {len(sy['not_in_callees'])} / {sy['caller_pairs']} | {len(sy['not_in_impact'])} / {sy['caller_pairs']} | {len(sy['not_in_callers'])} / {sy['callee_pairs']} | {len(sy['sure_here_maybe_there'])} / {sy['callee_pairs']}" + f" | {len(r['read']['wrong'])} / {r['read']['symbols']} | {len(r['symbols']['wrong'])} / {r['symbols']['symbols']} | {len(r['search']['not_found'])} / {r['search']['symbols']}" + f" | {len(r['edit']['wrong'])} / {r['edit']['replaced'] + r['edit']['deleted']} | {len(h['missing_base']) + len(h['missing_subtype'])} / {2 * h['bases']}" + f" | {len(d['named_elsewhere'])} / {d['likely']} | {broke} |") + md += ["", "## The disagreements", ""] + for name, r in all_res.items(): + items = ([("callers→callees", x) for x in r["symmetry"]["not_in_callees"]] + [("callers→impact", x) for x in r["symmetry"]["not_in_impact"]] + + [("callees→callers", x) for x in r["symmetry"]["not_in_callers"] + r["symmetry"]["sure_here_maybe_there"]] + [("read", x) for x in r["read"]["wrong"]] + + [("symbols", x) for x in r["symbols"]["wrong"]] + [("search", x) for x in r["search"]["not_found"]] + [("edit", x) for x in r["edit"]["wrong"]] + + [("hierarchy", x) for x in r["hierarchy"]["missing_base"] + r["hierarchy"]["missing_subtype"]] + + [("dead", x) for x in r["dead"]["named_elsewhere"]] + + ([("dead deleted", x) for x in r["dead"]["broke"]["compile_errors"] + r["dead"]["broke"]["tests"]] if r["dead"]["broke"] else [])) + if items: + md.append(f"**{name}**") + md += [f"- {k}: {x}" for k, x in items[:25]] + ([f"- … and {len(items) - 25} more"] if len(items) > 25 else []) + [""] + path.write_text("\n".join(md) + "\n") + print("\n".join(md[:8 + len(all_res)])) + + +if __name__ == "__main__": + main() diff --git a/eval/results/COVERAGE.md b/eval/results/COVERAGE.md index c4241e1..23966c6 100644 --- a/eval/results/COVERAGE.md +++ b/eval/results/COVERAGE.md @@ -1,6 +1,6 @@ # What checks each gitnext function -Three levels, strongest first. A command on the last level answers on real code but nobody has checked +Four levels, strongest first. A command on the last level answers on real code but nobody has checked that the answer is right. ## 1. Measured against the truth (10 projects, 5 languages; the truth is the project's own compiler and tests) @@ -17,7 +17,7 @@ that the answer is right. | `why`, `check ` | place of the function at fault for a seeded bug (`why_rank.py`, `why-rank.md`) | with the learned map the culprit is first in 81% of 57 seeded bugs (Python 90%, Go 82%, Rust 70%); from the call graph alone in 18% | | the tools as a small model uses them | `agent.py`, Qwen 3.5 4B: find / callers / rename / fix | see the agent runs in `HISTORY.md` | -## 2. Unit tests (`cargo test --workspace`, 115 tests) +## 2. Unit tests (`cargo test --workspace`, 122 tests) | function | tests | |---|---| @@ -30,27 +30,48 @@ that the answer is right. | `tests-for`, the learned map | `tests_for.rs`, `testmap.rs` (8) | | `learn-tests`: Go, Rust | unit tests of the parsers of coverage output, and one end-to-end test each (skipped without `go` / `llvm-tools`) | | `check`, `context`, `locate`, `outline` | 3, 4, 2, 1 | +| `look` (a place in a file), `edit`, `dead` | `look.rs` (3), `edit.rs` (3), `dead.rs` (1) | | semantic diff (`diff`, `since`, `changelog`), `libs`, `setup`, MCP server, TUI | 4, 3, 3, 2, 3 | -## 3. Answers on every project, correctness not checked (`eval/smoke.py`, `smoke.md`) +## 3. Agreement with each other and with the source (`eval/consistency.py`, `consistency.md`) + +No compiler is asked here; what can be checked without one is, on the 10 projects, 30 functions each: + +| function | what is checked | result | +|---|---|---| +| `callers` ↔ `callees` ↔ `impact` | B is a caller of A ⇒ A is a callee of B and B is in the impact of A, and the other way round; a call that is a guess for one is a guess for the other | 399 + 369 pairs, no disagreement (before: 5 calls sure in `callees`, "maybe" in `callers`) | +| `read` (and the code part of `look`, `show`) | the code printed is exactly the lines of the file it names | 300 symbols, all exact | +| `symbols`, `outline` | the lines exist, the name is in them, a method lies inside its type | 6642 symbols, all right | +| `search` | a symbol is found by its own name | 296, all found | +| `edit replace`, `edit delete` | a symbol replaced with its own code leaves the file as it was; a delete removes the symbol with its doc comment and nothing else | 200 edits, all right (before: 11 wrong in the two Java projects — a body line indented less than the first, tabs under spaces, a Javadoc without `*`) | +| `hierarchy` | the bases in a class's own signature (Python, Java, TypeScript) are in `extends` / `implements`, and the class is among their subtypes | 195 bases, both ways, all there | +| `dead` | the name is written nowhere else in the code; with everything it calls likely dead deleted, the project compiles and no test starts failing (Python, Go, Rust, TypeScript) | see below | + +`dead` before this check named 225 symbols "likely dead" on the 10 projects; 160 of them were named elsewhere in +the code — used as a value, a type, a base, an import, a callback, or a Go `init` — and deleting them broke the +build of at least 6 of the 8 projects where that is tried. Now the name must be written nowhere outside the symbol +itself: 64 are left. indexmap 57 — deleted, it compiles and its tests pass (not tried with every feature and the +benches built). sqlparse 5 — deleted, eight tests fail: the methods are reached by names built at run time +(`getattr(self, "_process_" + …)`). The two Java projects, one each — not tried (the delete run would need Maven +twice per project). The `public but unused` list is not checked. + +## 4. Answers on every project, correctness not checked (`eval/smoke.py`, `smoke.md`) Every command and every MCP tool is called once on each of the 10 projects with a real function, class, file and folder: 86 commands and tools × 10 = 860 calls, none failed, crashed, came back empty or took 20 s. -Only this level: `callees`, `hierarchy`, `deps`, `search`, `read`, `show`, `symbols`, `map`, `overview`, -`activity`, `hotspots`, `dead`, `cycles`, `entrypoints`, `history`, `brief`, `journal`, `bench`, `branches`, -`timeline`, `msg`, `status`, `add`, `reset`, `commit`, `undo`, `edit replace / insert / delete`, and `look` -as a whole (its parts are on level 1; that the parts are put together right is not checked). +Only this level: `deps`, `show` beyond its code, `map`, `overview`, `activity`, `hotspots`, `cycles`, +`entrypoints`, `history`, `brief`, `journal`, `bench`, `branches`, `timeline`, `msg`, `status`, `add`, `reset`, +`commit`, `undo`, `edit insert-after`, and `look` as a whole (its parts are on levels 1 and 3; that the parts are +put together right is not checked). ## Not checked at all `tui` beyond three unit tests, `lsp` (needs language servers), `explain` (needs a model), `hook`, `init` on an empty folder, what `doctor` and `setup` write for each agent. -## What would move level 3 up +## What would move things up -- `callees`, `impact`, `look`: must agree with `callers` — if B is among the callers of A, A is among the callees - of B and B is in the impact of A. One script over the eval's own truth. -- `dead`: delete what it names and see that the project still compiles and its tests pass. -- `hierarchy`: against compile errors after a method is removed from the base type. -- `search`, `read`, `outline`: exactness against the source (the lines printed are the lines of the symbol). +- `dead`: names built at run time; the delete run for Java and for Rust with every feature; the public list. +- `hierarchy`: against compile errors after a method is removed from the base type (now only against signatures). +- `look` as a whole: its callers, callees and tests must be the ones `callers`, `callees` and `tests-for` give. diff --git a/eval/results/HISTORY.md b/eval/results/HISTORY.md index 159c093..69f17e5 100644 --- a/eval/results/HISTORY.md +++ b/eval/results/HISTORY.md @@ -35,6 +35,27 @@ What the conversations showed, and what was changed after each run: the model called `check ` and the map was there, the culprit was first 5 times of 5 and 3 were solved; in the other two it did not see the wrong line — hence the changed lines in the answer of `why`. +### A trial on a rented GPU: the same `fix` tasks with a 27B model + +The harness on Kaggle (two T4 cards, llama.cpp with CUDA, the settings of the Mac runs), gitnext 0.3.0, the `fix` +tasks of click and semver with the coverage map, three bugs per project. One run, 6 tasks per cell — the counts of +solved tasks are too few to compare; the steps and tokens differ in every single task. + +| model | mode | solved | median steps | tokens sent, all 6 | seconds, all 6 | +|---|---|---:|---:|---:|---:| +| Qwen 3.5 27B | plain | 5 of 6 | 16 | 498 k | 1184 | +| | gitnext | 6 of 6 | 5 | 58 k | 378 | +| | lite | 6 of 6 | 7 | 105 k | 676 | +| Qwen 3.5 4B | gitnext | 2 of 6 | 16 | 499 k | 385 | +| | lite | 2 of 6 | 17 | 332 k | 322 | + +- The 27B model with gitnext does the same thing every time: `check `, one edit, `check`, done — 4 or 5 steps. +- The 4B model: 37 of its and the 27B model's 89 `look` calls named a place the way a traceback does + (`src/eval.rs:62`, `tests/test_options.py:460-500`, `eval.rs::matches_greater`) and got "nothing found"; with the + five tools of `lite` there is no other way to read lines, and three of its four lost `lite` runs ended at the step + limit on such calls. `look` takes these now. In the three lost `gitnext` runs out of four the model named the bug + and stopped without an edit. One run ended on a `replace` cut by the token limit — the harness asks again now. + ## Run 8 · in TypeScript the test is the file; coverage maps for Go and Rust **tests-for, TypeScript.** `it('…', () => …)` is not a named function; its calls are booked to the module of the diff --git a/eval/results/consistency.json b/eval/results/consistency.json new file mode 100644 index 0000000..3c566d0 --- /dev/null +++ b/eval/results/consistency.json @@ -0,0 +1,439 @@ +{ + "cobra": { + "symmetry": { + "functions": 30, + "caller_pairs": 59, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 79, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 288, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 0, + "bases": 0, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 23, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 7 + }, + "chi": { + "symmetry": { + "functions": 30, + "caller_pairs": 19, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 33, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 351, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 0, + "bases": 0, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 6, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 5 + }, + "click": { + "symmetry": { + "functions": 30, + "caller_pairs": 26, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 32, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 761, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 45, + "bases": 47, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 3, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 13 + }, + "sqlparse": { + "symmetry": { + "functions": 30, + "caller_pairs": 55, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 49, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 287, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 25, + "bases": 27, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 5, + "public_unused": 2, + "named_elsewhere": [], + "deleted": 5, + "broke": { + "compile_errors": [], + "tests": [ + "test_duplicate_linebreaks", + "test_except_formatting", + "test_format_invalid_where_clause", + "test_having_produces_newline", + "test_identifier_list", + "test_identifier_list_comment_first", + "test_identifier_list_with_wrap_after", + "test_issue315_utf8_by_default" + ], + "tests_ran": true + } + }, + "seconds": 9 + }, + "zustand": { + "symmetry": { + "functions": 30, + "caller_pairs": 21, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 24, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 175, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 0, + "bases": 0, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 0, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 5 + }, + "yaml": { + "symmetry": { + "functions": 30, + "caller_pairs": 47, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 36, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 440, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 16, + "bases": 16, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 2, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 9 + }, + "jsoup": { + "symmetry": { + "functions": 30, + "caller_pairs": 67, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 19, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 2661, + "wrong": [] + }, + "search": { + "symbols": 27, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 100, + "bases": 100, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 1, + "public_unused": 2, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 244 + }, + "json-java": { + "symmetry": { + "functions": 30, + "caller_pairs": 51, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 42, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 540, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 5, + "bases": 5, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 1, + "public_unused": 0, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 18 + }, + "semver": { + "symmetry": { + "functions": 30, + "caller_pairs": 45, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 25, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 113, + "wrong": [] + }, + "search": { + "symbols": 30, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 0, + "bases": 0, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 0, + "public_unused": 1, + "named_elsewhere": [], + "deleted": 0, + "broke": null + }, + "seconds": 3 + }, + "indexmap": { + "symmetry": { + "functions": 30, + "caller_pairs": 9, + "not_in_callees": [], + "not_in_impact": [], + "callee_pairs": 30, + "not_in_callers": [], + "sure_here_maybe_there": [] + }, + "read": { + "symbols": 30, + "wrong": [] + }, + "symbols": { + "symbols": 1026, + "wrong": [] + }, + "search": { + "symbols": 29, + "not_found": [] + }, + "edit": { + "replaced": 10, + "deleted": 10, + "wrong": [] + }, + "hierarchy": { + "types": 0, + "bases": 0, + "missing_base": [], + "missing_subtype": [] + }, + "dead": { + "likely": 57, + "public_unused": 12, + "named_elsewhere": [], + "deleted": 57, + "broke": { + "compile_errors": [], + "tests": [], + "tests_ran": true + } + }, + "seconds": 14 + } +} \ No newline at end of file diff --git a/eval/results/consistency.md b/eval/results/consistency.md new file mode 100644 index 0000000..512caab --- /dev/null +++ b/eval/results/consistency.md @@ -0,0 +1,30 @@ +# Do the commands agree with each other and with the source? + +`eval/consistency.py`, 30 functions per project (seed 5). What each column checks is at the top of the script. +A cell is `disagreements / checked`. + +| project | callers→callees | callers→impact | callees→callers | callees sure, callers maybe | read | symbols | search | edit | hierarchy | dead: named elsewhere | dead deleted: new compile errors / failing tests | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| cobra | 0 / 59 | 0 / 59 | 0 / 79 | 0 / 79 | 0 / 30 | 0 / 288 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | +| chi | 0 / 19 | 0 / 19 | 0 / 33 | 0 / 33 | 0 / 30 | 0 / 351 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | +| click | 0 / 26 | 0 / 26 | 0 / 32 | 0 / 32 | 0 / 30 | 0 / 761 | 0 / 30 | 0 / 20 | 0 / 94 | 0 / 0 | — | +| sqlparse | 0 / 55 | 0 / 55 | 0 / 49 | 0 / 49 | 0 / 30 | 0 / 287 | 0 / 30 | 0 / 20 | 0 / 54 | 0 / 5 | 0 / 8 (5 deleted) | +| zustand | 0 / 21 | 0 / 21 | 0 / 24 | 0 / 24 | 0 / 30 | 0 / 175 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | +| yaml | 0 / 47 | 0 / 47 | 0 / 36 | 0 / 36 | 0 / 30 | 0 / 440 | 0 / 30 | 0 / 20 | 0 / 32 | 0 / 0 | — | +| jsoup | 0 / 67 | 0 / 67 | 0 / 19 | 0 / 19 | 0 / 30 | 0 / 2661 | 0 / 27 | 0 / 20 | 0 / 200 | 0 / 1 | — | +| json-java | 0 / 51 | 0 / 51 | 0 / 42 | 0 / 42 | 0 / 30 | 0 / 540 | 0 / 30 | 0 / 20 | 0 / 10 | 0 / 1 | — | +| semver | 0 / 45 | 0 / 45 | 0 / 25 | 0 / 25 | 0 / 30 | 0 / 113 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | +| indexmap | 0 / 9 | 0 / 9 | 0 / 30 | 0 / 30 | 0 / 30 | 0 / 1026 | 0 / 29 | 0 / 20 | 0 / 0 | 0 / 57 | 0 / 0 (57 deleted) | + +## The disagreements + +**sqlparse** +- dead deleted: test_duplicate_linebreaks +- dead deleted: test_except_formatting +- dead deleted: test_format_invalid_where_clause +- dead deleted: test_having_produces_newline +- dead deleted: test_identifier_list +- dead deleted: test_identifier_list_comment_first +- dead deleted: test_identifier_list_with_wrap_after +- dead deleted: test_issue315_utf8_by_default + From 14bcb17d6bbb702eeaac7d65402fd0851c536a1e Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 02:54:56 +0300 Subject: [PATCH 05/18] README: DOI badge (Zenodo, the latest archived release); CITATION.cff Co-Authored-By: Claude Fable 5.2 --- CITATION.cff | 18 ++++++++++++++++++ README.md | 3 ++- README.ru.md | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..61c1a7b --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,18 @@ +cff-version: 1.2.0 +message: "If you use gitnext or its evaluation, please cite it." +title: "gitnext: code navigation and symbol-level editing tools for AI agents" +type: software +authors: + - family-names: Kononenko + given-names: Leonid A. + orcid: "https://orcid.org/0009-0009-8407-9214" + affiliation: Independent researcher +repository-code: "https://github.com/pnnnx/gitnext" +license: MIT +keywords: + - AI agents + - code navigation + - call graph + - tree-sitter + - evaluation + - small language models diff --git a/README.md b/README.md index acf8132..cfaf179 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@

CI MIT + DOI Rust

@@ -56,7 +57,7 @@ Most of an agent session is the system prompt re-sent on every step, so the win ### What is checked, and what is not good yet -- 115 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 122 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` finds 19% of the failing test files (it was 3%); the coverage map exists for Python, Go and Rust only. - A rename breaks the build or the tests in 9% of cases — mostly dynamic access and names shared by unrelated types. - A 4B model rarely fixes a seeded bug yet: 3 of 12 on click. diff --git a/README.ru.md b/README.ru.md index 612462c..72e2a52 100644 --- a/README.ru.md +++ b/README.ru.md @@ -8,6 +8,7 @@

CI MIT + DOI Rust

@@ -57,7 +58,7 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он ### Что проверено и что пока плохо -- 115 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 122 модульных теста; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` находит 19% падающих файлов тестов (было 3%); карта покрытия есть только для Python, Go и Rust. - Переименование ломает сборку или тесты в 9% случаев — в основном динамический доступ и одинаковые имена у несвязанных типов. - Модель на 4B пока редко чинит подсаженную ошибку: 3 из 12 на click. From fb84f6d1967479b2fb643fdcc2c242489cf0aa3c Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 03:58:35 +0300 Subject: [PATCH 06/18] rust: functions declared in a macro call are symbols; tests in quickcheck! and the like Rust tests are often declared in the body of a macro call: `quickcheck! { fn prop(..) -> bool { .. } }`, `proptest! { #[test] fn t(..) { .. } }`, or a wrapper of the project's own like indexmap's `quickcheck_limit!`. The parser did not look into a macro called where items go, so in indexmap 43 of 192 tests were not functions of the graph: learn-tests could not run them, tests-for did not name them, and why found no test for 4 of 10 seeded bugs. The body of such a call (file, module, impl or trait level) is parsed again as code: the same source, only the bytes between the brackets, so lines, columns and calls are those of a real function. Its `fn` items become symbols; a stretch that is not Rust (`x in 0..10`) keeps the function, without arguments. They are tests in a test file, under #[test], or when the macro is a test macro: quickcheck*, proptest*, or a macro_rules! of the same file that writes #[test]. A test such a macro writes takes no arguments (they are the property's), so its arguments stay in the signature only and why can run it. Macros inside functions are not items and are left as they were. PARSER_VERSION 31 -> 32. Measured on semver and indexmap (eval/why_rank.py --bugs 10 --learn, the same bugs): - indexmap tests outside the graph: 43 of 192 -> 4 (the 4 get their name as a macro argument, `move_index_oob!(name, 0, 10);`); semver: 0 -> 0 - learned map, indexmap: 146 tests, 3865 pairs -> 185 tests, 5202-5208 pairs - why, the culprit first: 70% -> 80-90% in three runs (indexmap 40% -> 60-80%); in the first 8: 80% -> 100%; empty lists: 4 -> 0. quickcheck draws new inputs on each run, so the learned map moves a little and two bugs swap between first and third place; eval/results/why-rank-rust.md is the first run - eval/run.py --repos semver,indexmap: callers 99.3% / 100% unchanged, rename 86.8% -> 89.5% (+-8), tests recall 52.8% -> 61.4% (+-13), caught by the top 10 72.7% -> 81.8% - gitnext index on indexmap, from scratch: median 114 ms -> 121 ms Co-Authored-By: Claude Opus 5.5 --- .../gitnext-index/tests/rust_macro_tests.rs | 92 +++++++++++ crates/gitnext-parse/src/lib.rs | 2 +- crates/gitnext-parse/src/rust.rs | 143 ++++++++++++++++- .../tests/fixtures/rust/macros.rs | 80 ++++++++++ .../tests/fixtures/rust/macros.rs.expected | 84 ++++++++++ eval/results/why-rank-rust.json | 150 +++++++++--------- eval/results/why-rank-rust.md | 14 +- 7 files changed, 480 insertions(+), 85 deletions(-) create mode 100644 crates/gitnext-index/tests/rust_macro_tests.rs create mode 100644 crates/gitnext-parse/tests/fixtures/rust/macros.rs create mode 100644 crates/gitnext-parse/tests/fixtures/rust/macros.rs.expected diff --git a/crates/gitnext-index/tests/rust_macro_tests.rs b/crates/gitnext-index/tests/rust_macro_tests.rs new file mode 100644 index 0000000..81d8148 --- /dev/null +++ b/crates/gitnext-index/tests/rust_macro_tests.rs @@ -0,0 +1,92 @@ +//! Rust tests declared inside a macro call (`quickcheck! { fn prop(..) -> bool { .. } }`, indexmap's +//! `quickcheck_limit!`): they are functions of the graph, marked as tests, and the library functions they call +//! have them as callers. + +use gitnext_index::{Graph, Index}; + +fn graph(name: &str, files: &[(&str, &str)]) -> Graph { + let dir = std::env::temp_dir().join(format!("gitnext-rust-macro-tests-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for (path, code) in files { + let p = dir.join(path); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, code).unwrap(); + } + let mut idx = Index::open(&dir).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + g +} + +const CARGO: &str = "[package]\nname = \"demo\"\nversion = \"0.1.0\"\n"; + +const LIB: &str = r#"pub struct Bag { + items: Vec, +} + +impl Bag { + pub fn new() -> Self { + Bag { items: Vec::new() } + } + pub fn insert(&mut self, x: u32) { + self.items.push(x); + } + pub fn holds(&self, x: u32) -> bool { + self.items.contains(&x) + } +} +"#; + +const QUICK: &str = r#"use demo::Bag; + +macro_rules! quickcheck_limit { + { $( fn $f:ident($($a:ident : $t:ty),*) -> $r:ty { $($c:tt)* } )* } => { + quickcheck::quickcheck! { $( #[test] fn $f() {} )* } + }; +} + +fn filled(xs: &[u32]) -> Bag { + let mut bag = Bag::new(); + for &x in xs { + bag.insert(x); + } + bag +} + +quickcheck_limit! { + fn inserted_are_contained(xs: Vec) -> bool { + let bag = filled(&xs); + xs.iter().all(|&x| bag.holds(x)) + } + + fn empty_contains_nothing(x: u32) -> bool { + !Bag::new().holds(x) + } +} +"#; + +#[test] +fn tests_in_a_macro_are_tests_of_the_graph() { + let g = graph("quick", &[("Cargo.toml", CARGO), ("src/lib.rs", LIB), ("tests/quick.rs", QUICK)]); + for t in ["inserted_are_contained", "empty_contains_nothing"] { + let found = g.find(t); + assert_eq!(found.len(), 1, "{t}: {found:?}"); + let s = &g.syms[found[0]].sym; + assert!(s.test, "{t} is a test"); + assert!(s.params.is_empty(), "{t}: what cargo runs is `fn {t}()`"); + assert_eq!(s.start_line, if t == "inserted_are_contained" { 18 } else { 23 }); + } + let callers_of = |target: &str| -> Vec { + let found = g.find(target); + assert_eq!(found.len(), 1, "{target}: {found:?}"); + let mut names: Vec = g.callers(found[0]).into_iter().map(|(c, _)| g.syms[c].sym.name.clone()).collect(); + names.sort(); + names.dedup(); + names + }; + assert_eq!(callers_of("filled"), vec!["inserted_are_contained"]); + assert!(callers_of("Bag::new").contains(&"empty_contains_nothing".to_string())); + let contains = callers_of("Bag::holds"); + assert!(contains.contains(&"inserted_are_contained".to_string()) && contains.contains(&"empty_contains_nothing".to_string()), "{contains:?}"); +} diff --git a/crates/gitnext-parse/src/lib.rs b/crates/gitnext-parse/src/lib.rs index 496ea79..9cf0241 100644 --- a/crates/gitnext-parse/src/lib.rs +++ b/crates/gitnext-parse/src/lib.rs @@ -9,7 +9,7 @@ //! TypeScript, JavaScript, Go and Java. /// Bump when extractor output changes; the index re-parses everything on mismatch. -pub const PARSER_VERSION: u32 = 31; +pub const PARSER_VERSION: u32 = 32; mod cfamily; mod generic; diff --git a/crates/gitnext-parse/src/rust.rs b/crates/gitnext-parse/src/rust.rs index b885cf3..cef5815 100644 --- a/crates/gitnext-parse/src/rust.rs +++ b/crates/gitnext-parse/src/rust.rs @@ -1,7 +1,7 @@ //! Rust extractor: walks the tree-sitter CST and emits symbols and edges. use crate::{name_pos, CrateHint, first_sentence, hash_hex, Edge, EdgeKind, FileAnalysis, Kind, Param, Symbol}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Component, Path}; use tree_sitter::{Node, Parser}; @@ -17,7 +17,14 @@ pub fn analyze(rel: &Path, src: &str, krate: Option<&CrateHint>) -> anyhow::Resu None => module_path(rel), }; - let mut ex = Extractor { src: src.as_bytes(), out: FileAnalysis::default(), vars: HashMap::new(), fields: field_types(root, src.as_bytes()), owner_ty: None }; + let mut ex = Extractor { + src: src.as_bytes(), + out: FileAnalysis::default(), + vars: HashMap::new(), + fields: field_types(root, src.as_bytes()), + owner_ty: None, + test_macros: test_macros(root, src.as_bytes()), + }; ex.out.module_doc = inner_doc(root, ex.src); // File-level module symbol: anchor for `use` edges and the file's purpose. @@ -115,6 +122,8 @@ struct Extractor<'a> { fields: HashMap>>, /// name of the type whose `impl` the current function is in (`Self` in field lookups) owner_ty: Option, + /// `macro_rules!` of this file whose rules expand to `#[test]` functions (indexmap's `quickcheck_limit!`) + test_macros: HashSet, } impl<'a> Extractor<'a> { @@ -143,6 +152,13 @@ impl<'a> Extractor<'a> { "const_item" => self.simple(n, ctx, Kind::Const), "static_item" => self.simple(n, ctx, Kind::Static), "macro_definition" => self.simple(n, ctx, Kind::Macro), + "macro_invocation" => self.macro_items(n, ctx), + // `foo!(…);` + "expression_statement" => { + if let Some(m) = n.named_child(0).filter(|m| m.kind() == "macro_invocation") { + self.macro_items(m, ctx); + } + } "extern_crate_declaration" => { if let Some(name) = n.child_by_field_name("name") { let to = self.text(name).to_string(); @@ -362,6 +378,65 @@ impl<'a> Extractor<'a> { self.container(body, &inner); } + /// Functions declared in the body of a macro called where items go (`quickcheck! { fn prop(..) -> bool { .. } }`, + /// `proptest! { #[test] fn t(..) { .. } }`): the body, a token tree to the parser, is parsed again as code — + /// the same source, only the bytes between the brackets — so lines, columns and calls are those of a real function. + /// They are tests in a test file, under `#[test]`, or when the macro is a test macro: `quickcheck*`, `proptest*`, + /// or a `macro_rules!` of this file that expands to `#[test]`. + fn macro_items(&mut self, n: Node, ctx: &Ctx) { + let mut cur = n.walk(); + let Some(tt) = n.named_children(&mut cur).filter(|t| t.kind() == "token_tree").last() else { return }; + if tt.end_byte() < tt.start_byte() + 2 { + return; + } + let name = n.child_by_field_name("macro").map(|m| self.text(m)).unwrap_or(""); + let name = name.rsplit("::").next().unwrap_or(name); + let test = name.starts_with("quickcheck") || name.starts_with("proptest") || self.test_macros.contains(name); + let (s, e) = (tt.start_position(), tt.end_position()); + let inner = tree_sitter::Range { + start_byte: tt.start_byte() + 1, + end_byte: tt.end_byte() - 1, + start_point: tree_sitter::Point { row: s.row, column: s.column + 1 }, + end_point: tree_sitter::Point { row: e.row, column: e.column.saturating_sub(1) }, + }; + let mut parser = Parser::new(); + if parser.set_language(&tree_sitter_rust::LANGUAGE.into()).is_err() || parser.set_included_ranges(&[inner]).is_err() { + return; + } + let Some(tree) = parser.parse(self.src, None) else { return }; + let inner_ctx = Ctx { + module: ctx.module.clone(), + owner: ctx.owner.clone(), + trait_impl: ctx.trait_impl.clone(), + in_trait: ctx.in_trait, + in_test: ctx.in_test || test, + }; + self.macro_functions(tree.root_node(), &inner_ctx, test); + } + + /// The `fn` items at the top of a re-parsed macro body; a stretch that is not Rust (`x in 0..10`) is an `ERROR` + /// node the functions may sit in. A test such a macro writes takes no arguments — those of the declaration are + /// the property's (`fn prop(xs: Vec)` inside `#[test] fn name()`) — so a test keeps them in its signature only. + fn macro_functions(&mut self, n: Node, ctx: &Ctx, test_macro: bool) { + let mut cur = n.walk(); + let children: Vec = n.named_children(&mut cur).collect(); + for c in children { + match c.kind() { + "function_item" => { + let before = self.out.symbols.len(); + self.function(c, ctx); + if test_macro || has_test_attr(c, self.src) { + if let Some(s) = self.out.symbols.get_mut(before) { + s.params.clear(); + } + } + } + "ERROR" => self.macro_functions(c, ctx, test_macro), + _ => {} + } + } + } + fn simple(&mut self, n: Node, ctx: &Ctx, kind: Kind) { let Some(name) = n.child_by_field_name("name") else { return }; let mut sym = self.base(n, ctx, kind, self.text(name).to_string()); @@ -792,6 +867,26 @@ fn has_test_attr(n: Node, src: &[u8]) -> bool { false } +/// Names of the `macro_rules!` in a file whose rules write `#[test]`: what they are called with are tests. +fn test_macros(root: Node, src: &[u8]) -> HashSet { + let mut out = HashSet::new(); + let mut stack = vec![root]; + while let Some(n) = stack.pop() { + if n.kind() == "macro_definition" { + if let Some(name) = n.child_by_field_name("name") { + let body = n.utf8_text(src).unwrap_or(""); + if body.contains("#[test]") { + out.insert(name.utf8_text(src).unwrap_or("").to_string()); + } + } + continue; + } + let mut cur = n.walk(); + stack.extend(n.named_children(&mut cur).filter(|c| !matches!(c.kind(), "function_item" | "token_tree"))); + } + out +} + fn visibility(n: Node, src: &[u8]) -> Option { let mut cur = n.walk(); let v = n @@ -944,6 +1039,50 @@ fn helper(x: u32) -> Option { None } assert!(flag("tests") && flag("t") && flag("top")); } + #[test] + fn functions_in_macro_bodies() { + let src = r#"macro_rules! check { + { $(fn $f:ident($($a:ident : $t:ty),*) -> $r:ty { $($c:tt)* })* } => { $( #[test] fn $f() {} )* }; +} +fn helper(x: u32) -> u32 { x } +check! { + fn doubles(x: u32) -> bool { + let v = Vec::new(); + helper(x) == x && v.is_empty() + } +} +quickcheck::quickcheck! { + fn qc(x: u8) -> bool { helper(x as u32) > 0 } +} +proptest! { + #[test] + fn pt(x in 0..10u32) { helper(x); } +} +thread_local! { static DEPTH: u32 = 0; } +make_fns!(fn generated() -> u32 { helper(1) }); +impl Thing { + getters! { fn width(&self) -> u32 { self.w } } +} +fn outer() { inner! { fn hidden() {} } } +"#; + let a = analyze(Path::new("src/lib.rs"), src, None).unwrap(); + let sym = |n: &str| a.symbols.iter().find(|s| s.name == n).unwrap_or_else(|| panic!("{n}: {:?}", a.symbols.iter().map(|s| &s.name).collect::>())); + let d = sym("doubles"); + assert_eq!((d.start_line, d.end_line, d.test), (6, 9, true), "a local macro that writes #[test]"); + assert_eq!(d.signature, "fn doubles(x: u32) -> bool"); + assert!(sym("qc").test, "quickcheck!"); + assert!(sym("pt").test, "#[test] inside proptest!"); + assert!(!sym("generated").test, "not a test macro"); + assert!(d.params.is_empty() && sym("qc").params.is_empty(), "the test the macro writes takes no arguments"); + assert_eq!(sym("width").params.len(), 1, "not a test: the arguments stay"); + assert_eq!(sym("width").qualified, "crate::Thing::width"); + assert!(!a.symbols.iter().any(|s| s.name == "hidden"), "macros inside functions are not items"); + let calls = |from: &str, to: &str| a.edges.iter().any(|e| e.from.ends_with(from) && e.to == to && e.kind == EdgeKind::Calls); + assert!(calls("::doubles", "helper") && calls("::qc", "helper") && calls("::pt", "helper") && calls("::generated", "helper")); + let at = a.edges.iter().find(|e| e.from.ends_with("::doubles") && e.to == "helper").unwrap(); + assert_eq!((at.line, at.col), (8, 8)); + } + #[test] fn blanket_impls_skipped() { let a = analyze(Path::new("src/lib.rs"), "trait T {} impl T for &M {} impl T for X {}", None).unwrap(); diff --git a/crates/gitnext-parse/tests/fixtures/rust/macros.rs b/crates/gitnext-parse/tests/fixtures/rust/macros.rs new file mode 100644 index 0000000..55e9898 --- /dev/null +++ b/crates/gitnext-parse/tests/fixtures/rust/macros.rs @@ -0,0 +1,80 @@ +//! Functions declared inside macro calls: property tests and generated items. + +use std::collections::HashMap; + +/// Counts words. +pub struct Counter { + words: HashMap, +} + +impl Counter { + pub fn new() -> Self { + Counter { words: HashMap::new() } + } + + pub fn add(&mut self, w: &str) { + *self.words.entry(w.to_string()).or_default() += 1; + } + + pub fn get(&self, w: &str) -> usize { + self.words.get(w).copied().unwrap_or(0) + } + + accessors! { + /// How many different words. + pub fn distinct(&self) -> usize { self.words.len() } + } +} + +fn normalize(w: &str) -> String { + w.to_lowercase() +} + +// A wrapper like indexmap's `quickcheck_limit!`: its rules write `#[test]`. +macro_rules! props { + { $( fn $name:ident($($a:ident : $t:ty),*) -> bool { $($body:tt)* } )* } => { + $( #[test] fn $name() { fn prop($($a: $t),*) -> bool { $($body)* } let _ = prop; } )* + }; +} + +props! { + /// Adding a word makes it count once. + fn add_then_get(w: String) -> bool { + let mut c = Counter::new(); + c.add(&w); + c.get(&w) == 1 + } + + fn normalize_is_idempotent(w: String) -> bool { + normalize(&normalize(&w)) == normalize(&w) + } +} + +quickcheck::quickcheck! { + fn distinct_never_exceeds_adds(ws: Vec) -> bool { + let mut c = Counter::new(); + for w in &ws { + c.add(w); + } + c.distinct() <= ws.len() + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(16))] + + #[test] + fn get_of_unknown_is_zero(w in "[a-z]{1,8}") { + prop_assert_eq!(Counter::new().get(&w), 0); + } +} + +thread_local! { + static SEEN: std::cell::Cell = std::cell::Cell::new(0); +} + +generate!(fn generated_helper() -> String { normalize("X") }); + +fn not_items() { + inner! { fn hidden() {} } +} diff --git a/crates/gitnext-parse/tests/fixtures/rust/macros.rs.expected b/crates/gitnext-parse/tests/fixtures/rust/macros.rs.expected new file mode 100644 index 0000000..aac4253 --- /dev/null +++ b/crates/gitnext-parse/tests/fixtures/rust/macros.rs.expected @@ -0,0 +1,84 @@ +module doc: Functions declared inside macro calls: property tests and generated items. +symbols: +L1-81 module crate::rust::macros [pub] + sig: mod crate::rust::macros + doc: Functions declared inside macro calls: property tests and generated items. +L6-8 struct crate::rust::macros::Counter [pub] + sig: pub struct Counter + parent: crate::rust::macros + in: words: HashMap + doc: Counts words. +L11-13 method crate::rust::macros::Counter::new [pub] + sig: pub fn new() -> Self + parent: crate::rust::macros::Counter + out: Self +L15-17 method crate::rust::macros::Counter::add [pub] + sig: pub fn add(&mut self, w: &str) + parent: crate::rust::macros::Counter + in: self: &mut self, w: &str +L19-21 method crate::rust::macros::Counter::get [pub] + sig: pub fn get(&self, w: &str) -> usize + parent: crate::rust::macros::Counter + in: self: &self, w: &str + out: usize +L25-25 method crate::rust::macros::Counter::distinct [pub] + sig: pub fn distinct(&self) -> usize + parent: crate::rust::macros::Counter + in: self: &self + out: usize + doc: How many different words. +L29-31 function crate::rust::macros::normalize + sig: fn normalize(w: &str) -> String + parent: crate::rust::macros + in: w: &str + out: String +L34-38 macro crate::rust::macros::props + sig: macro_rules! props + parent: crate::rust::macros +L42-46 function crate::rust::macros::add_then_get (test) + sig: fn add_then_get(w: String) -> bool + parent: crate::rust::macros + out: bool + doc: Adding a word makes it count once. +L48-50 function crate::rust::macros::normalize_is_idempotent (test) + sig: fn normalize_is_idempotent(w: String) -> bool + parent: crate::rust::macros + out: bool +L54-60 function crate::rust::macros::distinct_never_exceeds_adds (test) + sig: fn distinct_never_exceeds_adds(ws: Vec) -> bool + parent: crate::rust::macros + out: bool +L67-69 function crate::rust::macros::get_of_unknown_is_zero (test) + sig: fn get_of_unknown_is_zero(w in "[a-z]{1,8}") + parent: crate::rust::macros +L76-76 function crate::rust::macros::generated_helper + sig: fn generated_helper() -> String + parent: crate::rust::macros + out: String +L78-80 function crate::rust::macros::not_items + sig: fn not_items() + parent: crate::rust::macros +edges: + L3 imports crate::rust::macros -> std::collections::HashMap + L12 calls crate::rust::macros::Counter::new -> HashMap::new + L12 uses crate::rust::macros::Counter::new -> Counter + L16 calls crate::rust::macros::Counter::add -> .entry + L16 calls crate::rust::macros::Counter::add -> .or_default + L16 calls crate::rust::macros::Counter::add -> .to_string + L20 calls crate::rust::macros::Counter::get -> .copied + L20 calls crate::rust::macros::Counter::get -> .get + L20 calls crate::rust::macros::Counter::get -> .unwrap_or + L25 calls crate::rust::macros::Counter::distinct -> .len + L30 calls crate::rust::macros::normalize -> .to_lowercase + L43 calls crate::rust::macros::add_then_get -> Counter::new + L44 calls crate::rust::macros::add_then_get -> Counter::add + L45 calls crate::rust::macros::add_then_get -> Counter::get + L49 calls crate::rust::macros::normalize_is_idempotent -> normalize + L55 calls crate::rust::macros::distinct_never_exceeds_adds -> Counter::new + L57 calls crate::rust::macros::distinct_never_exceeds_adds -> Counter::add + L59 calls crate::rust::macros::distinct_never_exceeds_adds -> .len + L59 calls crate::rust::macros::distinct_never_exceeds_adds -> Counter::distinct + L68 calls crate::rust::macros::get_of_unknown_is_zero -> .get + L68 calls crate::rust::macros::get_of_unknown_is_zero -> Counter::new + L76 calls crate::rust::macros::generated_helper -> normalize + L79 calls crate::rust::macros::not_items -> hidden diff --git a/eval/results/why-rank-rust.json b/eval/results/why-rank-rust.json index b2b2938..1244d9b 100644 --- a/eval/results/why-rank-rust.json +++ b/eval/results/why-rank-rust.json @@ -334,9 +334,9 @@ "lang": "rust", "target": "indexmap::inner::Core::increment_indices", "variant": "static", - "test": "shift_insert", - "place": 49, - "listed": 94, + "test": "occupied_entry_move_index", + "place": 27, + "listed": 30, "named": false, "learned": false }, @@ -345,9 +345,9 @@ "lang": "rust", "target": "indexmap::inner::Core::increment_indices", "variant": "nochange", - "test": "shift_insert", - "place": 14, - "listed": 42, + "test": "occupied_entry_move_index", + "place": 7, + "listed": 38, "named": false, "learned": true }, @@ -356,9 +356,9 @@ "lang": "rust", "target": "indexmap::inner::Core::increment_indices", "variant": "default", - "test": "shift_insert", - "place": 5, - "listed": 42, + "test": "occupied_entry_move_index", + "place": 4, + "listed": 38, "named": false, "learned": true }, @@ -367,9 +367,9 @@ "lang": "rust", "target": "indexmap::inner::Core::move_index_inner", "variant": "static", - "test": "insert_sorted_bad", + "test": "indexed_entry_move_index", "place": 1, - "listed": 123, + "listed": 29, "named": true, "learned": false }, @@ -378,9 +378,9 @@ "lang": "rust", "target": "indexmap::inner::Core::move_index_inner", "variant": "nochange", - "test": "insert_sorted_bad", + "test": "indexed_entry_move_index", "place": 1, - "listed": 46, + "listed": 36, "named": true, "learned": true }, @@ -389,9 +389,9 @@ "lang": "rust", "target": "indexmap::inner::Core::move_index_inner", "variant": "default", - "test": "insert_sorted_bad", + "test": "indexed_entry_move_index", "place": 1, - "listed": 46, + "listed": 36, "named": true, "learned": true }, @@ -400,9 +400,9 @@ "lang": "rust", "target": "indexmap::inner::entry::OccupiedEntry::swap_indices", "variant": "static", - "test": null, - "place": null, - "listed": 0, + "test": "occupied_entry_swap_indices", + "place": 1, + "listed": 30, "named": false, "learned": false }, @@ -411,31 +411,31 @@ "lang": "rust", "target": "indexmap::inner::entry::OccupiedEntry::swap_indices", "variant": "nochange", - "test": null, - "place": null, - "listed": 0, + "test": "occupied_entry_swap_indices", + "place": 1, + "listed": 33, "named": false, - "learned": false + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::inner::entry::OccupiedEntry::swap_indices", "variant": "default", - "test": null, - "place": null, - "listed": 0, + "test": "occupied_entry_swap_indices", + "place": 1, + "listed": 33, "named": false, - "learned": false + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::map::IndexMap::with_capacity_and_hasher", "variant": "static", - "test": null, - "place": null, - "listed": 0, + "test": "with_cap", + "place": 2, + "listed": 47, "named": false, "learned": false }, @@ -444,31 +444,31 @@ "lang": "rust", "target": "indexmap::map::IndexMap::with_capacity_and_hasher", "variant": "nochange", - "test": null, - "place": null, - "listed": 0, + "test": "with_cap", + "place": 7, + "listed": 7, "named": false, - "learned": false + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::map::IndexMap::with_capacity_and_hasher", "variant": "default", - "test": null, - "place": null, - "listed": 0, + "test": "with_cap", + "place": 1, + "listed": 7, "named": false, - "learned": false + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::inner::Core::shift_remove_finish", "variant": "static", - "test": "shift_remove_entry", - "place": 2, - "listed": 68, + "test": "shift_remove", + "place": 3, + "listed": 71, "named": true, "learned": false }, @@ -477,9 +477,9 @@ "lang": "rust", "target": "indexmap::inner::Core::shift_remove_finish", "variant": "nochange", - "test": "shift_remove_entry", - "place": 2, - "listed": 30, + "test": "shift_remove", + "place": 3, + "listed": 36, "named": true, "learned": true }, @@ -488,9 +488,9 @@ "lang": "rust", "target": "indexmap::inner::Core::shift_remove_finish", "variant": "default", - "test": "shift_remove_entry", - "place": 2, - "listed": 30, + "test": "shift_remove", + "place": 1, + "listed": 36, "named": true, "learned": true }, @@ -499,10 +499,10 @@ "lang": "rust", "target": "indexmap::inner::Core::swap_indices", "variant": "static", - "test": null, - "place": null, - "listed": 0, - "named": false, + "test": "indexed_entry_swap_indices", + "place": 1, + "listed": 29, + "named": true, "learned": false }, { @@ -510,32 +510,32 @@ "lang": "rust", "target": "indexmap::inner::Core::swap_indices", "variant": "nochange", - "test": null, - "place": null, - "listed": 0, - "named": false, - "learned": false + "test": "indexed_entry_swap_indices", + "place": 1, + "listed": 32, + "named": true, + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::inner::Core::swap_indices", "variant": "default", - "test": null, - "place": null, - "listed": 0, - "named": false, - "learned": false + "test": "indexed_entry_swap_indices", + "place": 1, + "listed": 32, + "named": true, + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::map::IndexMap::replace_index", "variant": "static", - "test": null, - "place": null, - "listed": 0, - "named": false, + "test": "replace_index", + "place": 2, + "listed": 69, + "named": true, "learned": false }, { @@ -543,22 +543,22 @@ "lang": "rust", "target": "indexmap::map::IndexMap::replace_index", "variant": "nochange", - "test": null, - "place": null, - "listed": 0, - "named": false, - "learned": false + "test": "replace_index", + "place": 2, + "listed": 30, + "named": true, + "learned": true }, { "repo": "indexmap", "lang": "rust", "target": "indexmap::map::IndexMap::replace_index", "variant": "default", - "test": null, - "place": null, - "listed": 0, - "named": false, - "learned": false + "test": "replace_index", + "place": 2, + "listed": 30, + "named": true, + "learned": true }, { "repo": "indexmap", @@ -577,7 +577,7 @@ "target": "indexmap::inner::entry::OccupiedEntry::move_index", "variant": "nochange", "test": "insert_sorted_bad", - "place": 12, + "place": 14, "listed": 46, "named": false, "learned": true diff --git a/eval/results/why-rank-rust.md b/eval/results/why-rank-rust.md index 2df5d6f..cfbba4b 100644 --- a/eval/results/why-rank-rust.md +++ b/eval/results/why-rank-rust.md @@ -1,13 +1,13 @@ | order | repo | bugs | first | in 3 | in 8 | in the list | empty list | named in the failure | from the learned map | |---|---|---:|---:|---:|---:|---:|---:|---:|---:| | static | semver | 10 | 2 (20%) | 3 (30%) | 4 (40%) | 6 | 1 | 0 | 0 | -| static | indexmap | 10 | 1 (10%) | 3 (30%) | 4 (40%) | 6 | 4 | 2 | 0 | -| static | all | 20 | 3 (15%) | 6 (30%) | 8 (40%) | 12 | 5 | 2 | 0 | +| static | indexmap | 10 | 3 (30%) | 7 (70%) | 8 (80%) | 10 | 0 | 4 | 0 | +| static | all | 20 | 5 (25%) | 10 (50%) | 12 (60%) | 16 | 1 | 4 | 0 | | nochange | semver | 10 | 3 (30%) | 4 (40%) | 9 (90%) | 10 | 0 | 0 | 10 | -| nochange | indexmap | 10 | 1 (10%) | 4 (40%) | 4 (40%) | 6 | 4 | 2 | 6 | -| nochange | all | 20 | 4 (20%) | 8 (40%) | 13 (65%) | 16 | 4 | 2 | 16 | +| nochange | indexmap | 10 | 3 (30%) | 7 (70%) | 9 (90%) | 10 | 0 | 4 | 10 | +| nochange | all | 20 | 6 (30%) | 11 (55%) | 18 (90%) | 20 | 0 | 4 | 20 | | default | semver | 10 | 10 (100%) | 10 (100%) | 10 (100%) | 10 | 0 | 0 | 10 | -| default | indexmap | 10 | 4 (40%) | 5 (50%) | 6 (60%) | 6 | 4 | 2 | 6 | -| default | all | 20 | 14 (70%) | 15 (75%) | 16 (80%) | 16 | 4 | 2 | 16 | +| default | indexmap | 10 | 8 (80%) | 9 (90%) | 10 (100%) | 10 | 0 | 4 | 10 | +| default | all | 20 | 18 (90%) | 19 (95%) | 20 (100%) | 20 | 0 | 4 | 20 | -60 answers in 309 s. +60 answers in 61 s. From 511a2558f1a6799b5621ec90ab22f886c0039ff7 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 04:00:26 +0300 Subject: [PATCH 07/18] dead: names built next to reflection, functions run by an attribute; protected is API Deleting everything dead called likely dead still broke two of the ten eval projects: - sqlparse: 5 methods reached by names built at run time (getattr(self, f'_process_{type(tlist).__name__}'.lower())); deleting them failed 16 tests. A piece of a name - a string of 3+ letters glued into a name ("_visit_" + x, f"_process_{x}", "_handle_%s" % x, x + "_handler", obj[`on${x}`]) within 3 lines of reflection (getattr, globals(), getMethod, MethodByName, ...) - now keeps every symbol whose name starts or ends with it. On the ten projects it keeps these 5 and nothing else. - indexmap: all 57 were #[bench] functions. They looked deletable only because the benches were never built; with every feature and target (nightly, #![feature(test)]) deleting them gives 17 compile errors. Functions run by #[bench], #[no_mangle], #[export_name], #[proc_macro...], #[...::main] are not dead. - json-java: a protected method is there for subclasses outside the repo: it is listed as public but unused, not likely dead. likely dead on the ten projects: 64 -> 1 (jsoup, deleted with no new compile error or failing test). eval/consistency.py now deletes in the Java projects too (Maven before and after) and builds Rust with --all-features --all-targets, on nightly where stable cannot build the benches; each project gets a one-line verdict. public but unused: of 265 without the named-elsewhere check, 213 are named elsewhere; of 37 of the remaining 52 read by hand, 29 are API, 4 are reached another way, 4 are dead (COVERAGE.md, level 3). Co-Authored-By: Claude Opus 5.5 --- crates/gitnext-ops/src/insight.rs | 152 +++++++++++++++++++++++++++++- crates/gitnext-ops/tests/dead.rs | 136 ++++++++++++++++++++++++++ eval/consistency.py | 20 +++- eval/results/COVERAGE.md | 39 ++++++-- eval/results/consistency.json | 83 ++++++++-------- eval/results/consistency.md | 18 +--- 6 files changed, 381 insertions(+), 67 deletions(-) diff --git a/crates/gitnext-ops/src/insight.rs b/crates/gitnext-ops/src/insight.rs index ef7696f..14fa523 100644 --- a/crates/gitnext-ops/src/insight.rs +++ b/crates/gitnext-ops/src/insight.rs @@ -269,13 +269,15 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { // (`-> _SourceVarsDict`), a base or metaclass, an import, a callback (`map(helper)`): if the name // is written anywhere outside the symbol itself, something refers to it. let named = named_elsewhere(g, root, &candidates); + let run_by_attribute = entry_attribute(g, root, &candidates); for i in candidates { - if named.contains(&i) { + if named.contains(&i) || run_by_attribute.contains(&i) { continue; } let (r, s) = (&g.syms[i], &g.syms[i].sym); let line = format!("{} {} {}:{}", s.kind.as_str(), g.short_name(i), r.file, s.start_line); - if s.is_public() && !s.name.starts_with('_') { + // `protected`: there for subclasses, outside the repo too — API, not dead + if (s.is_public() || is_protected(s)) && !s.name.starts_with('_') { public_unused.push(line); } else { likely.push(line); @@ -285,7 +287,8 @@ pub fn dead(g: &Graph, root: &Path) -> Dead { } /// Of `candidates`, the symbols whose name is written in the code outside their own lines -/// (comment lines aside). One pass over the files. +/// (comment lines aside), or is built from a piece written next to reflection +/// (`getattr(self, f"_process_{kind}")`: every `_process_*` is reached). One pass over the files. fn named_elsewhere(g: &Graph, root: &Path, candidates: &[usize]) -> HashSet { let mut by_name: HashMap<&str, Vec> = HashMap::new(); for &i in candidates { @@ -295,8 +298,11 @@ fn named_elsewhere(g: &Graph, root: &Path, candidates: &[usize]) -> HashSet = HashSet::new(); for f in g.files.iter().filter(|f| f.parsed && f.lines < 50_000) { let Ok(text) = std::fs::read_to_string(root.join(&f.path)) else { continue }; + let mut found: Vec<(u32, NamePiece)> = Vec::new(); + let mut reflection: Vec = Vec::new(); for (n, line) in text.lines().enumerate() { let t = line.trim_start(); if t.starts_with('#') || t.starts_with("//") || t.starts_with('*') || t.starts_with("/*") { @@ -312,11 +318,151 @@ fn named_elsewhere(g: &Graph, root: &Path, candidates: &[usize]) -> HashSet name.len() > x.len() && name.starts_with(x.as_str()), + NamePiece::Suffix(x) => name.len() > x.len() && name.ends_with(x.as_str()), + }) { + named.insert(i); + } } } named } +/// Calls that reach a member by a name held in a string. +const REFLECTION: &[&str] = &[ + "getattr(", "hasattr(", "globals()", "locals()", "vars(", "__dict__", "methodcaller(", // Python + "getMethod(", "getDeclaredMethod(", "getField(", "getDeclaredField(", "forName(", // Java + "MethodByName(", "FieldByName(", // Go +]; + +/// The start or the end of a name glued together at run time, lowercased (sqlparse lowercases it). +#[derive(Debug, PartialEq, Eq, Hash)] +enum NamePiece { + Prefix(String), + Suffix(String), +} + +/// String pieces of `line` that are glued to something else into a name: `"_process_" + x`, +/// `f'_process_{x}'`, `'_process_%s' % x`, `'_process_{}'.format(x)`, `` `on${x}` ``, `x + "_handler"`, +/// `f'{x}_handler'`. Only identifier characters, at least 3 letters or digits (`f"_{name}"` would make +/// every `_x` alive). The flag: the string stands right after `[` (`obj["on" + x]`), itself a lookup by name. +fn name_pieces(line: &str) -> Vec<(NamePiece, bool)> { + fn piece(s: &str) -> Option { + (s.chars().all(|c| c.is_alphanumeric() || c == '_') && s.chars().filter(|c| c.is_alphanumeric()).count() >= 3) + .then(|| s.to_lowercase()) + } + let b: Vec = line.chars().collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let q = b[i]; + if !matches!(q, '"' | '\'' | '`') { + i += 1; + continue; + } + // f'…' / F'…' / rf'…': an interpolating string + let fstr = q == '`' || (i > 0 && matches!(b[i - 1], 'f' | 'F') && (i < 2 || !b[i - 2].is_alphanumeric() || matches!(b[i - 2], 'r' | 'R'))); + let mut j = i + 1; + while j < b.len() && b[j] != q { + j += if b[j] == '\\' { 2 } else { 1 }; + } + if j >= b.len() { + break; + } + let body: String = b[i + 1..j].iter().collect(); + let before = b[..i - usize::from(fstr && q != '`')].iter().rev().find(|c| !c.is_whitespace()).copied(); + let after = b[j + 1..].iter().find(|c| !c.is_whitespace()).copied(); + let indexed = before == Some('['); + let open = if q == '`' { "${" } else { "{" }; + if fstr && body.contains(open) { + let head = &body[..body.find(open).unwrap()]; + if let Some(p) = piece(head) { + out.push((NamePiece::Prefix(p), indexed)); + } + if let Some(k) = body.rfind('}') { + if let Some(p) = piece(&body[k + 1..]) { + out.push((NamePiece::Suffix(p), indexed)); + } + } + } else if let Some(head) = body.strip_suffix("%s").or_else(|| body.strip_suffix("{}")) { + if let Some(p) = piece(head) { + out.push((NamePiece::Prefix(p), indexed)); + } + } else if let Some(p) = piece(&body) { + if after == Some('+') { + out.push((NamePiece::Prefix(p), indexed)); + } else if before == Some('+') { + out.push((NamePiece::Suffix(p), indexed)); + } + } + i = j + 1; + } + out +} + +/// Rust functions a harness or the linker runs by an attribute: `#[bench]`, `#[no_mangle]`, +/// `#[export_name = ..]`, `#[proc_macro..]`, `#[tokio::main]`. Read from the lines above the item. +fn entry_attribute(g: &Graph, root: &Path, candidates: &[usize]) -> HashSet { + let mut by_file: BTreeMap<&str, Vec> = BTreeMap::new(); + for &i in candidates { + let r = &g.syms[i]; + if r.file.ends_with(".rs") && r.sym.kind.is_callable() { + by_file.entry(r.file.as_str()).or_default().push(i); + } + } + let mut out = HashSet::new(); + for (file, ids) in by_file { + let Ok(text) = std::fs::read_to_string(root.join(file)) else { continue }; + let lines: Vec<&str> = text.lines().collect(); + for i in ids { + let start = g.syms[i].sym.start_line as usize; // 1-based: lines[start - 1] is the item + let mut k = start.saturating_sub(1); + // the item's own first line too: an attribute may share it (`#[bench] fn f(..)`) + let mut attrs = lines.get(k).map(|l| l.trim_start().to_string()).filter(|l| l.starts_with("#[")).unwrap_or_default(); + while k > 0 { + k -= 1; + let t = lines[k].trim_start(); + if t.starts_with("#[") || t.starts_with("//") { + attrs.push_str(t); + } else { + break; + } + } + if ["#[bench]", "no_mangle", "export_name", "#[proc_macro", "::main]", "::main("].iter().any(|a| attrs.contains(a)) { + out.insert(i); + } + } + } + out +} + +/// `protected` in Java / C++ / TypeScript: written before the name in the signature. +fn is_protected(s: &gitnext_parse::Symbol) -> bool { + s.visibility.as_deref() == Some("protected") + || s.signature.find(s.name.as_str()).is_some_and(|k| s.signature[..k].split_whitespace().any(|w| w == "protected")) +} + pub fn render_dead(d: &Dead) -> String { let mut s = format!("unreferenced, not public ({}): likely dead\n", d.likely.len()); for l in &d.likely { diff --git a/crates/gitnext-ops/tests/dead.rs b/crates/gitnext-ops/tests/dead.rs index 63ee122..f94e0fe 100644 --- a/crates/gitnext-ops/tests/dead.rs +++ b/crates/gitnext-ops/tests/dead.rs @@ -70,3 +70,139 @@ fn a_name_used_as_a_value_or_a_type_is_not_dead() { } let _ = std::fs::remove_dir_all(&dir); } + +fn dead_in(tag: &str, files: &[(&str, &str)]) -> gitnext_ops::insight::Dead { + let dir = std::env::temp_dir().join(format!("gitnext-dead-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for (p, c) in files { + let f = dir.join(p); + std::fs::create_dir_all(f.parent().unwrap()).unwrap(); + std::fs::write(f, c).unwrap(); + } + let mut idx = Index::open(&dir).unwrap(); + idx.refresh().unwrap(); + let d = dead(&idx.graph().unwrap(), &dir); + let _ = std::fs::remove_dir_all(&dir); + d +} + +const FILTERS: &str = "\ +class Reindent: + def process(self, tlist): + func_name = f'_process_{type(tlist).__name__}' + func = getattr(self, func_name.lower(), self.process) + return func(tlist) + + def _process_where(self, tlist): + return tlist + + def visit(self, kind): + return getattr(self, '_visit_' + kind)() + + def _visit_call(self): + return 1 + + def handle(self, kind): + return getattr(self, '_handle_%s' % kind)() + + def _handle_join(self): + return 2 + + def click(self, kind): + return getattr(self, kind + '_handler')() + + def _click_handler(self): + return 3 + + def _orphan_thing(self): + return 4 + + def _really_dead(self): + return 5 + + +def __getattr__(name): + return globals()[f\"_{name}\"] + + +def describe(x): + return '_orphan_' + x +"; + +#[test] +fn a_name_built_next_to_reflection_is_not_dead() { + let d = dead_in("py", &[("pkg/__init__.py", "from .filters import Reindent, describe\n"), ("pkg/filters.py", FILTERS)]); + let all = d.likely.join("\n") + "\n" + &d.public_unused.join("\n"); + // f-string + .lower(), concatenation, `%`, a suffix: each next to getattr + for used in ["_process_where", "_visit_call", "_handle_join", "_click_handler"] { + assert!(!all.contains(used), "`{used}` is reached through getattr, not dead:\n{all}"); + } + // `f"_{name}"` is too short to be a piece of a name; `'_orphan_' + x` has no reflection near it + for unused in ["_really_dead", "_orphan_thing"] { + assert!(d.likely.iter().any(|l| l.contains(unused)), "`{unused}` is dead:\n{all}"); + } +} + +const HANDLERS: &str = "\ +package org.example; + +public class Handlers { + public Object dispatch(String kind) throws Exception { + return getClass().getDeclaredMethod(\"handle\" + kind).invoke(this); + } + + private Object handleSave() { + return null; + } + + private Object neverCalled() { + return null; + } + + protected char previous() { + return 'a'; + } +} +"; + +#[test] +fn java_reflection_and_protected() { + let d = dead_in("java", &[("src/main/java/org/example/Handlers.java", HANDLERS)]); + let all = d.likely.join("\n") + "\n" + &d.public_unused.join("\n"); + assert!(!all.contains("handleSave"), "reached through getDeclaredMethod(\"handle\" + kind):\n{all}"); + assert!(d.likely.iter().any(|l| l.contains("neverCalled")), "private and unreferenced:\n{all}"); + // protected: API for subclasses, not something to delete + assert!(!d.likely.iter().any(|l| l.contains("previous")), "protected is not likely dead:\n{all}"); + assert!(d.public_unused.iter().any(|l| l.contains("previous")), "protected is public but unused:\n{all}"); +} + +const BENCH: &str = "\ +#![feature(test)] +extern crate test; + +use test::Bencher; + +fn helper() -> u32 { + 1 +} + +#[bench] +fn insert_10_000(b: &mut Bencher) { + b.iter(helper); +} + +#[no_mangle] +extern \"C\" fn exported_hook() {} + +fn unused_in_bench() {} +"; + +#[test] +fn rust_functions_run_by_an_attribute_are_not_dead() { + let d = dead_in("rs", &[("Cargo.toml", "[package]\nname = \"b\"\nversion = \"0.1.0\"\n"), ("src/lib.rs", "pub fn f() {}\n"), ("benches/bench.rs", BENCH)]); + let all = d.likely.join("\n") + "\n" + &d.public_unused.join("\n"); + for used in ["insert_10_000", "exported_hook", "helper"] { + assert!(!all.contains(used), "`{used}` is run by the bench harness or the linker:\n{all}"); + } + assert!(d.likely.iter().any(|l| l.contains("unused_in_bench")), "a plain function nothing calls:\n{all}"); +} diff --git a/eval/consistency.py b/eval/consistency.py index 2c3ecbc..5c86d90 100644 --- a/eval/consistency.py +++ b/eval/consistency.py @@ -259,8 +259,23 @@ def check_dead(root: Path, repo: dict, syms: list[dict], run: bool) -> dict: if hits: res["named_elsewhere"].append(f"{kind} {name} {file}:{line} — named at {', '.join(hits[:3])}{' …' if len(hits) > 3 else ''}") L = ev.LANG.get(repo["lang"]) - if not run or not L or not ent["likely"] or repo["lang"] not in ("python", "go", "rust", "typescript"): + res["verdict"] = f"{len(ent['likely'])} named likely dead" + if not ent["likely"]: + res["verdict"] += ": nothing to delete" return res + if not run or not L: + res["verdict"] += ": deleting not tried" + ("" if run else " (--no-dead-run)") + return res + L = dict(L) + if repo["lang"] == "rust": + # every feature and every target: code behind a feature, examples, benches and tests all build. + # Benches with `#![feature(test)]` build on nightly only: where stable fails before anything is deleted, nightly + tc = "" + build = "cargo {}test --all-features --all-targets --no-run --no-fail-fast --message-format human 2>&1" + if ev.sh(build.format(""), root, timeout=1800)[0] and "nightly" in ev.sh("rustup toolchain list", root)[1]: + tc = "+nightly " + L["compile"], L["tests"] = build.format(tc), f"cargo {tc}test --all-features --no-fail-fast 2>&1" + res["toolchain"] = tc.strip("+ ") or "stable" compiled = lambda: set(map(str, L["errors"](ev.sh(L["compile"], root, timeout=1800)[1], root))) if L.get("compile") and L.get("errors") else set() # noqa: E731 before_c = compiled() before_t = set(L["failed"](ev.sh(L["tests"], root, timeout=1800)[1], root)) @@ -274,6 +289,9 @@ def check_dead(root: Path, repo: dict, syms: list[dict], run: bool) -> dict: # pytest: a test module that no longer imports is an ERROR line without a test name after_t |= {f"{m} (does not import)" for m in re.findall(r"^ERROR (\S+\.py)(?: - .*)?$", out, re.M)} res["broke"] = {"compile_errors": sorted(after_c - before_c)[:8], "tests": sorted(after_t - before_t)[:8], "tests_ran": bool(L["ran"](out, root))} + new_c, new_t = len(after_c - before_c), len(after_t - before_t) + res["verdict"] += (f", {res['deleted']} deleted: {new_c} new compile errors, {new_t} new failing tests" + + ("" if res["broke"]["tests_ran"] else " (the tests did not run)") + (f" ({res['toolchain']})" if "toolchain" in res else "")) ev.restore(root) ev.gx(["index"], root) return res diff --git a/eval/results/COVERAGE.md b/eval/results/COVERAGE.md index 23966c6..b6b9f1a 100644 --- a/eval/results/COVERAGE.md +++ b/eval/results/COVERAGE.md @@ -45,15 +45,42 @@ No compiler is asked here; what can be checked without one is, on the 10 project | `search` | a symbol is found by its own name | 296, all found | | `edit replace`, `edit delete` | a symbol replaced with its own code leaves the file as it was; a delete removes the symbol with its doc comment and nothing else | 200 edits, all right (before: 11 wrong in the two Java projects — a body line indented less than the first, tabs under spaces, a Javadoc without `*`) | | `hierarchy` | the bases in a class's own signature (Python, Java, TypeScript) are in `extends` / `implements`, and the class is among their subtypes | 195 bases, both ways, all there | -| `dead` | the name is written nowhere else in the code; with everything it calls likely dead deleted, the project compiles and no test starts failing (Python, Go, Rust, TypeScript) | see below | +| `dead` | the name is written nowhere else in the code, nor built next to reflection; with everything it calls likely dead deleted, the project compiles and no test starts failing (all five languages; Rust with every feature and every target) | see below | `dead` before this check named 225 symbols "likely dead" on the 10 projects; 160 of them were named elsewhere in the code — used as a value, a type, a base, an import, a callback, or a Go `init` — and deleting them broke the -build of at least 6 of the 8 projects where that is tried. Now the name must be written nowhere outside the symbol -itself: 64 are left. indexmap 57 — deleted, it compiles and its tests pass (not tried with every feature and the -benches built). sqlparse 5 — deleted, eight tests fail: the methods are reached by names built at run time -(`getattr(self, "_process_" + …)`). The two Java projects, one each — not tried (the delete run would need Maven -twice per project). The `public but unused` list is not checked. +build of at least 6 of the 8 projects where that is tried. Then the name had to be written nowhere outside the +symbol itself: 64 were left, and deleting them still broke two projects: + +- sqlparse 5 — 16 tests fail: the methods are reached by names built at run time + (`getattr(self, f'_process_{type(tlist).__name__}'.lower())`). Now a piece of a name — a string of at least + 3 letters glued into a name (`"_visit_" + x`, `f"_process_{x}"`, `"_handle_%s" % x`, `x + "_handler"`, + `` obj[`on${x}`] ``) within 3 lines of reflection (`getattr`, `globals()`, `getMethod`, `MethodByName`, …) — + keeps every symbol whose name starts (or ends) with it alive. On the 10 projects it keeps these 5 and nothing else. +- indexmap 57 — all of them `#[bench]` functions; with every feature and every target built (nightly: the benches use + `#![feature(test)]`), deleting them gives 17 compile errors. Before, the benches were not built, so it looked fine. + Functions run by an attribute — `#[bench]`, `#[no_mangle]`, `#[export_name]`, `#[proc_macro…]`, `#[…::main]` — + are not dead now. +- json-java 1 — `protected`: there for subclasses, outside the repo too. It is in `public but unused` now. + +What deleting everything left in "likely dead" does (`eval/consistency.py`, Java with Maven before and after, +Rust with `--all-features --all-targets`): + +| project | likely dead: before → now | deleted, new compile errors / new failing tests | +|---|---:|---| +| cobra, chi, click, zustand, yaml, semver | 0 → 0 | nothing to delete | +| sqlparse | 5 → 0 | before: 0 / 16 | +| indexmap | 57 → 0 | before: 17 / 0 (nightly, every feature and target) | +| json-java | 1 → 0 | before: 0 / 0 (moved to public but unused) | +| jsoup | 1 → 1 | 0 / 0 (`HtmlTreeBuilder.getBaseUri`, package-private, really unused) | + +`public but unused`: 51 → 52 (the `protected` method above; the list is otherwise as it was). Without +the "named elsewhere" check it would be 265 on the 10 projects; 213 of them are named elsewhere, 52 are left +(cobra 23, indexmap 12, chi 6, click 3, sqlparse 2, yaml 2, jsoup 2, json-java 1, semver 1, zustand 0). +Of 37 read by hand (up to 10 a project, seed 5): 29 are API, 4 are reached another way (an `io` method the +standard library calls, a `sort.Interface` method, a no_std crate that only has to compile), 4 are dead code +(an unused function in chi's `_examples`, sqlparse's `Having`, yaml's `primitiveKey` that the package does not +export, jsoup's `internal` `StringUtil.startsWithNewline`). So the list is mostly API, as its name says. ## 4. Answers on every project, correctness not checked (`eval/smoke.py`, `smoke.md`) diff --git a/eval/results/consistency.json b/eval/results/consistency.json index 3c566d0..7d6a341 100644 --- a/eval/results/consistency.json +++ b/eval/results/consistency.json @@ -37,7 +37,8 @@ "public_unused": 23, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, "seconds": 7 }, @@ -79,9 +80,10 @@ "public_unused": 6, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 5 + "seconds": 6 }, "click": { "symmetry": { @@ -121,9 +123,10 @@ "public_unused": 3, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 13 + "seconds": 14 }, "sqlparse": { "symmetry": { @@ -159,26 +162,14 @@ "missing_subtype": [] }, "dead": { - "likely": 5, + "likely": 0, "public_unused": 2, "named_elsewhere": [], - "deleted": 5, - "broke": { - "compile_errors": [], - "tests": [ - "test_duplicate_linebreaks", - "test_except_formatting", - "test_format_invalid_where_clause", - "test_having_produces_newline", - "test_identifier_list", - "test_identifier_list_comment_first", - "test_identifier_list_with_wrap_after", - "test_issue315_utf8_by_default" - ], - "tests_ran": true - } + "deleted": 0, + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 9 + "seconds": 6 }, "zustand": { "symmetry": { @@ -218,9 +209,10 @@ "public_unused": 0, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 5 + "seconds": 6 }, "yaml": { "symmetry": { @@ -260,9 +252,10 @@ "public_unused": 2, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 9 + "seconds": 11 }, "jsoup": { "symmetry": { @@ -301,10 +294,15 @@ "likely": 1, "public_unused": 2, "named_elsewhere": [], - "deleted": 0, - "broke": null + "deleted": 1, + "broke": { + "compile_errors": [], + "tests": [], + "tests_ran": true + }, + "verdict": "1 named likely dead, 1 deleted: 0 new compile errors, 0 new failing tests" }, - "seconds": 244 + "seconds": 392 }, "json-java": { "symmetry": { @@ -340,13 +338,14 @@ "missing_subtype": [] }, "dead": { - "likely": 1, - "public_unused": 0, + "likely": 0, + "public_unused": 1, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 18 + "seconds": 21 }, "semver": { "symmetry": { @@ -386,9 +385,10 @@ "public_unused": 1, "named_elsewhere": [], "deleted": 0, - "broke": null + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 3 + "seconds": 4 }, "indexmap": { "symmetry": { @@ -424,16 +424,13 @@ "missing_subtype": [] }, "dead": { - "likely": 57, + "likely": 0, "public_unused": 12, "named_elsewhere": [], - "deleted": 57, - "broke": { - "compile_errors": [], - "tests": [], - "tests_ran": true - } + "deleted": 0, + "broke": null, + "verdict": "0 named likely dead: nothing to delete" }, - "seconds": 14 + "seconds": 9 } } \ No newline at end of file diff --git a/eval/results/consistency.md b/eval/results/consistency.md index 512caab..b22859c 100644 --- a/eval/results/consistency.md +++ b/eval/results/consistency.md @@ -8,23 +8,13 @@ A cell is `disagreements / checked`. | cobra | 0 / 59 | 0 / 59 | 0 / 79 | 0 / 79 | 0 / 30 | 0 / 288 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | | chi | 0 / 19 | 0 / 19 | 0 / 33 | 0 / 33 | 0 / 30 | 0 / 351 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | | click | 0 / 26 | 0 / 26 | 0 / 32 | 0 / 32 | 0 / 30 | 0 / 761 | 0 / 30 | 0 / 20 | 0 / 94 | 0 / 0 | — | -| sqlparse | 0 / 55 | 0 / 55 | 0 / 49 | 0 / 49 | 0 / 30 | 0 / 287 | 0 / 30 | 0 / 20 | 0 / 54 | 0 / 5 | 0 / 8 (5 deleted) | +| sqlparse | 0 / 55 | 0 / 55 | 0 / 49 | 0 / 49 | 0 / 30 | 0 / 287 | 0 / 30 | 0 / 20 | 0 / 54 | 0 / 0 | — | | zustand | 0 / 21 | 0 / 21 | 0 / 24 | 0 / 24 | 0 / 30 | 0 / 175 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | | yaml | 0 / 47 | 0 / 47 | 0 / 36 | 0 / 36 | 0 / 30 | 0 / 440 | 0 / 30 | 0 / 20 | 0 / 32 | 0 / 0 | — | -| jsoup | 0 / 67 | 0 / 67 | 0 / 19 | 0 / 19 | 0 / 30 | 0 / 2661 | 0 / 27 | 0 / 20 | 0 / 200 | 0 / 1 | — | -| json-java | 0 / 51 | 0 / 51 | 0 / 42 | 0 / 42 | 0 / 30 | 0 / 540 | 0 / 30 | 0 / 20 | 0 / 10 | 0 / 1 | — | +| jsoup | 0 / 67 | 0 / 67 | 0 / 19 | 0 / 19 | 0 / 30 | 0 / 2661 | 0 / 27 | 0 / 20 | 0 / 200 | 0 / 1 | 0 / 0 (1 deleted) | +| json-java | 0 / 51 | 0 / 51 | 0 / 42 | 0 / 42 | 0 / 30 | 0 / 540 | 0 / 30 | 0 / 20 | 0 / 10 | 0 / 0 | — | | semver | 0 / 45 | 0 / 45 | 0 / 25 | 0 / 25 | 0 / 30 | 0 / 113 | 0 / 30 | 0 / 20 | 0 / 0 | 0 / 0 | — | -| indexmap | 0 / 9 | 0 / 9 | 0 / 30 | 0 / 30 | 0 / 30 | 0 / 1026 | 0 / 29 | 0 / 20 | 0 / 0 | 0 / 57 | 0 / 0 (57 deleted) | +| indexmap | 0 / 9 | 0 / 9 | 0 / 30 | 0 / 30 | 0 / 30 | 0 / 1026 | 0 / 29 | 0 / 20 | 0 / 0 | 0 / 0 | — | ## The disagreements -**sqlparse** -- dead deleted: test_duplicate_linebreaks -- dead deleted: test_except_formatting -- dead deleted: test_format_invalid_where_clause -- dead deleted: test_having_produces_newline -- dead deleted: test_identifier_list -- dead deleted: test_identifier_list_comment_first -- dead deleted: test_identifier_list_with_wrap_after -- dead deleted: test_issue315_utf8_by_default - From 028ccbc3d965ef28cf1552b6dd77d1737088f71b Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 04:14:55 +0300 Subject: [PATCH 08/18] changelog: Rust tests inside macros, dead by deletion; 127 unit tests Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 7 +++++++ README.md | 2 +- README.ru.md | 2 +- eval/results/COVERAGE.md | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c11f9..a14ab6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ whose lines do not start with `*` was cut in half, and the delete refused). - **`callees`** lists a method called on an object of unknown type as "maybe", as `callers` does; `look` counts such calls instead of naming them. +- **Rust: functions declared inside a macro call are symbols** (`quickcheck! { fn prop(..) -> bool { .. } }`, + `proptest!`, a local `macro_rules!` that writes `#[test]`). In indexmap 43 of 192 tests were outside the graph and + the coverage map, now 4; `why` put the function at fault first in 14 of 20 seeded Rust bugs, now in 16–18, and + no longer answers with an empty list (4 of 10 on indexmap). +- `dead` again: a name built next to reflection (`getattr(self, "_process_" + kind)`) is used; a Rust function run + by an attribute (`#[bench]`, `#[no_mangle]`, …) is not dead; `protected` is API. On the ten projects what it + calls likely dead went 225 → 64 → 1, and that one deletes cleanly. - `eval/consistency.py`: do the commands agree with each other and with the source (`eval/results/consistency.md`). - eval: a tool call cut by the token limit (a long `replace`) is asked for again instead of ending the run. - Release: the Intel Mac binary is built again (its runner was retired); `CITATION.cff`. diff --git a/README.md b/README.md index cfaf179..d6b0715 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Most of an agent session is the system prompt re-sent on every step, so the win ### What is checked, and what is not good yet -- 122 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 127 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` finds 19% of the failing test files (it was 3%); the coverage map exists for Python, Go and Rust only. - A rename breaks the build or the tests in 9% of cases — mostly dynamic access and names shared by unrelated types. - A 4B model rarely fixes a seeded bug yet: 3 of 12 on click. diff --git a/README.ru.md b/README.ru.md index 72e2a52..7452916 100644 --- a/README.ru.md +++ b/README.ru.md @@ -58,7 +58,7 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он ### Что проверено и что пока плохо -- 122 модульных теста; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 127 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` находит 19% падающих файлов тестов (было 3%); карта покрытия есть только для Python, Go и Rust. - Переименование ломает сборку или тесты в 9% случаев — в основном динамический доступ и одинаковые имена у несвязанных типов. - Модель на 4B пока редко чинит подсаженную ошибку: 3 из 12 на click. diff --git a/eval/results/COVERAGE.md b/eval/results/COVERAGE.md index b6b9f1a..af18517 100644 --- a/eval/results/COVERAGE.md +++ b/eval/results/COVERAGE.md @@ -17,7 +17,7 @@ that the answer is right. | `why`, `check ` | place of the function at fault for a seeded bug (`why_rank.py`, `why-rank.md`) | with the learned map the culprit is first in 81% of 57 seeded bugs (Python 90%, Go 82%, Rust 70%); from the call graph alone in 18% | | the tools as a small model uses them | `agent.py`, Qwen 3.5 4B: find / callers / rename / fix | see the agent runs in `HISTORY.md` | -## 2. Unit tests (`cargo test --workspace`, 122 tests) +## 2. Unit tests (`cargo test --workspace`, 127 tests) | function | tests | |---|---| From ffa233b4e568f7c75ad5e527ddbc23aa89bd329d Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 03:23:53 +0300 Subject: [PATCH 09/18] tests-for: a learned map of a JS/TS project names test files tests_for takes the module of a JavaScript / TypeScript test file as the test (it('...') is not a named function), but tests_for_at looked map rows up among functions only, so every row of such a map was dropped and the answer silently came from the call graph. The lookup now takes the same test units as tests_for. Co-Authored-By: Claude Opus 5.5 --- crates/gitnext-ops/src/locate.rs | 3 ++- crates/gitnext-ops/tests/tests_for.rs | 34 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/crates/gitnext-ops/src/locate.rs b/crates/gitnext-ops/src/locate.rs index e692a34..4290b3f 100644 --- a/crates/gitnext-ops/src/locate.rs +++ b/crates/gitnext-ops/src/locate.rs @@ -309,7 +309,8 @@ pub fn tests_for_at(g: &Graph, root: &Path, i: usize, depth: usize) -> (Vec = std::collections::HashMap::new(); for (k, s) in g.syms.iter().enumerate() { - if s.sym.test && s.sym.kind.is_callable() { + // a JS/TS test file is one test: the map names it by its module, as `tests_for` does + if s.sym.test && is_test_unit(s) { by_place.entry((s.file.as_str(), s.sym.name.as_str())).or_insert(k); } } diff --git a/crates/gitnext-ops/tests/tests_for.rs b/crates/gitnext-ops/tests/tests_for.rs index 9905165..dfab95d 100644 --- a/crates/gitnext-ops/tests/tests_for.rs +++ b/crates/gitnext-ops/tests/tests_for.rs @@ -189,3 +189,37 @@ fn a_typescript_test_file_is_a_test() { assert!(locate::tests_for(&g, g.find("unused")[0], 4).is_empty()); let _ = std::fs::remove_dir_all(&dir); } + +/// A learned map of a JavaScript / TypeScript project names test files (their module): `tests-for` +/// takes them from the map, also a file the call graph does not link to the function. +#[test] +fn a_learned_typescript_test_file_comes_from_the_map() { + let dir = project( + "learned-ts", + &[ + ("src/price.ts", "export function totalPrice(items: number[]): number {\n return items.reduce((a, b) => a + b, 0)\n}\n\nconst table = { total: totalPrice }\n\nexport function byName(name: 'total', items: number[]): number {\n return table[name](items)\n}\n"), + ("tests/cart.ts", "import { expect, it } from 'vitest'\nimport { totalPrice } from '../src/price'\n\nit('adds up', () => {\n expect(totalPrice([1, 2])).toBe(3)\n})\n"), + ("tests/by-name.test.ts", "import { expect, it } from 'vitest'\nimport { byName } from '../src/price'\n\nit('looks up', () => {\n expect(byName('total', [1])).toBe(1)\n})\n"), + ], + ); + let mut idx = Index::open(&dir).unwrap(); + idx.refresh().unwrap(); + let g = idx.graph().unwrap(); + let target = g.find("totalPrice")[0]; + let (t, source) = locate::tests_for_at(&g, &dir, target, 4); + assert_eq!(source, locate::Source::Graph); + assert_eq!(t.iter().map(|x| g.syms[x.symbol].file.as_str()).collect::>(), ["tests/cart.ts"]); + // the map books both files to it by the name of their module symbol + let module = |file: &str| g.syms.iter().find(|s| s.file == file && s.sym.test).map(|s| s.sym.name.clone()).unwrap(); + let hash = g.syms[target].sym.body_hash.clone(); + let rows: Vec<_> = ["tests/cart.ts", "tests/by-name.test.ts"].iter().map(|f| (g.unique_name(target), hash.clone(), f.to_string(), module(f))).collect(); + gitnext_ops::gitnext_index::TestMap::write(&dir, "vitest-v8", &rows).unwrap(); + let (t, source) = locate::tests_for_at(&g, &dir, target, 4); + assert_eq!(source, locate::Source::Learned { stale: false }); + let mut files: Vec<&str> = t.iter().map(|x| g.syms[x.symbol].file.as_str()).collect(); + files.sort(); + assert_eq!(files, ["tests/by-name.test.ts", "tests/cart.ts"]); + let text = locate::render_tests_for(&g, &dir, target, 4, false); + assert!(text.starts_with("tests that ran totalPrice (2)") && text.contains("tests/by-name.test.ts tests/cart.ts"), "{text}"); + let _ = std::fs::remove_dir_all(&dir); +} From f476a8d55ca8aa107684afe52f25b9abb726cdde Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 03:24:36 +0300 Subject: [PATCH 10/18] learn-tests: a language can be known by several file suffixes A language of learn-tests is now in the repository when a file ends with any of its suffixes, and its tests still run once. Python, Go and Rust keep one suffix each; JavaScript / TypeScript needs .ts, .tsx, .js, .mjs and the like. Co-Authored-By: Claude Opus 5.5 --- crates/gitnext-ops/src/learn.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/gitnext-ops/src/learn.rs b/crates/gitnext-ops/src/learn.rs index ce8eb3e..9e42eb1 100644 --- a/crates/gitnext-ops/src/learn.rs +++ b/crates/gitnext-ops/src/learn.rs @@ -33,9 +33,10 @@ pub(crate) struct Run { pub notes: Vec, } -/// The language is in the repository when a file ends so; its tests are run by the function. +/// The language is in the repository when a file ends with one of its suffixes; its tests are +/// run once by the function. type Learner = fn(&Path, &Graph) -> Result; -const LANGUAGES: &[(&str, &str, Learner)] = &[("Python", ".py", python::run), ("Go", ".go", go::run), ("Rust", ".rs", rust::run)]; +const LANGUAGES: &[(&str, &[&str], Learner)] = &[("Python", &[".py"], python::run), ("Go", &[".go"], go::run), ("Rust", &[".rs"], rust::run)]; pub struct Learned { pub command: String, @@ -84,7 +85,7 @@ pub(crate) fn owners_at(g: &Graph, header: bool) -> HashMap<&str, Vec Result { - let here: Vec<&(&str, &str, Learner)> = LANGUAGES.iter().filter(|(_, ext, _)| g.files.iter().any(|f| f.path.ends_with(ext))).collect(); + let here: Vec<&(&str, &[&str], Learner)> = LANGUAGES.iter().filter(|(_, exts, _)| g.files.iter().any(|f| exts.iter().any(|e| f.path.ends_with(e)))).collect(); if here.is_empty() { bail!("learn-tests has no way to run the tests of this repository with coverage (it knows: {})", LANGUAGES.iter().map(|l| l.0).collect::>().join(", ")); } From 54913deb120560fbff24b11e2e3a977496c04830 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 03:31:00 +0300 Subject: [PATCH 11/18] learn-tests: JavaScript and TypeScript on vitest Every test file vitest lists (vitest list --filesOnly --json) runs alone with v8 coverage; a statement that ran is booked to the innermost function whose body holds its line, and the test is the file's module symbol, as tests-for and why know it. The declaration line of a function is not counted: it runs when the module is imported. On yaml and zustand the map has 285 / 48 functions and 2943 / 164 pairs (the prototype in eval: 284 / 47 and 2919 / 163), in 36 s and 22 s. Coverage counts the top directories with code of the index: the project's own coverage.include may name nothing (zustand's 'src/**/' matches directories only). A file passed when vitest's JSON result counts no failed test; the counters are read from the text, since the result quotes test strings and a lone surrogate in them is not JSON that serde_json accepts (yaml's stringify tests). Nothing is installed: without package.json, without vitest in node_modules or without @vitest/coverage-v8 the message says what to run (the coverage package of vitest's own version); a project on jest is told that only vitest is known. Files that fail, take longer than 10 minutes or write no report (a test file that does not load says why) are notes. Tests: the parsing of a saved coverage report, vitest list, the JSON result and package.json in learn/js.rs; the messages in tests/learn_js.rs without node; a run on a small project that installs vitest from npm only with GITNEXT_TEST_NPM=1. Co-Authored-By: Claude Opus 5.5 --- crates/gitnext-ops/src/learn.rs | 6 +- crates/gitnext-ops/src/learn/js.rs | 354 ++++++++++++++++++ .../tests/data/learn_js/coverage-final.json | 2 + crates/gitnext-ops/tests/learn_js.rs | 94 +++++ 4 files changed, 455 insertions(+), 1 deletion(-) create mode 100644 crates/gitnext-ops/src/learn/js.rs create mode 100644 crates/gitnext-ops/tests/data/learn_js/coverage-final.json create mode 100644 crates/gitnext-ops/tests/learn_js.rs diff --git a/crates/gitnext-ops/src/learn.rs b/crates/gitnext-ops/src/learn.rs index 9e42eb1..d2f09db 100644 --- a/crates/gitnext-ops/src/learn.rs +++ b/crates/gitnext-ops/src/learn.rs @@ -12,6 +12,7 @@ use anyhow::{bail, Result}; use gitnext_index::{Graph, TestMap}; mod go; +mod js; mod python; mod rust; @@ -36,7 +37,10 @@ pub(crate) struct Run { /// The language is in the repository when a file ends with one of its suffixes; its tests are /// run once by the function. type Learner = fn(&Path, &Graph) -> Result; -const LANGUAGES: &[(&str, &[&str], Learner)] = &[("Python", &[".py"], python::run), ("Go", &[".go"], go::run), ("Rust", &[".rs"], rust::run)]; +const LANGUAGES: &[(&str, &[&str], Learner)] = &[ + ("Python", &[".py"], python::run), ("Go", &[".go"], go::run), ("Rust", &[".rs"], rust::run), + ("JavaScript/TypeScript", &[".ts", ".tsx", ".mts", ".js", ".jsx", ".mjs", ".cjs"], js::run), +]; pub struct Learned { pub command: String, diff --git a/crates/gitnext-ops/src/learn/js.rs b/crates/gitnext-ops/src/learn/js.rs new file mode 100644 index 0000000..2bee3bc --- /dev/null +++ b/crates/gitnext-ops/src/learn/js.rs @@ -0,0 +1,354 @@ +//! JavaScript / TypeScript: every test file of vitest run alone with v8 coverage +//! (`vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json`). +//! A test is the file: `it('…', () => …)` is not a named function, and the index knows a test +//! file by its `module` symbol, so the pairs name that symbol. Measured on yaml and zustand: the +//! map finds every test file that fails when a function breaks (the call graph: 16% and 68%), for +//! 2.6–6.8× a plain test run. +//! +//! A statement that ran belongs to the innermost function whose body holds its line, from the +//! line after the declaration: the declaration runs when the module is imported, and counting it +//! would book every function of every imported module to the test. jest is not learned. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use gitnext_index::Graph; +use gitnext_parse::Kind; + +use super::{owners, Run}; + +/// One test file with coverage that takes longer than this is given up. +const FILE_LIMIT: Duration = Duration::from_secs(600); +/// How many file names a note lists before it says "and N more". +const NAMES: usize = 5; + +pub(super) fn run(root: &Path, g: &Graph) -> Result { + run_with("npx", root, g) +} + +/// The temporary reports; removed however `run` ends. +struct Scratch(PathBuf); + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn last_line(text: &str) -> &str { + text.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("").trim() +} + +/// The test runner `package.json` names: a dependency first (`vitest` wins over `jest`), then the +/// `test` script. `@testing-library/jest-dom` and the like are not jest. +fn runner(package_json: &str) -> Option<&'static str> { + let v: serde_json::Value = serde_json::from_str(package_json).ok()?; + let has = |name: &str| ["devDependencies", "dependencies"].iter().any(|k| v.get(k).and_then(|d| d.get(name)).is_some()); + if has("vitest") { + return Some("vitest"); + } + if has("jest") { + return Some("jest"); + } + let script = v.get("scripts").and_then(|s| s.get("test")).and_then(|s| s.as_str()).unwrap_or(""); + ["vitest", "jest"].into_iter().find(|r| script.split(|c: char| !c.is_alphanumeric()).any(|w| w == *r)) +} + +/// Learning can start: vitest in `package.json` and installed with its v8 coverage, and an `npx` +/// that runs. Nothing is installed here; the message says what to run. Gives vitest's version. +fn ready(npx: &str, root: &Path) -> Result { + let Ok(package) = std::fs::read_to_string(root.join("package.json")) else { + bail!("no package.json at the root of the repository (packages in subdirectories are not learned yet)"); + }; + match runner(&package) { + Some("vitest") => {} + Some(other) => bail!("the tests run with {other}; learn-tests knows vitest only"), + None => bail!("package.json names no test runner learn-tests knows (it knows vitest)"), + } + let installed = std::fs::read_to_string(root.join("node_modules/vitest/package.json")).ok().and_then(|t| serde_json::from_str::(&t).ok()); + let Some(version) = installed.and_then(|v| v.get("version").and_then(|x| x.as_str()).map(String::from)) else { + bail!("vitest is not installed in node_modules: run `npm install` first"); + }; + if !root.join("node_modules/@vitest/coverage-v8").is_dir() { + bail!("vitest's v8 coverage is not installed: npm i -D @vitest/coverage-v8@{version}"); + } + match Command::new(npx).arg("--version").stdin(Stdio::null()).output() { + Ok(o) if o.status.success() => Ok(version), + Ok(o) => bail!("`{npx} --version` failed: {}", last_line(&String::from_utf8_lossy(&o.stderr))), + Err(_) => bail!("`{npx}` is not in PATH: install Node.js (https://nodejs.org) to learn which test runs which function"), + } +} + +/// `path` relative to the repository, when it is inside it. +fn relative(path: &str, roots: &[String]) -> Option { + roots.iter().find_map(|r| path.strip_prefix(&format!("{r}/"))).map(String::from) +} + +/// `vitest list --filesOnly --json` → the test files, relative to the repository. +fn listed_files(json: &str, roots: &[String]) -> Vec { + let v: serde_json::Value = serde_json::from_str(json.trim()).unwrap_or_default(); + let mut out: Vec = v.as_array().into_iter().flatten().filter_map(|e| e.get("file")?.as_str()).filter_map(|f| relative(f, roots)).collect(); + out.sort(); + out.dedup(); + out +} + +/// `coverage-final.json` (istanbul format) → file relative to the repository → the first lines +/// of the statements that ran. +fn covered_lines(json: &str, roots: &[String]) -> BTreeMap> { + let v: serde_json::Value = serde_json::from_str(json).unwrap_or_default(); + let mut out = BTreeMap::new(); + for (path, file) in v.as_object().into_iter().flatten() { + let Some(rel) = relative(path, roots) else { continue }; + let (Some(counts), Some(map)) = (file.get("s").and_then(|s| s.as_object()), file.get("statementMap")) else { continue }; + let lines: BTreeSet = counts + .iter() + .filter(|(_, n)| n.as_u64().is_some_and(|n| n > 0)) + .filter_map(|(k, _)| map.get(k)?.get("start")?.get("line")?.as_u64()) + .map(|l| l as u32) + .collect(); + if !lines.is_empty() { + out.insert(rel, lines); + } + } + out +} + +/// vitest's `--reporter=json` result: did every test of the file pass. The counters are read from +/// the text: the result quotes the tests' strings, and a lone surrogate (`"\\ud800"`, yaml's tests +/// have them) is not JSON serde_json accepts. They stand at the top, before `testResults`. +fn passed(json: &str) -> bool { + let count = |key: &str| { + let at = json.find(&format!("\"{key}\":"))? + key.len() + 3; + json[at..].trim_start().split(|c: char| !c.is_ascii_digit()).next()?.parse::().ok() + }; + count("numTotalTests").is_some() && count("numFailedTests") == Some(0) && count("numFailedTestSuites") == Some(0) +} + +/// Why a test file of vitest's `--reporter=json` result failed as a whole: the first line of its message. +fn load_error(json: &str) -> Option { + let v: serde_json::Value = serde_json::from_str(json).ok()?; + let m = v.get("testResults")?.as_array()?.iter().find_map(|r| r.get("message")?.as_str().filter(|m| !m.trim().is_empty()))?; + Some(m.lines().next().unwrap_or("").trim().chars().take(160).collect()) +} + +fn some(names: &[String]) -> String { + let more = names.len().saturating_sub(NAMES); + let head = names.iter().take(NAMES).cloned().collect::>().join(", "); + if more > 0 { + format!("{head} and {more} more") + } else { + head + } +} + +/// What coverage counts: every top directory with code of the index (`src/**/*.{ts,…}`) and the +/// files with code at the root. The project's own `coverage.include` is overridden: it may name +/// nothing (zustand's `src/**/` matches directories only) and is meant for its report, not for this. +fn include_globs<'a>(files: impl Iterator) -> Vec { + let mut out: Vec = files.map(|f| f.split_once('/').map_or(f.to_string(), |(top, _)| format!("{top}/**/*.{{ts,tsx,mts,cts,js,jsx,mjs,cjs}}"))).collect(); + out.sort(); + out.dedup(); + out +} + +/// One test file with coverage; `false` when it did not finish in time. The report goes to +/// `/coverage` (vitest empties that directory first), the log and the result next to it. +fn run_file(npx: &str, root: &Path, file: &str, include: &[String], dir: &Path) -> Result { + let log = std::fs::File::create(dir.join("out.log"))?; + let mut child = Command::new(npx) + .args(["vitest", "run", file, "--coverage.enabled", "--coverage.provider=v8", "--coverage.reporter=json", "--coverage.clean=true", "--reporter=json"]) + .args(include.iter().map(|g| format!("--coverage.include={g}"))) + .arg(format!("--coverage.reportsDirectory={}", dir.join("coverage").display())) + .arg(format!("--outputFile={}", dir.join("result.json").display())) + .env("FORCE_COLOR", "0") + .current_dir(root) + .stdin(Stdio::null()) + .stdout(log.try_clone()?) + .stderr(log) + .spawn() + .with_context(|| format!("running `{npx} vitest run {file}`"))?; + let started = Instant::now(); + loop { + if child.try_wait()?.is_some() { + return Ok(true); + } + if started.elapsed() > FILE_LIMIT { + let _ = child.kill(); + let _ = child.wait(); + return Ok(false); + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +fn run_with(npx: &str, root: &Path, g: &Graph) -> Result { + let version = ready(npx, root)?; + let started = Instant::now(); + let mut roots = vec![root.display().to_string()]; + if let Ok(c) = root.canonicalize() { + roots.push(c.display().to_string()); + } + // the test's symbol in the graph: a test file is its `module` + let module_of: HashMap<&str, &str> = g + .syms + .iter() + .filter(|s| s.sym.test && s.sym.kind == Kind::Module && gitnext_parse::family_of(&s.file) == Some("js")) + .map(|s| (s.file.as_str(), s.sym.name.as_str())) + .collect(); + let mut notes = Vec::new(); + let list = Command::new(npx).args(["vitest", "list", "--filesOnly", "--json"]).env("FORCE_COLOR", "0").current_dir(root).stdin(Stdio::null()).output().context("running `npx vitest list`")?; + let mut files = listed_files(&String::from_utf8_lossy(&list.stdout), &roots); + if files.is_empty() { + // an older vitest without `list --filesOnly`: the test files the index knows + notes.push(format!("JS/TS: `vitest list --filesOnly --json` gave no test file ({}); ran the test files of the index", last_line(&String::from_utf8_lossy(&list.stderr)))); + files = module_of.keys().map(|f| f.to_string()).collect(); + files.sort(); + } + if files.is_empty() { + bail!("vitest found no test file"); + } + let owner_of = owners(g); + let include = include_globs(owner_of.keys().copied().filter(|f| gitnext_parse::family_of(f) == Some("js"))); + if include.is_empty() { + bail!("the index has no JavaScript / TypeScript function outside the tests"); + } + let scratch = Scratch(root.join(".gitnext").join(format!("learn-js-{}", std::process::id()))); + std::fs::create_dir_all(&scratch.0)?; + + let mut pairs = BTreeSet::new(); + let (mut failing, mut timed_out, mut unknown, mut no_report) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for (k, file) in files.iter().enumerate() { + let Some(&name) = module_of.get(file.as_str()) else { + unknown.push(file.clone()); + continue; + }; + let dir = scratch.0.join(k.to_string()); + std::fs::create_dir_all(&dir)?; + if !run_file(npx, root, file, &include, &dir)? { + timed_out.push(file.clone()); + continue; + } + let result = std::fs::read_to_string(dir.join("result.json")).unwrap_or_default(); + let Ok(report) = std::fs::read_to_string(dir.join("coverage").join("coverage-final.json")) else { + // the file did not load (a missing fixture, a config error) or vitest itself stopped + let log = std::fs::read_to_string(dir.join("out.log")).unwrap_or_default(); + no_report.push(format!("{file} ({})", load_error(&result).unwrap_or_else(|| last_line(&log).to_string()))); + continue; + }; + if !passed(&result) { + failing.push(file.clone()); + } + for (f, lines) in covered_lines(&report, &roots) { + let Some(owners) = owner_of.get(f.as_str()) else { continue }; // not code of the index (built output, a test file) + for l in lines { + if let Some(Some(i)) = owners.get(l as usize) { + pairs.insert((*i, file.clone(), name.to_string())); + } + } + } + let _ = std::fs::remove_dir_all(&dir); + } + if !failing.is_empty() { + notes.push(format!("JS/TS: {} test files fail (what they ran is kept): {}", failing.len(), some(&failing))); + } + if !timed_out.is_empty() { + notes.push(format!("JS/TS: {} test files took longer than 10 minutes and are not in the map: {}", timed_out.len(), some(&timed_out))); + } + if !no_report.is_empty() { + notes.push(format!("JS/TS: {} test files wrote no coverage report and are not in the map: {}", no_report.len(), some(&no_report))); + } + if !unknown.is_empty() { + notes.push(format!("JS/TS: {} test files have no symbol in the index and were not run: {}", unknown.len(), some(&unknown))); + } + let ran = files.len() - unknown.len(); + Ok(Run { + source: "vitest-v8", + command: format!("npx vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json --coverage.include={}, each test file alone ({ran} files, vitest {version})", include.join(",")), + seconds: started.elapsed().as_secs_f32(), + tests_failed: !failing.is_empty() || !timed_out.is_empty() || !no_report.is_empty(), + pairs, + notes, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A report vitest 4 wrote for `tests/calc.test.ts` of the project in `tests/learn_js.rs`, + /// the repository's path replaced by `/r/demo`. + const REPORT: &str = include_str!("../../tests/data/learn_js/coverage-final.json"); + + #[test] + fn a_report_gives_the_lines_that_ran_by_file_of_the_repository() { + let roots = vec!["/r/demo".to_string()]; + let got = covered_lines(REPORT, &roots); + assert_eq!(got.keys().collect::>(), ["src/calc.ts"]); + let lines = &got["src/calc.ts"]; + // `add` and `double` ran (their bodies), `unused` did not + assert!(lines.contains(&2) && lines.contains(&6), "{lines:?}"); + assert!(!lines.contains(&10), "{lines:?}"); + // a file outside the repository is not counted + assert!(covered_lines(REPORT, &["/elsewhere".to_string()]).is_empty()); + } + + #[test] + fn the_test_files_come_from_vitest_list() { + let roots = vec!["/r/demo".to_string(), "/private/r/demo".to_string()]; + let json = r#"[{"file":"/r/demo/tests/b.test.ts"},{"file":"/private/r/demo/tests/a.ts","projectName":"x"},{"file":"/other/c.ts"}]"#; + assert_eq!(listed_files(json, &roots), ["tests/a.ts", "tests/b.test.ts"]); + assert!(listed_files("not json", &roots).is_empty()); + } + + #[test] + fn the_runner_is_read_from_package_json() { + assert_eq!(runner(r#"{"devDependencies":{"@testing-library/jest-dom":"1","vitest":"4"}}"#), Some("vitest")); + assert_eq!(runner(r#"{"devDependencies":{"jest":"29"}}"#), Some("jest")); + assert_eq!(runner(r#"{"scripts":{"test":"vitest run"}}"#), Some("vitest")); + assert_eq!(runner(r#"{"devDependencies":{"@testing-library/jest-dom":"1"},"scripts":{"test":"mocha"}}"#), None); + } + + #[test] + fn a_failed_test_run_is_seen_in_the_result() { + assert!(passed(r#"{"numFailedTestSuites":0,"numTotalTests":2,"numFailedTests":0,"success":true}"#)); + assert!(!passed(r#"{"numFailedTestSuites":0,"numTotalTests":2,"numFailedTests":1,"success":false}"#)); + // a file that did not load fails as a suite with no test + assert!(!passed(r#"{"numFailedTestSuites":1,"numTotalTests":0,"numFailedTests":0,"success":false}"#)); + // not JSON for serde_json: a lone surrogate quoted from a test + let quoted = r#"{"numFailedTestSuites":0,"numTotalTests":1,"numFailedTests":0,"testResults":[{"assertionResults":[{"title":"\ud800"}]}]}"#; + assert!(serde_json::from_str::(quoted).is_err() && passed(quoted)); + assert!(!passed("")); + let broken = r#"{"success":false,"testResults":[{"status":"failed","message":"ENOENT: no such file or directory, scandir 'tests/suite/src'\n at x"}]}"#; + assert_eq!(load_error(broken).as_deref(), Some("ENOENT: no such file or directory, scandir 'tests/suite/src'")); + assert_eq!(load_error(r#"{"testResults":[{"message":""}]}"#), None); + } + + #[test] + fn without_npx_the_message_says_what_to_install() { + let dir = std::env::temp_dir().join(format!("gitnext-learn-js-ready-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("node_modules/vitest")).unwrap(); + std::fs::create_dir_all(dir.join("node_modules/@vitest/coverage-v8")).unwrap(); + std::fs::write(dir.join("package.json"), r#"{"devDependencies":{"vitest":"4"}}"#).unwrap(); + std::fs::write(dir.join("node_modules/vitest/package.json"), r#"{"version":"4.1.11"}"#).unwrap(); + let e = ready("/nonexistent/bin/npx", &dir).unwrap_err().to_string(); + assert!(e.contains("is not in PATH") && e.contains("nodejs.org"), "{e}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn coverage_counts_the_top_directories_with_code_and_the_files_at_the_root() { + let got = include_globs(["src/a.ts", "src/x/b.tsx", "examples/demo/c.js", "rollup.config.mjs"].into_iter()); + assert_eq!(got, ["examples/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}", "rollup.config.mjs", "src/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}"]); + } + + #[test] + fn a_long_list_of_names_is_cut() { + let names: Vec = (1..=7).map(|i| format!("t{i}.ts")).collect(); + assert_eq!(some(&names), "t1.ts, t2.ts, t3.ts, t4.ts, t5.ts and 2 more"); + } +} diff --git a/crates/gitnext-ops/tests/data/learn_js/coverage-final.json b/crates/gitnext-ops/tests/data/learn_js/coverage-final.json new file mode 100644 index 0000000..df221ac --- /dev/null +++ b/crates/gitnext-ops/tests/data/learn_js/coverage-final.json @@ -0,0 +1,2 @@ +{"/r/demo/src/calc.ts": {"path":"/r/demo/src/calc.ts","statementMap":{"0":{"start":{"line":2,"column":2},"end":{"line":2,"column":null}},"1":{"start":{"line":6,"column":2},"end":{"line":6,"column":null}},"2":{"start":{"line":10,"column":2},"end":{"line":10,"column":null}},"3":{"start":{"line":13,"column":14},"end":{"line":13,"column":null}},"4":{"start":{"line":16,"column":2},"end":{"line":16,"column":null}}},"fnMap":{"0":{"name":"add","decl":{"start":{"line":1,"column":16},"end":{"line":1,"column":20}},"loc":{"start":{"line":1,"column":50},"end":{"line":3,"column":null}},"line":1},"1":{"name":"double","decl":{"start":{"line":5,"column":16},"end":{"line":5,"column":23}},"loc":{"start":{"line":5,"column":42},"end":{"line":7,"column":null}},"line":5},"2":{"name":"unused","decl":{"start":{"line":9,"column":16},"end":{"line":9,"column":33}},"loc":{"start":{"line":9,"column":33},"end":{"line":11,"column":null}},"line":9},"3":{"name":"byName","decl":{"start":{"line":15,"column":16},"end":{"line":15,"column":23}},"loc":{"start":{"line":15,"column":57},"end":{"line":17,"column":null}},"line":15}},"branchMap":{},"s":{"0":1,"1":1,"2":0,"3":1,"4":0},"f":{"0":1,"1":1,"2":0,"3":0},"b":{},"meta":{"lastBranch":0,"lastFunction":4,"lastStatement":5,"seen":{"f:1:16:1:20":0,"s:2:2:2:Infinity":0,"f:5:16:5:23":1,"s:6:2:6:Infinity":1,"f:9:16:9:33":2,"s:10:2:10:Infinity":2,"s:13:14:13:Infinity":3,"f:15:16:15:23":3,"s:16:2:16:Infinity":4},"fnNames":{}}} +} diff --git a/crates/gitnext-ops/tests/learn_js.rs b/crates/gitnext-ops/tests/learn_js.rs new file mode 100644 index 0000000..739914c --- /dev/null +++ b/crates/gitnext-ops/tests/learn_js.rs @@ -0,0 +1,94 @@ +//! `gitnext learn-tests` in a JavaScript / TypeScript project on vitest: the map names the test +//! files that really ran each function. What is missing is said without running node; the run +//! itself installs vitest from npm and is skipped unless `GITNEXT_TEST_NPM=1` (the parsing of a +//! saved coverage report is tested without node in `learn/js.rs`). + +use gitnext_ops::gitnext_index::{Graph, Index, TestMap}; +use gitnext_ops::{learn, locate}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn project(name: &str, files: &[(&str, &str)]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("gitnext-learn-js-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for (p, c) in files { + let f = dir.join(p); + std::fs::create_dir_all(f.parent().unwrap()).unwrap(); + std::fs::write(f, c).unwrap(); + } + dir +} + +fn graph(dir: &Path) -> Graph { + let mut idx = Index::open(dir).unwrap(); + idx.refresh().unwrap(); + idx.graph().unwrap() +} + +const PACKAGE: &str = "{ \"name\": \"demo\", \"private\": true, \"type\": \"module\", \"scripts\": { \"test\": \"vitest run\" },\n \"devDependencies\": { \"vitest\": \"4.1.11\", \"@vitest/coverage-v8\": \"4.1.11\" } }\n"; +const CALC: &str = "export function add(a: number, b: number): number {\n return a + b\n}\n\nexport function double(x: number): number {\n return add(x, x)\n}\n\nexport function unused(): number {\n return 7\n}\n\nconst table = { twice: double }\n\nexport function byName(name: \"twice\", x: number): number {\n return table[name](x)\n}\n"; +const CALC_TEST: &str = "import { expect, it } from 'vitest'\nimport { double } from '../src/calc'\n\nit('doubles', () => {\n expect(double(2)).toBe(4)\n})\n"; +const BY_NAME_TEST: &str = "import { expect, it } from 'vitest'\nimport { byName } from '../src/calc'\n\nit('looks up', () => {\n expect(byName('twice', 3)).toBe(6)\n})\n"; + +fn learn_error(dir: &Path) -> String { + let g = graph(dir); + let e = learn::learn(dir, &g).err().map(|e| format!("{e:#}")).unwrap_or_default(); + let _ = std::fs::remove_dir_all(dir); + e +} + +#[test] +fn a_project_on_jest_is_told_so_in_one_line() { + let dir = project("jest", &[("package.json", "{ \"devDependencies\": { \"jest\": \"29.7.0\" } }\n"), ("src/calc.ts", CALC), ("tests/calc.test.ts", CALC_TEST)]); + let e = learn_error(&dir); + assert_eq!(e, "JavaScript/TypeScript: the tests run with jest; learn-tests knows vitest only"); +} + +#[test] +fn what_is_not_installed_is_named_with_the_command_that_installs_it() { + let files = [("package.json", PACKAGE), ("src/calc.ts", CALC), ("tests/calc.test.ts", CALC_TEST)]; + let e = learn_error(&project("no-modules", &files)); + assert!(e.contains("vitest is not installed") && e.contains("npm install"), "{e}"); + // vitest is there, its v8 coverage is not: the version to install is vitest's own + let dir = project("no-coverage", &files); + std::fs::create_dir_all(dir.join("node_modules/vitest")).unwrap(); + std::fs::write(dir.join("node_modules/vitest/package.json"), "{ \"name\": \"vitest\", \"version\": \"4.1.11\" }\n").unwrap(); + let e = learn_error(&dir); + assert!(e.ends_with("vitest's v8 coverage is not installed: npm i -D @vitest/coverage-v8@4.1.11"), "{e}"); + let e = learn_error(&project("no-package", &[("src/calc.ts", CALC)])); + assert!(e.contains("no package.json at the root"), "{e}"); +} + +#[test] +fn a_vitest_project_learns_which_test_file_ran_which_function() { + if std::env::var("GITNEXT_TEST_NPM").as_deref() != Ok("1") { + eprintln!("skipped: set GITNEXT_TEST_NPM=1 to install vitest from npm and run it"); + return; + } + let dir = project("vitest", &[("package.json", PACKAGE), ("src/calc.ts", CALC), ("tests/calc.test.ts", CALC_TEST), ("tests/by-name.test.ts", BY_NAME_TEST)]); + // npm 10 stops on the peer dependencies of vitest 4 without --legacy-peer-deps + let installed = Command::new("npm").args(["install", "--no-audit", "--no-fund", "--loglevel=error", "--legacy-peer-deps"]).current_dir(&dir).status(); + assert!(installed.is_ok_and(|s| s.success()), "npm install failed in {}", dir.display()); + let g = graph(&dir); + let l = learn::learn(&dir, &g).unwrap(); + assert!(!l.tests_failed && l.tests == 2, "{}", learn::render(&l)); + assert!(l.command.contains("2 files"), "{}", l.command); + let files_of = |name: &str| { + let q = g.unique_name(g.find(name)[0]); + let mut t: Vec = TestMap::open_existing(&dir).unwrap().tests_of(&q).into_iter().map(|c| c.file).collect(); + t.sort(); + t + }; + assert_eq!(files_of("double"), ["tests/by-name.test.ts", "tests/calc.test.ts"]); + // reached through an object: the call graph does not see it, the coverage run does + assert_eq!(files_of("add"), ["tests/by-name.test.ts", "tests/calc.test.ts"]); + assert_eq!(files_of("byName"), ["tests/by-name.test.ts"]); + assert!(files_of("unused").is_empty()); + let (t, source) = locate::tests_for_at(&g, &dir, g.find("byName")[0], 4); + assert_eq!(source, locate::Source::Learned { stale: false }); + assert_eq!(t.iter().map(|x| g.syms[x.symbol].file.as_str()).collect::>(), ["tests/by-name.test.ts"]); + // the temporary reports are gone + let left: Vec<_> = std::fs::read_dir(dir.join(".gitnext")).unwrap().flatten().filter(|e| e.file_name().to_string_lossy().starts_with("learn-js")).collect(); + assert!(left.is_empty()); + let _ = std::fs::remove_dir_all(&dir); +} From 719cfd8a8abcb4aed67dd6dc8b1afbe75a064268 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 04:07:03 +0300 Subject: [PATCH 12/18] eval: learn-tests for JS/TS measured through the product learn_tests_js.py --product runs gitnext learn-tests on the eval's checkout and scores tests-for by test file before the map (the call graph) and after it, on the functions the prototype took (the eval's cached ones and seed 7 random ones). yaml, 32 functions: recall 16% -> 100%, precision 33% -> 100%, failing among the first 10 34% -> 100%, caught by the first 10 41% -> 100%. zustand, 19 functions: recall 68% -> 100%, precision 68% -> 94%, first 10 68% -> 94%, caught 74% -> 100% - the prototype's numbers on both. The map takes 37 s and 22 s, 6.6x and 2.6x a plain run. Results: eval/results/learn-tests-{yaml,zustand}.product.{md,json}. Co-Authored-By: Claude Opus 5.5 --- eval/learn_tests_js.py | 151 ++- eval/results/learn-tests-yaml.product.json | 1045 +++++++++++++++++ eval/results/learn-tests-yaml.product.md | 68 ++ eval/results/learn-tests-zustand.product.json | 655 +++++++++++ eval/results/learn-tests-zustand.product.md | 53 + 5 files changed, 1970 insertions(+), 2 deletions(-) create mode 100644 eval/results/learn-tests-yaml.product.json create mode 100644 eval/results/learn-tests-yaml.product.md create mode 100644 eval/results/learn-tests-zustand.product.json create mode 100644 eval/results/learn-tests-zustand.product.md diff --git a/eval/learn_tests_js.py b/eval/learn_tests_js.py index 29158bc..1e86036 100644 --- a/eval/learn_tests_js.py +++ b/eval/learn_tests_js.py @@ -7,22 +7,29 @@ counting it would book every function of every imported module to the test. `tests-for` is scored by file as in `run.check_tests`. The driver is `learn_tests_go.measure`, with the test name being the test file. +`--product` measures the product instead: `gitnext learn-tests` writes the map, and `tests-for` is scored before +it (the call graph) and after it (the map), on the same functions (`--pick-from`, `--n2`, `--seed`). Results go to +`eval/results/learn-tests-.product.{md,json}`; the prototype's files stay as they are. `--causes` writes, for every function whose failing test files `tests-for` misses, how each missed file reaches the code (package name through the vitest alias, `index.ts` re-export, a relative import of `src/`, a helper), whether gitnext sees test symbols in it, and what `tests-for` named instead. GITNEXT=$PWD/target/release/gitnext python3 eval/learn_tests_js.py [--repos zustand,yaml] [--n2 12] [--seed 7] \\ - [--pick-from ] [--causes] + [--pick-from ] [--causes | --product] """ from __future__ import annotations +import argparse import json +import random import re +import statistics import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) +import learn_tests as lt # noqa: E402 metrics of the Python prototype import learn_tests_go as lg # noqa: E402 the shared driver of 005 import run # noqa: E402 the eval: same checkout, ground truth and cache @@ -294,9 +301,149 @@ def render_causes(res: dict) -> str: return "\n".join(L) + "\n" +# ---------------------------------------------------------------- the product: gitnext learn-tests + +def timed_learn(root: Path) -> tuple[float, str]: + t = time.time() + _, out = run.gx(["learn-tests"], root, timeout=3600) + return round(time.time() - t, 1), out.strip() + + +def product(repo: dict, a) -> dict: + """`tests-for` by file before and after `gitnext learn-tests`, on the functions `lg.measure` would pick.""" + key = f"{repo['name']}@{repo['commit'][:12]}" + prefix = f"{key}:{TS.get('tests_key', 'tests')}:" + root = run.prepare(repo) + run.gx(["init"], root, timeout=1800) + (root / ".gitnext" / "tests.db").unlink(missing_ok=True) + syms = run.symbols(root) + by_q = {s["qualified"]: s for s in lg.fn_syms(syms)} + cost: dict = {} + plain = [] + for _ in range(a.repeat): + t = time.time() + run.sh(TS["tests"], root, timeout=3600) + plain.append(time.time() - t) + cost["plain_s"] = round(statistics.median(plain), 1) + # the product installs nothing: what it says when the coverage package is missing, then the prototype's install + if not (root / "node_modules" / "@vitest" / "coverage-v8").is_dir(): + cost["message_without_coverage"] = run.gx(["learn-tests"], root)[1].strip() + version = json.loads((root / "node_modules" / "vitest" / "package.json").read_text())["version"] + t = time.time() + code, out = run.sh(f"npm i --no-save --no-audit --no-fund --loglevel=error --legacy-peer-deps @vitest/coverage-v8@{version}", + root, timeout=900) + cost["install_s"] = round(time.time() - t, 1) + if code: + raise RuntimeError(f"npm i @vitest/coverage-v8@{version}: {out[-400:]}") + + # the same sample as lg.measure: the eval's functions, then random ones of the library + if not run.CACHE: + run._load_cache() + known = json.loads(Path(a.pick_from).read_text()) if a.pick_from else run.CACHE + cached = [q[len(prefix):] for q in known if q.startswith(prefix)] + pick = [by_q[q] for q in sorted(cached) if q in by_q] + cands = [s for s in lg.fn_syms(syms) if not s.get("test") and js_library(s["file"]) and s["qualified"] not in set(cached) + and (b := run.body_start(root, s, repo["lang"])) and s["end_line"] - b + 1 >= 3] + random.Random(a.seed).shuffle(cands) + groups = (("pick", pick), ("random", cands[:a.n2])) + base = set(run.cached(f"{key}:baseline", lambda: run.baseline_tests(root, TS)).get("failed", [])) + + graph = {s["qualified"]: tests_for_files(root, s["qualified"])[0] for _, ss in groups for s in ss} + cost["learn_cold_s"], cost["learn_output"] = timed_learn(root) + cost["learn_warm_s"], _ = timed_learn(root) + cost["learn_x_plain"] = round(cost["learn_warm_s"] / cost["plain_s"], 1) if cost["plain_s"] else None + db = root / ".gitnext" / "tests.db" + cost["map_sqlite_bytes"] = db.stat().st_size if db.exists() else 0 + mapped, from_map = {}, set() + for _, ss in groups: + for s in ss: + q = s["qualified"] + mapped[q] = tests_for_files(root, q)[0] + if run.gx(["tests-for", q], root)[1].startswith("tests that ran"): + from_map.add(q) + + rows = [] + for group, ss in groups: + for s in ss: + q = s["qualified"] + print(f" {repo['name']} {group} {q}", flush=True) + tr = run.cached(f"{prefix}{q}", lambda: run.failing_tests(root, repo, s, base), + keep=lambda v: v.get("skipped") != "the test suite did not run") + r = {"group": group, "symbol": q, "file": s["file"], "from_map": q in from_map, + "named_graph": len(graph[q]), "named_map": len(mapped[q])} + if "skipped" in tr or not tr.get("failed"): + r["skipped"] = tr.get("skipped") or "no test fails when it throws" + rows.append(r) + continue + truth = set(tr["failed"]) + r.update(truth=len(truth), missed_by_map=sorted(truth - set(mapped[q]))[:10], + score={"graph": lt.score(graph[q], truth), "map": lt.score(mapped[q], truth)}) + rows.append(r) + summary = {} + for g in ("pick", "random", "all"): + rs = [r for r in rows if r.get("score") and (g == "all" or r["group"] == g)] + summary[g] = {src: {m: lt.mean([r["score"][src][m] for r in rs]) for m in ("recall", "precision", "top_precision", "caught")} + for src in ("graph", "map")} + summary[g]["functions"] = len(rs) + summary[g]["from_map"] = sum(1 for r in rs if r["from_map"]) + return {"repo": repo["name"], "lang": repo["lang"], "commit": repo["commit"][:12], "seed": a.seed, + "gitnext_commit": run.sh("git rev-parse --short HEAD", run.HERE)[1].strip(), + "functions": rows, "summary": summary, "cost": cost} + + +def render_product(res: dict) -> str: + pct = lt.pct + L = [f"# gitnext learn-tests на {res['repo']}: `tests-for` до карты (граф) и после (карта)", + f"{res['repo']} {res['commit']}, gitnext {res['gitnext_commit']}, seed {res['seed']}. Истина — eval " + "(файлы тестов, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные.", + "Ответ — `tests-for <функция> --all` по файлам; «из карты» — сколько функций `tests-for` ответил по карте.", "", + "| выборка | функций | из карты | источник | полнота | точность | из 10 падают | поймано |", + "|---|---:|---:|---|---:|---:|---:|---:|"] + for g, gname in (("pick", "pick (кэш eval)"), ("random", "случайные"), ("all", "все")): + s = res["summary"][g] + for src, sname in (("graph", "граф"), ("map", "карта")): + m = s[src] + L.append(f"| {gname} | {s['functions']} | {s['from_map']} | {sname} | {pct(m['recall'])} | {pct(m['precision'])} | " + f"{pct(m['top_precision'])} | {pct(m['caught'])} |") + L += ["", "## По функциям", "", "| выборка | функция | падают | граф: полн / из 10 | карта: полн / точн / из 10 | мимо карты |", + "|---|---|---:|---|---|---|"] + for r in res["functions"]: + if "score" not in r: + L.append(f"| {r['group']} | `{r['symbol']}` | 0 | {r['skipped']} | названо {r['named_map']} | |") + continue + g, c = r["score"]["graph"], r["score"]["map"] + L.append(f"| {r['group']} | `{r['symbol']}` | {r['truth']} | {pct(g['recall'])} / {pct(g['top_precision'])} | " + f"{pct(c['recall'])} / {pct(c['precision'])} / {pct(c['top_precision'])} | {', '.join(r['missed_by_map'][:4])} |") + L += ["", "## Цена", ""] + for k, v in res["cost"].items(): + L += [f"- `{k}`:", "", "```", str(v), "```"] if "\n" in str(v) else [f"- `{k}`: {v}"] + return "\n".join(L) + "\n" + + +def main_product() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--product", action="store_true") + ap.add_argument("--repos", default="zustand,yaml") + ap.add_argument("--n2", type=int, default=12) + ap.add_argument("--seed", type=int, default=7) + ap.add_argument("--repeat", type=int, default=3) + ap.add_argument("--pick-from") + a = ap.parse_args() + repos = {r["name"]: r for r in json.loads((run.HERE / "repos.json").read_text())} + for name in a.repos.split(","): + res = product(repos[name], a) + out = run.HERE / "results" / f"learn-tests-{name}.product" + Path(f"{out}.json").write_text(json.dumps(res, ensure_ascii=False, indent=1)) + md = render_product(res) + Path(f"{out}.md").write_text(md) + print("\n" + md, flush=True) + + def main() -> None: + if "--product" in sys.argv: + main_product() + return if "--causes" in sys.argv: - import argparse ap = argparse.ArgumentParser() ap.add_argument("--repos", default="zustand,yaml") ap.add_argument("--pick-from") diff --git a/eval/results/learn-tests-yaml.product.json b/eval/results/learn-tests-yaml.product.json new file mode 100644 index 0000000..c58f628 --- /dev/null +++ b/eval/results/learn-tests-yaml.product.json @@ -0,0 +1,1045 @@ +{ + "repo": "yaml", + "lang": "typescript", + "commit": "528ef30d6ded", + "seed": 7, + "gitnext_commit": "eeae698", + "functions": [ + { + "group": "pick", + "symbol": "src::compose::compose_collection::composeCollection", + "file": "src/compose/compose-collection.ts", + "from_map": true, + "named_graph": 0, + "named_map": 17, + "truth": 17, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::compose_doc::composeDoc", + "file": "src/compose/compose-doc.ts", + "from_map": true, + "named_graph": 13, + "named_map": 18, + "truth": 18, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.6111111111111112, + "precision": 0.8461538461538461, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::compose_scalar::composeScalar", + "file": "src/compose/compose-scalar.ts", + "from_map": true, + "named_graph": 0, + "named_map": 18, + "truth": 18, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::composer::Composer::compose", + "file": "src/compose/composer.ts", + "from_map": true, + "named_graph": 14, + "named_map": 17, + "truth": 17, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.7058823529411765, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::resolve_block_seq::resolveBlockSeq", + "file": "src/compose/resolve-block-seq.ts", + "from_map": true, + "named_graph": 0, + "named_map": 13, + "truth": 13, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::resolve_end::resolveEnd", + "file": "src/compose/resolve-end.ts", + "from_map": true, + "named_graph": 13, + "named_map": 19, + "truth": 19, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.5789473684210527, + "precision": 0.8461538461538461, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 19, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::resolve_flow_scalar::resolveFlowScalar", + "file": "src/compose/resolve-flow-scalar.ts", + "from_map": true, + "named_graph": 0, + "named_map": 19, + "skipped": "mutant does not compile" + }, + { + "group": "pick", + "symbol": "src::compose::util_contains_newline::containsNewline", + "file": "src/compose/util-contains-newline.ts", + "from_map": true, + "named_graph": 0, + "named_map": 17, + "truth": 17, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::compose::util_flow_indent_check::flowIndentCheck", + "file": "src/compose/util-flow-indent-check.ts", + "from_map": true, + "named_graph": 0, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::doc::Document::Document::setSchema", + "file": "src/doc/Document.ts", + "from_map": true, + "named_graph": 1, + "named_map": 20, + "truth": 20, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.05, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::doc::NodeCreator::NodeCreator::setAnchors", + "file": "src/doc/NodeCreator.ts", + "from_map": true, + "named_graph": 7, + "named_map": 20, + "truth": 20, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.35, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::doc::directives::Directives::atDocument", + "file": "src/doc/directives.ts", + "from_map": true, + "named_graph": 0, + "named_map": 18, + "truth": 18, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::YAMLMap::YAMLMap::#stringifyFlowMap", + "file": "src/nodes/YAMLMap.ts", + "from_map": true, + "named_graph": 0, + "named_map": 8, + "truth": 8, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::YAMLSeq::YAMLSeq::#stringifyBlockSeq", + "file": "src/nodes/YAMLSeq.ts", + "from_map": true, + "named_graph": 0, + "named_map": 12, + "truth": 12, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::YAMLSet::YAMLSet::#stringifyBlockSet", + "file": "src/nodes/YAMLSet.ts", + "from_map": true, + "named_graph": 0, + "named_map": 2, + "truth": 2, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::YAMLSet::YAMLSet::#stringifyFlowSet", + "file": "src/nodes/YAMLSet.ts", + "from_map": true, + "named_graph": 0, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::toJS::ToJSContext::resolveAlias", + "file": "src/nodes/toJS.ts", + "from_map": true, + "named_graph": 0, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::nodes::toJS::ToJSContext::setAnchor", + "file": "src/nodes/toJS.ts", + "from_map": true, + "named_graph": 0, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::parse::cst::tokenType", + "file": "src/parse/cst.ts", + "from_map": true, + "named_graph": 14, + "named_map": 19, + "truth": 19, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.631578947368421, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 19, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::parse::lexer::Lexer::flow", + "file": "src/parse/lexer.ts", + "from_map": true, + "named_graph": 16, + "named_map": 15, + "truth": 15, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.6, + "precision": 0.5625, + "predicted": 16, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 15, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::public_api::parseAllDocuments", + "file": "src/public-api.ts", + "from_map": true, + "named_graph": 2, + "named_map": 4, + "truth": 4, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.25, + "precision": 0.5, + "predicted": 2, + "top_hits": 1, + "top_n": 2, + "top_precision": 0.5, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "pick", + "symbol": "src::schema::yaml_1_1::pairs::resolvePairs", + "file": "src/schema/yaml-1.1/pairs.ts", + "from_map": true, + "named_graph": 0, + "named_map": 4, + "truth": 4, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::nodes::YAMLSeq::YAMLSeq::toJS", + "file": "src/nodes/YAMLSeq.ts", + "from_map": true, + "named_graph": 2, + "named_map": 8, + "truth": 8, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.125, + "precision": 0.5, + "predicted": 2, + "top_hits": 1, + "top_n": 2, + "top_precision": 0.5, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::nodes::Alias::Alias::toJS", + "file": "src/nodes/Alias.ts", + "from_map": true, + "named_graph": 2, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::doc::anchors::anchorIsValid", + "file": "src/doc/anchors.ts", + "from_map": true, + "named_graph": 0, + "named_map": 8, + "truth": 8, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::schema::core::float::couldBeFloat", + "file": "src/schema/core/float.ts", + "from_map": true, + "named_graph": 0, + "named_map": 18, + "truth": 18, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::schema::yaml_1_1::pairs::createPairs", + "file": "src/schema/yaml-1.1/pairs.ts", + "from_map": true, + "named_graph": 0, + "named_map": 3, + "truth": 3, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::doc::Document::Document::set", + "file": "src/doc/Document.ts", + "from_map": true, + "named_graph": 0, + "named_map": 5, + "truth": 5, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::nodes::util_clone_map_or_set::cloneMapOrSet", + "file": "src/nodes/util-clone-map-or-set.ts", + "from_map": true, + "named_graph": 0, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::parse::lexer::Lexer::indicators", + "file": "src/parse/lexer.ts", + "from_map": true, + "named_graph": 5, + "named_map": 20, + "truth": 20, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.2, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::stringify::stringify::getResolvedOptions", + "file": "src/stringify/stringify.ts", + "from_map": true, + "named_graph": 1, + "named_map": 17, + "truth": 17, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 1, + "top_hits": 0, + "top_n": 1, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::stringify::stringifyNumber::stringifyNumber", + "file": "src/stringify/stringifyNumber.ts", + "from_map": true, + "named_graph": 1, + "named_map": 11, + "truth": 11, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.09090909090909091, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::parse::cst_scalar::createScalarToken", + "file": "src/parse/cst-scalar.ts", + "from_map": true, + "named_graph": 0, + "named_map": 1, + "skipped": "mutant does not compile" + }, + { + "group": "random", + "symbol": "src::parse::lexer::lex", + "file": "src/parse/lexer.ts", + "from_map": true, + "named_graph": 17, + "named_map": 20, + "truth": 20, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.75, + "precision": 0.8823529411764706, + "predicted": 17, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + ], + "summary": { + "pick": { + "graph": { + "recall": 0.18, + "precision": 0.308, + "top_precision": 0.319, + "caught": 0.381 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "top_precision": 1.0, + "caught": 1.0 + }, + "functions": 21, + "from_map": 21 + }, + "random": { + "graph": { + "recall": 0.136, + "precision": 0.38, + "top_precision": 0.382, + "caught": 0.455 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "top_precision": 1.0, + "caught": 1.0 + }, + "functions": 11, + "from_map": 11 + }, + "all": { + "graph": { + "recall": 0.165, + "precision": 0.333, + "top_precision": 0.341, + "caught": 0.406 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "top_precision": 1.0, + "caught": 1.0 + }, + "functions": 32, + "from_map": 32 + } + }, + "cost": { + "plain_s": 5.7, + "learn_cold_s": 37.3, + "learn_output": "learned which tests run which functions: 285 functions, 22 tests, 2942 pairs (37s: npx vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json --coverage.include=src/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}, each test file alone (24 files, vitest 4.1.11))\n54 functions were run by no test — for those `tests-for` still uses the call graph\nnote: JS/TS: 2 test files wrote no coverage report and are not in the map: tests/json-test-suite.ts (ENOENT: no such file or directory, scandir '/home/claude/gitnext/eval/.work/yaml/tests/json-test-suite/test_parsing'), tests/yaml-test-suite.ts (ENOENT: no such file or directory, scandir '/home/claude/gitnext/eval/.work/yaml/tests/yaml-test-suite/src')\nnote: the test run itself reported failures; the map holds what ran\n`tests-for`, `look` and `check` now use the map; run `gitnext learn-tests` again after larger changes", + "learn_warm_s": 37.8, + "learn_x_plain": 6.6, + "map_sqlite_bytes": 225280 + } +} \ No newline at end of file diff --git a/eval/results/learn-tests-yaml.product.md b/eval/results/learn-tests-yaml.product.md new file mode 100644 index 0000000..1684c6f --- /dev/null +++ b/eval/results/learn-tests-yaml.product.md @@ -0,0 +1,68 @@ +# gitnext learn-tests на yaml: `tests-for` до карты (граф) и после (карта) +yaml 528ef30d6ded, gitnext eeae698, seed 7. Истина — eval (файлы тестов, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. +Ответ — `tests-for <функция> --all` по файлам; «из карты» — сколько функций `tests-for` ответил по карте. + +| выборка | функций | из карты | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 21 | 21 | граф | 18% | 31% | 32% | 38% | +| pick (кэш eval) | 21 | 21 | карта | 100% | 100% | 100% | 100% | +| случайные | 11 | 11 | граф | 14% | 38% | 38% | 46% | +| случайные | 11 | 11 | карта | 100% | 100% | 100% | 100% | +| все | 32 | 32 | граф | 16% | 33% | 34% | 41% | +| все | 32 | 32 | карта | 100% | 100% | 100% | 100% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | карта: полн / точн / из 10 | мимо карты | +|---|---|---:|---|---|---| +| pick | `src::compose::compose_collection::composeCollection` | 17 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::compose::compose_doc::composeDoc` | 18 | 61% / 90% | 100% / 100% / 100% | | +| pick | `src::compose::compose_scalar::composeScalar` | 18 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::compose::composer::Composer::compose` | 17 | 71% / 90% | 100% / 100% / 100% | | +| pick | `src::compose::resolve_block_seq::resolveBlockSeq` | 13 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::compose::resolve_end::resolveEnd` | 19 | 58% / 90% | 100% / 100% / 100% | | +| pick | `src::compose::resolve_flow_scalar::resolveFlowScalar` | 0 | mutant does not compile | названо 19 | | +| pick | `src::compose::util_contains_newline::containsNewline` | 17 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::compose::util_flow_indent_check::flowIndentCheck` | 1 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::doc::Document::Document::setSchema` | 20 | 5% / 100% | 100% / 100% / 100% | | +| pick | `src::doc::NodeCreator::NodeCreator::setAnchors` | 20 | 35% / 100% | 100% / 100% / 100% | | +| pick | `src::doc::directives::Directives::atDocument` | 18 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::YAMLMap::YAMLMap::#stringifyFlowMap` | 8 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::YAMLSeq::YAMLSeq::#stringifyBlockSeq` | 12 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::YAMLSet::YAMLSet::#stringifyBlockSet` | 2 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::YAMLSet::YAMLSet::#stringifyFlowSet` | 1 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::toJS::ToJSContext::resolveAlias` | 6 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::nodes::toJS::ToJSContext::setAnchor` | 6 | 0% / 0% | 100% / 100% / 100% | | +| pick | `src::parse::cst::tokenType` | 19 | 63% / 90% | 100% / 100% / 100% | | +| pick | `src::parse::lexer::Lexer::flow` | 15 | 60% / 60% | 100% / 100% / 100% | | +| pick | `src::public_api::parseAllDocuments` | 4 | 25% / 50% | 100% / 100% / 100% | | +| pick | `src::schema::yaml_1_1::pairs::resolvePairs` | 4 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::nodes::YAMLSeq::YAMLSeq::toJS` | 8 | 12% / 50% | 100% / 100% / 100% | | +| random | `src::nodes::Alias::Alias::toJS` | 6 | 33% / 100% | 100% / 100% / 100% | | +| random | `src::doc::anchors::anchorIsValid` | 8 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::schema::core::float::couldBeFloat` | 18 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::schema::yaml_1_1::pairs::createPairs` | 3 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::doc::Document::Document::set` | 5 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::nodes::util_clone_map_or_set::cloneMapOrSet` | 1 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::parse::lexer::Lexer::indicators` | 20 | 20% / 80% | 100% / 100% / 100% | | +| random | `src::stringify::stringify::getResolvedOptions` | 17 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::stringify::stringifyNumber::stringifyNumber` | 11 | 9% / 100% | 100% / 100% / 100% | | +| random | `src::parse::cst_scalar::createScalarToken` | 0 | mutant does not compile | названо 1 | | +| random | `src::parse::lexer::lex` | 20 | 75% / 90% | 100% / 100% / 100% | | + +## Цена + +- `plain_s`: 5.7 +- `learn_cold_s`: 37.3 +- `learn_output`: + +``` +learned which tests run which functions: 285 functions, 22 tests, 2942 pairs (37s: npx vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json --coverage.include=src/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}, each test file alone (24 files, vitest 4.1.11)) +54 functions were run by no test — for those `tests-for` still uses the call graph +note: JS/TS: 2 test files wrote no coverage report and are not in the map: tests/json-test-suite.ts (ENOENT: no such file or directory, scandir '/home/claude/gitnext/eval/.work/yaml/tests/json-test-suite/test_parsing'), tests/yaml-test-suite.ts (ENOENT: no such file or directory, scandir '/home/claude/gitnext/eval/.work/yaml/tests/yaml-test-suite/src') +note: the test run itself reported failures; the map holds what ran +`tests-for`, `look` and `check` now use the map; run `gitnext learn-tests` again after larger changes +``` +- `learn_warm_s`: 37.8 +- `learn_x_plain`: 6.6 +- `map_sqlite_bytes`: 225280 diff --git a/eval/results/learn-tests-zustand.product.json b/eval/results/learn-tests-zustand.product.json new file mode 100644 index 0000000..76618b4 --- /dev/null +++ b/eval/results/learn-tests-zustand.product.json @@ -0,0 +1,655 @@ +{ + "repo": "zustand", + "lang": "typescript", + "commit": "9fed2eb2f3af", + "seed": 7, + "gitnext_commit": "eeae698", + "functions": [ + { + "group": "pick", + "symbol": "src::middleware::persist::createJSONStorage", + "file": "src/middleware/persist.ts", + "from_map": true, + "named_graph": 3, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.5, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::devtools::setStateFromDevtools", + "file": "src/middleware/devtools.ts", + "from_map": true, + "named_graph": 1, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::persist::persistImpl", + "file": "src/middleware/persist.ts", + "from_map": true, + "named_graph": 0, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::persist::parse", + "file": "src/middleware/persist.ts", + "from_map": true, + "named_graph": 0, + "named_map": 5, + "truth": 3, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 0.6, + "predicted": 5, + "top_hits": 3, + "top_n": 5, + "top_precision": 0.6, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::devtools::maskToRegex", + "file": "src/middleware/devtools.ts", + "from_map": true, + "named_graph": 1, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::vanilla::subscribe", + "file": "src/vanilla.ts", + "from_map": true, + "named_graph": 3, + "named_map": 7, + "truth": 7, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.42857142857142855, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::vanilla::shallow::shallow", + "file": "src/vanilla/shallow.ts", + "from_map": true, + "named_graph": 2, + "named_map": 2, + "truth": 2, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::react::shallow::useShallow", + "file": "src/react/shallow.ts", + "from_map": true, + "named_graph": 2, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 0.5, + "predicted": 2, + "top_hits": 1, + "top_n": 2, + "top_precision": 0.5, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::devtools::parseDropdownArgs", + "file": "src/middleware/devtools.ts", + "from_map": true, + "named_graph": 1, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::react::createImpl", + "file": "src/react.ts", + "from_map": true, + "named_graph": 9, + "named_map": 9, + "truth": 9, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::vanilla::createStoreImpl", + "file": "src/vanilla.ts", + "from_map": true, + "named_graph": 13, + "named_map": 13, + "truth": 13, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::devtools::getTrackedConnectionState", + "file": "src/middleware/devtools.ts", + "from_map": true, + "named_graph": 1, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::vanilla::shallow::compareIterables", + "file": "src/vanilla/shallow.ts", + "from_map": true, + "named_graph": 2, + "named_map": 2, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 0.5, + "predicted": 2, + "top_hits": 1, + "top_n": 2, + "top_precision": 0.5, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 0.5, + "predicted": 2, + "top_hits": 1, + "top_n": 2, + "top_precision": 0.5, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::traditional::createWithEqualityFnImpl", + "file": "src/traditional.ts", + "from_map": true, + "named_graph": 2, + "named_map": 2, + "truth": 2, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::devtools::getParamNames", + "file": "src/middleware/devtools.ts", + "from_map": true, + "named_graph": 1, + "named_map": 1, + "truth": 1, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::persist::toSyncThenable", + "file": "src/middleware/persist.ts", + "from_map": false, + "named_graph": 0, + "named_map": 0, + "skipped": "mutant does not compile" + }, + { + "group": "random", + "symbol": "src::vanilla::shallow::hasIterableEntries", + "file": "src/vanilla/shallow.ts", + "from_map": true, + "named_graph": 2, + "named_map": 2, + "skipped": "mutant does not compile" + }, + { + "group": "random", + "symbol": "src::vanilla::shallow::compareEntries", + "file": "src/vanilla/shallow.ts", + "from_map": true, + "named_graph": 2, + "named_map": 2, + "truth": 2, + "missed_by_map": [], + "score": { + "graph": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::redux::reduxImpl", + "file": "src/middleware/redux.ts", + "from_map": true, + "named_graph": 0, + "named_map": 3, + "truth": 2, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::persist::toThenable", + "file": "src/middleware/persist.ts", + "from_map": true, + "named_graph": 0, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + }, + { + "group": "random", + "symbol": "src::middleware::persist::hydrate", + "file": "src/middleware/persist.ts", + "from_map": true, + "named_graph": 0, + "named_map": 6, + "truth": 6, + "missed_by_map": [], + "score": { + "graph": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + ], + "summary": { + "pick": { + "graph": { + "recall": 0.5, + "precision": 1.0, + "top_precision": 1.0, + "caught": 1.0 + }, + "map": { + "recall": 1.0, + "precision": 1.0, + "top_precision": 1.0, + "caught": 1.0 + }, + "functions": 1, + "from_map": 1 + }, + "random": { + "graph": { + "recall": 0.69, + "precision": 0.667, + "top_precision": 0.667, + "caught": 0.722 + }, + "map": { + "recall": 1.0, + "precision": 0.931, + "top_precision": 0.931, + "caught": 1.0 + }, + "functions": 18, + "from_map": 18 + }, + "all": { + "graph": { + "recall": 0.68, + "precision": 0.684, + "top_precision": 0.684, + "caught": 0.737 + }, + "map": { + "recall": 1.0, + "precision": 0.935, + "top_precision": 0.935, + "caught": 1.0 + }, + "functions": 19, + "from_map": 19 + } + }, + "cost": { + "plain_s": 8.5, + "learn_cold_s": 22.4, + "learn_output": "learned which tests run which functions: 48 functions, 14 tests, 164 pairs (22s: npx vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json --coverage.include=examples/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs},rollup.config.mjs,src/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}, each test file alone (14 files, vitest 5.1.0))\n38 functions were run by no test — for those `tests-for` still uses the call graph\n`tests-for`, `look` and `check` now use the map; run `gitnext learn-tests` again after larger changes", + "learn_warm_s": 22.3, + "learn_x_plain": 2.6, + "map_sqlite_bytes": 32768 + } +} \ No newline at end of file diff --git a/eval/results/learn-tests-zustand.product.md b/eval/results/learn-tests-zustand.product.md new file mode 100644 index 0000000..573c4a9 --- /dev/null +++ b/eval/results/learn-tests-zustand.product.md @@ -0,0 +1,53 @@ +# gitnext learn-tests на zustand: `tests-for` до карты (граф) и после (карта) +zustand 9fed2eb2f3af, gitnext eeae698, seed 7. Истина — eval (файлы тестов, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. +Ответ — `tests-for <функция> --all` по файлам; «из карты» — сколько функций `tests-for` ответил по карте. + +| выборка | функций | из карты | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 1 | 1 | граф | 50% | 100% | 100% | 100% | +| pick (кэш eval) | 1 | 1 | карта | 100% | 100% | 100% | 100% | +| случайные | 18 | 18 | граф | 69% | 67% | 67% | 72% | +| случайные | 18 | 18 | карта | 100% | 93% | 93% | 100% | +| все | 19 | 19 | граф | 68% | 68% | 68% | 74% | +| все | 19 | 19 | карта | 100% | 94% | 94% | 100% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | карта: полн / точн / из 10 | мимо карты | +|---|---|---:|---|---|---| +| pick | `src::middleware::persist::createJSONStorage` | 6 | 50% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::devtools::setStateFromDevtools` | 1 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::persist::persistImpl` | 6 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::middleware::persist::parse` | 3 | 0% / 0% | 100% / 60% / 60% | | +| random | `src::middleware::devtools::maskToRegex` | 1 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::vanilla::subscribe` | 7 | 43% / 100% | 100% / 100% / 100% | | +| random | `src::vanilla::shallow::shallow` | 2 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::react::shallow::useShallow` | 1 | 100% / 50% | 100% / 100% / 100% | | +| random | `src::middleware::devtools::parseDropdownArgs` | 1 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::react::createImpl` | 9 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::vanilla::createStoreImpl` | 13 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::devtools::getTrackedConnectionState` | 1 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::vanilla::shallow::compareIterables` | 1 | 100% / 50% | 100% / 50% / 50% | | +| random | `src::traditional::createWithEqualityFnImpl` | 2 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::devtools::getParamNames` | 1 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::persist::toSyncThenable` | 0 | mutant does not compile | названо 0 | | +| random | `src::vanilla::shallow::hasIterableEntries` | 0 | mutant does not compile | названо 2 | | +| random | `src::vanilla::shallow::compareEntries` | 2 | 100% / 100% | 100% / 100% / 100% | | +| random | `src::middleware::redux::reduxImpl` | 2 | 0% / 0% | 100% / 67% / 67% | | +| random | `src::middleware::persist::toThenable` | 6 | 0% / 0% | 100% / 100% / 100% | | +| random | `src::middleware::persist::hydrate` | 6 | 0% / 0% | 100% / 100% / 100% | | + +## Цена + +- `plain_s`: 8.5 +- `learn_cold_s`: 22.4 +- `learn_output`: + +``` +learned which tests run which functions: 48 functions, 14 tests, 164 pairs (22s: npx vitest run --coverage.enabled --coverage.provider=v8 --coverage.reporter=json --coverage.include=examples/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs},rollup.config.mjs,src/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}, each test file alone (14 files, vitest 5.1.0)) +38 functions were run by no test — for those `tests-for` still uses the call graph +`tests-for`, `look` and `check` now use the map; run `gitnext learn-tests` again after larger changes +``` +- `learn_warm_s`: 22.3 +- `learn_x_plain`: 2.6 +- `map_sqlite_bytes`: 32768 From 62c1bb7f86de23bfde33a2b94bb7c779f223b309 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 04:19:20 +0300 Subject: [PATCH 13/18] changelog: learn-tests for JS/TS; 138 unit tests Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 4 ++++ README.md | 2 +- README.ru.md | 2 +- eval/results/COVERAGE.md | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a14ab6b..de27da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ whose lines do not start with `*` was cut in half, and the delete refused). - **`callees`** lists a method called on an object of unknown type as "maybe", as `callers` does; `look` counts such calls instead of naming them. +- **`gitnext learn-tests` for JavaScript and TypeScript** (vitest with v8 coverage, every test file alone): + `tests-for`, `look` and `check` name the test files that really run the code — 100% of the failing ones on yaml + and zustand (the call graph: 16% and 68%), for 2.6–6.6 ordinary test runs. Says which package to install when + `@vitest/coverage-v8` is missing; jest is not supported yet. - **Rust: functions declared inside a macro call are symbols** (`quickcheck! { fn prop(..) -> bool { .. } }`, `proptest!`, a local `macro_rules!` that writes `#[test]`). In indexmap 43 of 192 tests were outside the graph and the coverage map, now 4; `why` put the function at fault first in 14 of 20 seeded Rust bugs, now in 16–18, and diff --git a/README.md b/README.md index d6b0715..71b2baa 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Most of an agent session is the system prompt re-sent on every step, so the win ### What is checked, and what is not good yet -- 127 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 138 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` finds 19% of the failing test files (it was 3%); the coverage map exists for Python, Go and Rust only. - A rename breaks the build or the tests in 9% of cases — mostly dynamic access and names shared by unrelated types. - A 4B model rarely fixes a seeded bug yet: 3 of 12 on click. diff --git a/README.ru.md b/README.ru.md index 7452916..f7ecde4 100644 --- a/README.ru.md +++ b/README.ru.md @@ -58,7 +58,7 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он ### Что проверено и что пока плохо -- 127 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- 138 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). - TypeScript: `tests-for` находит 19% падающих файлов тестов (было 3%); карта покрытия есть только для Python, Go и Rust. - Переименование ломает сборку или тесты в 9% случаев — в основном динамический доступ и одинаковые имена у несвязанных типов. - Модель на 4B пока редко чинит подсаженную ошибку: 3 из 12 на click. diff --git a/eval/results/COVERAGE.md b/eval/results/COVERAGE.md index af18517..86bdbce 100644 --- a/eval/results/COVERAGE.md +++ b/eval/results/COVERAGE.md @@ -17,7 +17,7 @@ that the answer is right. | `why`, `check ` | place of the function at fault for a seeded bug (`why_rank.py`, `why-rank.md`) | with the learned map the culprit is first in 81% of 57 seeded bugs (Python 90%, Go 82%, Rust 70%); from the call graph alone in 18% | | the tools as a small model uses them | `agent.py`, Qwen 3.5 4B: find / callers / rename / fix | see the agent runs in `HISTORY.md` | -## 2. Unit tests (`cargo test --workspace`, 127 tests) +## 2. Unit tests (`cargo test --workspace`, 138 tests) | function | tests | |---|---| From f1400f4b76d2662d498a1e67358a820cdb0d5664 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 04:31:18 +0300 Subject: [PATCH 14/18] eval: learn-tests prototype for Java (JaCoCo), three ways to take the map eval/learn_tests_java.py maps test methods to the functions they run with the JaCoCo agent (0.8.14), scored by test method name as the eval scores Java, and also by test class. Driver: learn_tests_go.measure. - per-test: one Maven run; a JUnit Platform listener (JUnit 4 runs through vintage under surefire 3) takes the agent's data when a test starts and ends. What a test class runs outside its tests (static blocks, BeforeAll) goes to all its tests; what initializing a library class runs goes to every test that runs a line of its file. - per-test-fork: the same, a JVM per test class (-DreuseForks=false). - per-class: one Maven run per test class (-Dtest=...). All 33 (jsoup) / 34 (json-java) functions, by method: recall / precision / fail among the first 10: tests-for 29 / 64 / 66 71 / 48 / 61 per-test 94 / 94 / 94 99 / 94 / 95 per-class 100 / 36 / 33 100 / 21 / 32 Map cost against a plain run: per-test 3.5x / 2.1x (50 s / 10 s), per-test-fork 8.4x / 4.1x, per-class 27x / 16x. per-test misses tests that fail through state a JVM builds once (jsoup: Normalizer.lowerCase 18% of 1689); by test class it finds 98% / 100%, the fork 100% / 100%. The build is changed only in a copy of pom.xml, removed after the run. Co-Authored-By: Claude Opus 5.5 --- eval/learn_tests_java.py | 633 ++ eval/results/learn-tests-json-java.json | 9061 +++++++++++++++++++++++ eval/results/learn-tests-json-java.md | 318 + eval/results/learn-tests-jsoup.json | 8979 ++++++++++++++++++++++ eval/results/learn-tests-jsoup.md | 315 + 5 files changed, 19306 insertions(+) create mode 100644 eval/learn_tests_java.py create mode 100644 eval/results/learn-tests-json-java.json create mode 100644 eval/results/learn-tests-json-java.md create mode 100644 eval/results/learn-tests-jsoup.json create mode 100644 eval/results/learn-tests-jsoup.md diff --git a/eval/learn_tests_java.py b/eval/learn_tests_java.py new file mode 100644 index 0000000..777188a --- /dev/null +++ b/eval/learn_tests_java.py @@ -0,0 +1,633 @@ +"""learn-tests prototype for Java (jsoup, json-java): does a per-test coverage map (JaCoCo) find the tests that fail +better than `gitnext tests-for`, what does it cost, and how to take it? + +The eval's truth for Java is the bare name of a test method (`level="name"`, `run.java_failed`), so the map is per +test method too. Three ways to take it, all with the JaCoCo agent: + +- `per-test` (one Maven run): a listener of the JUnit Platform (`TestExecutionListener`, `META-INF/services`; + JUnit 4 runs through the platform too, vintage engine of surefire 3) takes the agent's data, and resets it, when a test + starts and when it ends (`RT.getAgent().getExecutionData(true)`), one `.exec` per test. What runs in a test class + outside its tests (`@BeforeAll`, `@BeforeClass`, `@AfterAll`, and its static blocks: the listener initializes the + class when it starts, else they would run in its first test) is booked to every test of the class: when it + throws, they all fail. The invocations of a parameterized test are one test method. +- `per-test-fork`: the same, with a JVM of its own for every test class (`-DreuseForks=false`): what a JVM does + once (a lazily built table, a cache) shows up in the first test of every class, not only in the first one of all. +- `per-class` (`-Dtest=`, one Maven run per test class): one `.exec` per class, every method of the class is + booked with all the class ran. + +One JVM (`Analyze`, jacoco core) turns the `.exec` files into lines that ran; a line is booked to the innermost +function whose body holds it (from `run.body_start`): a class's field initializers and static blocks run when the +class loads, booking them to the function around them would book it to every test that loads the class. +The build is changed only in a copy of `pom.xml` (`.gnx-pom.xml` in the checkout, removed after the run): the +agent goes through surefire's `argLine` (jsoup sets one in its pom, `-DargLine` cannot add to it) and the listener +through `additionalClasspathElements` (a profile of jsoup sets that list, `-Dmaven.test.additionalClasspath` would +replace it). The listener, `Analyze` and `Inits` are compiled from the sources below into the work directory; +the JaCoCo jars come from the local Maven repository (`mvn dependency:get` the first time). +The driver is `learn_tests_go.measure`. + + GITNEXT=$PWD/target/release/gitnext python3 eval/learn_tests_java.py [--repos jsoup,json-java] [--n2 12] \\ + [--seed 7] [--pick-from ] [--how per-test,per-test-fork,per-class] \\ + [--owner body|line] [--render] +""" +from __future__ import annotations + +import argparse +import json +import re +import time +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +import learn_tests_go as lg # noqa: E402 the shared driver of 005 +import run # noqa: E402 the eval: same checkout, ground truth and cache + +JACOCO = "0.8.14" +M2 = Path.home() / ".m2" / "repository" +HOWS = ("per-test", "per-test-fork", "per-class") +PLATFORM = "1.14.4" # the JUnit Platform surefire 3.6 brings for JUnit 4 (vintage) + +DUMP = r""" +package gnx; + +import java.io.*; +import java.nio.file.*; + +/** The JaCoCo agent's data of every test on its own: taken, and reset, when a test starts and when it ends. */ +public final class Dump { + static final Path DIR = Paths.get(System.getProperty("gnx.dir", "gnx-exec")); + static final Object agent; + static final java.lang.reflect.Method get; + static int seq; + static PrintWriter index; + static { + try { + Object a = Class.forName("org.jacoco.agent.rt.RT").getMethod("getAgent").invoke(null); + agent = a; + get = Class.forName("org.jacoco.agent.rt.IAgent").getMethod("getExecutionData", boolean.class); + Files.createDirectories(DIR); + index = new PrintWriter(new FileWriter(DIR.resolve("index.tsv").toFile(), true), true); + } catch (Exception e) { + throw new ExceptionInInitializerError(e); + } + } + + static byte[] take() { + try { + return (byte[]) get.invoke(agent, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // a JVM per test class (`reuseForks=false`) writes to the same directory: the pid keeps the files apart + static final String JVM = Long.toString(ProcessHandle.current().pid()); + + static void write(String cls, String method, boolean failed, byte[] data) { + String n = JVM + "-" + seq++; + try { + Files.write(DIR.resolve(n + ".exec"), data); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + index.println(n + "\t" + cls + "\t" + method + "\t" + (failed ? 1 : 0)); + } + + /** A test class starts: what ran before it is no test's. Its static blocks run now, not in its first test: + * they are all of its tests'. */ + public static synchronized void classStarted(String cls) { + take(); + try { + Class.forName(cls, true, Thread.currentThread().getContextClassLoader()); + } catch (Throwable e) { + // its tests fail on their own + } + } + + /** What ran since the class started or since its last test (`@BeforeAll`, `@AfterAll`, a static block): all of its tests'. */ + public static synchronized void classPart(String cls) { + write(cls, "", false, take()); + } + + public static synchronized void started(String cls) { + classPart(cls); + } + + public static synchronized void finished(String cls, String method, boolean failed) { + write(cls, method, failed, take()); + } +} +""" + +DUMP5 = r""" +package gnx; + +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.descriptor.MethodSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; + +public final class Dump5 implements TestExecutionListener { + @Override + public void executionStarted(TestIdentifier id) { + if (id.isTest()) Dump.started(cls(id)); + else if (id.getSource().orElse(null) instanceof ClassSource) Dump.classStarted(((ClassSource) id.getSource().get()).getClassName()); + } + + static String cls(TestIdentifier id) { + Object s = id.getSource().orElse(null); + return s instanceof MethodSource ? ((MethodSource) s).getClassName() : "?"; + } + + @Override + public void executionFinished(TestIdentifier id, TestExecutionResult r) { + if (!id.isTest()) { + if (id.getSource().orElse(null) instanceof ClassSource) Dump.classPart(((ClassSource) id.getSource().get()).getClassName()); + return; + } + String cls = "?", m = id.getDisplayName(); + if (id.getSource().orElse(null) instanceof MethodSource) { + MethodSource s = (MethodSource) id.getSource().get(); + cls = s.getClassName(); + m = s.getMethodName(); + } + Dump.finished(cls, m, r.getStatus() != TestExecutionResult.Status.SUCCESSFUL); + } +} +""" + +ANALYZE = r""" +package gnx; + +import java.nio.file.*; +import java.util.*; +import org.jacoco.core.analysis.*; +import org.jacoco.core.data.*; +import org.jacoco.core.internal.data.CRC64; +import org.jacoco.core.tools.ExecFileLoader; + +/** args: classes dir, exec dir, out file. Every `.exec` → one JSON line {"i": "n", "l": {"pkg/File.java": [lines that ran]}}. */ +public final class Analyze { + public static void main(String[] a) throws Exception { + Path classes = Paths.get(a[0]), dir = Paths.get(a[1]); + // a multi-release jar holds a class twice (`META-INF/versions/11/…`): the one whose id the data names ran + Map> bytes = new HashMap<>(); + try (var w = Files.walk(classes)) { + for (Path p : (Iterable) w.filter(p -> p.toString().endsWith(".class"))::iterator) { + String rel = classes.relativize(p).toString().replaceFirst("^META-INF/versions/\\d+/", ""); + bytes.computeIfAbsent(rel.substring(0, rel.length() - 6), k -> new ArrayList<>()).add(Files.readAllBytes(p)); + } + } + List execs = new ArrayList<>(); + try (var s = Files.list(dir)) { + s.filter(p -> p.toString().endsWith(".exec")).sorted().forEach(execs::add); + } + try (var out = new java.io.PrintWriter(Files.newBufferedWriter(Paths.get(a[2])))) { + for (Path e : execs) { + ExecFileLoader l = new ExecFileLoader(); + l.load(e.toFile()); + ExecutionDataStore store = l.getExecutionDataStore(); + CoverageBuilder b = new CoverageBuilder(); + Analyzer an = new Analyzer(store, b); + for (ExecutionData d : store.getContents()) { + if (!d.hasHits()) continue; + for (byte[] c : bytes.getOrDefault(d.getName(), List.of())) { + if (CRC64.classId(c) == d.getId()) { + an.analyzeClass(c, d.getName()); + break; + } + } + } + Map> lines = new TreeMap<>(); + for (IClassCoverage cc : b.getClasses()) { + if (cc.getSourceFileName() == null) continue; + String f = (cc.getPackageName().isEmpty() ? "" : cc.getPackageName() + "/") + cc.getSourceFileName(); + for (int i = cc.getFirstLine(); i > 0 && i <= cc.getLastLine(); i++) { + int st = cc.getLine(i).getStatus(); + if (st == ICounter.FULLY_COVERED || st == ICounter.PARTLY_COVERED) + lines.computeIfAbsent(f, k -> new TreeSet<>()).add(i); + } + } + StringBuilder sb = new StringBuilder("{\"i\":\"").append(e.getFileName().toString().replace(".exec", "")).append("\",\"l\":{"); + String sep = ""; + for (var en : lines.entrySet()) { + sb.append(sep).append('"').append(en.getKey()).append("\":").append(en.getValue().toString().replace(" ", "")); + sep = ","; + } + out.println(sb.append("}}")); + } + } + } +} +""" + +INITS = r""" +package gnx; + +import java.net.*; +import java.nio.file.*; +import java.util.*; + +/** args: classes dir, out dir, classpath of the rest. Every class of the library is initialized on its own, in a + * class loader of its own (so all it needs is initialized again): `init-.exec` is what its static blocks and + * field initializers ran, `inits.tsv` names the class. */ +public final class Inits { + public static void main(String[] a) throws Exception { + Path classes = Paths.get(a[0]), out = Paths.get(a[1]); + List urls = new ArrayList<>(List.of(classes.toUri().toURL())); + for (String p : a[2].split(":")) if (!p.isEmpty()) urls.add(Paths.get(p).toUri().toURL()); + List names = new ArrayList<>(); + try (var w = Files.walk(classes)) { + w.filter(p -> p.toString().endsWith(".class") && !p.toString().contains("META-INF")) + .forEach(p -> { String r = classes.relativize(p).toString(); names.add(r.substring(0, r.length() - 6).replace('/', '.')); }); + } + Collections.sort(names); + System.setProperty("gnx.dir", out.toString()); + Dump.take(); + int n = 0; + try (var idx = new java.io.PrintWriter(Files.newBufferedWriter(out.resolve("inits.tsv")))) { + for (String c : names) { + try (URLClassLoader l = new URLClassLoader(urls.toArray(new URL[0]), ClassLoader.getPlatformClassLoader())) { + Class.forName(c, true, l); + } catch (Throwable e) { + // it needs what a test sets up: what ran until it threw still counts + } + Files.write(out.resolve("init-" + n + ".exec"), Dump.take()); + idx.println(n++ + "\t" + c); + } + } + } +} +""" + + +def jar(group: str, art: str, ver: str, cls: str = "") -> Path: + """A jar from the local Maven repository, fetched once (needs the network the first time).""" + p = M2 / group.replace(".", "/") / art / ver / f"{art}-{ver}{'-' + cls if cls else ''}.jar" + if not p.exists(): + code, out = run.sh(f"mvn -q -B dependency:get -Dartifact={group}:{art}:{ver}:jar{':' + cls if cls else ''}", + Path.cwd(), timeout=900) + if code or not p.exists(): + raise RuntimeError(f"no {p.name}: {out[-400:]}") + return p + + +def javac(work: Path, name: str, sources: dict[str, str], cp: list[Path]) -> Path: + src, out = work / name / "src", work / name / "classes" + for f, text in sources.items(): + (src / "gnx").mkdir(parents=True, exist_ok=True) + (src / "gnx" / f"{f}.java").write_text(text) + out.mkdir(parents=True, exist_ok=True) + code, o = run.sh(f"javac -nowarn --release 11 {'-cp ' + ':'.join(map(str, cp)) if cp else ''} -d {out} {src}/gnx/*.java", work) + if code: + raise RuntimeError(f"javac {name}: {o[-800:]}") + return out + + +def test_classpath(root: Path, work: Path) -> list[Path]: + f = work / "test-cp.txt" + code, o = run.sh(f"mvn -q -B -o dependency:build-classpath -Dmdep.includeScope=test -Dmdep.outputFile={f}", root, + timeout=900) + if code: + raise RuntimeError(f"test classpath: {o[-400:]}") + return [Path(x) for x in f.read_text().strip().split(":") if x] + + +def helpers(root: Path, work: Path) -> dict: + """The agent, the listener (on the test classpath through `${gnx.cp}`) and `Analyze`. + Surefire 3 runs JUnit 4 through the JUnit Platform too (vintage), so one listener of the platform serves both.""" + cp = test_classpath(root, work) + m = next((re.match(r"junit-platform-engine-([\d.]+)\.jar$", p.name) for p in cp + if p.name.startswith("junit-platform-engine-")), None) + ver = m.group(1) if m else PLATFORM + plat = [jar("org.junit.platform", a, ver) for a in ("junit-platform-launcher", "junit-platform-engine", "junit-platform-commons")] + listener = javac(work, "dump", {"Dump": DUMP, "Dump5": DUMP5}, cp + plat + [jar("org.opentest4j", "opentest4j", "1.3.0")]) + svc = listener / "META-INF" / "services" + svc.mkdir(parents=True, exist_ok=True) + (svc / "org.junit.platform.launcher.TestExecutionListener").write_text("gnx.Dump5\n") + core = jar("org.jacoco", "org.jacoco.core", JACOCO) + asm = [jar("org.ow2.asm", a, "9.9") for a in ("asm", "asm-commons", "asm-tree")] + analyze = javac(work, "analyze", {"Analyze": ANALYZE}, [core, *asm]) + inits = javac(work, "inits", {"Dump": DUMP, "Inits": INITS}, []) + return {"agent": jar("org.jacoco", "org.jacoco.agent", JACOCO, "runtime"), "listener": listener, + "junit": 5 if m else 4, "analyze_cp": [analyze, core, *asm], "inits": inits, "cp": cp} + + +def gnx_pom(root: Path) -> str: + """A copy of pom.xml whose surefire takes `${gnx.argLine}` (the agent) and `${gnx.cp}` (the listener).""" + p = (root / "pom.xml").read_text() + el = "${gnx.cp}" + m = re.search(r"maven-surefire-plugin.*?", p, re.S) + if m and "" in m.group(0): + blk = m.group(0) + if "" in blk: + blk = re.sub(r"(.*?)", lambda x: f"{x.group(1)} ${{gnx.argLine}}", + blk, count=1, flags=re.S) + else: + blk = blk.replace("", "${gnx.argLine}", 1) + if "" not in blk: + blk = blk.replace("", "", 1) + p = p[:m.start()] + blk + p[m.end():] + else: + plug = ("org.apache.maven.pluginsmaven-surefire-plugin" + "${gnx.argLine}" + "") + i = p.index("", p.index("")) + len("") + p = p[:i] + plug + p[i:] + # every list of extra classpath elements (a profile's too: it replaces the main one) gets the listener + p = p.replace("", "" + el) + (root / ".gnx-pom.xml").write_text(p) + return ".gnx-pom.xml" + + +def package_of(root: Path) -> str: + """The project's top package (`org.jsoup`): what the agent instruments.""" + pk = set() + for f in (root / "src" / "main" / "java").rglob("*.java"): + m = re.search(r"^package\s+([\w.]+)\s*;", f.read_text(errors="replace"), re.M) + if m: + pk.add(m.group(1)) + parts = [x.split(".") for x in pk] + pre = [] + for col in zip(*parts): + if len(set(col)) > 1: + break + pre.append(col[0]) + return ".".join(pre) or "*" + + +def analyze(h: dict, root: Path, d: Path) -> tuple[dict[str, dict[str, set[int]]], float]: + """`.exec` in d → n -> file (relative to the repository) -> lines that ran.""" + t = time.time() + out = d / "lines.jsonl" + code, o = run.sh(f"java -cp {':'.join(map(str, h['analyze_cp']))} gnx.Analyze {root}/target/classes {d} {out}", root, + timeout=3600) + if code: + raise RuntimeError(f"Analyze: {o[-800:]}") + rel: dict[str, str] = {} + for f in (root / "src" / "main").rglob("*.java"): + rel.setdefault(str(f.relative_to(root)).split("/java/", 1)[-1].split("/java11/", 1)[-1], str(f.relative_to(root))) + res = {} + for line in out.read_text().splitlines(): + j = json.loads(line) + res[j["i"]] = {rel.get(f, f): set(ls) for f, ls in j["l"].items()} + return res, time.time() - t + + +def bytes_of(d: Path) -> int: + return sum(p.stat().st_size for p in d.glob("*.exec")) + + +def per_test(root: Path, h: dict, work: Path, pkg: str, fork: bool = False) -> tuple[dict, dict]: + d = work / "exec-test" + d.mkdir(parents=True, exist_ok=True) + pom = gnx_pom(root) + t = time.time() + code, out = run.sh(f"mvn -B -o -f {pom} test -Dsurefire.printSummary=true " + f"'-Dgnx.argLine=-javaagent:{h['agent']}=output=none,includes={pkg}.*,jmx=false -Dgnx.dir={d}' " + f"-Dgnx.cp={h['listener']} {'-DreuseForks=false' if fork else ''} > {work}/per-test.log 2>&1", root, timeout=7200) + mvn_s = time.time() - t + (root / pom).unlink(missing_ok=True) + lines, an_s = analyze(h, root, d) + tests: dict = {} + shared: dict[str, dict[str, set[int]]] = {} # class -> what its setup and teardown ran + for row in (d / "index.tsv").read_text().splitlines(): + n, cls, m, failed = row.split("\t") + if not m: + for f, ls in lines.get(n, {}).items(): + shared.setdefault(cls, {}).setdefault(f, set()).update(ls) + continue + t = tests.setdefault(f"{cls}#{m}", {"name": m, "failed": False, "lines": {}}) + t["failed"] |= failed == "1" + for f, ls in lines.get(n, {}).items(): + t["lines"].setdefault(f, set()).update(ls) + for tid, t in tests.items(): + for f, ls in shared.get(tid.split("#")[0], {}).items(): + t["lines"].setdefault(f, set()).update(ls) + init_s, by_file = class_inits(root, h, work, pkg) + if INIT: + for t in tests.values(): + for f in list(t["lines"]): + for g, ls in by_file.get(f, {}).items(): + t["lines"].setdefault(g, set()).update(ls) + return tests, {"how": "per-test-fork" if fork else "per-test", "class_inits_s": round(init_s, 1), "class_inits_booked": INIT, + "files_with_init_lines": len(by_file), "mvn_s": round(mvn_s, 1), "analyze_s": round(an_s, 1), "map_s": round(mvn_s + an_s + init_s, 1), + "exec_bytes": bytes_of(d), "dumps": len(lines), "test_methods": len(tests), + "classes": len({k.split("#")[0] for k in tests}), "classes_with_setup_lines": len([c for c, v in shared.items() if v]), + "run_failed": bool(code)} + + +INIT = True # book what a library class's initialization runs to every test that runs a line of its file + + +def class_inits(root: Path, h: dict, work: Path, pkg: str) -> tuple[float, dict[str, dict[str, set[int]]]]: + """A library class's static blocks run once a JVM, in whichever test touches it first: in the others a function + they call never shows up, though it throwing fails them all. So: what initializing each class on its own runs, + by its file (relative) → file → lines.""" + d = work / "exec-init" + d.mkdir(parents=True, exist_ok=True) + t = time.time() + cp = ":".join(map(str, [h["inits"], *h["cp"]])) + rest = ":".join(str(p) for p in h["cp"] if p.suffix == ".jar") + code, o = run.sh(f"java -javaagent:{h['agent']}=output=none,includes={pkg}.*,jmx=false -cp {cp} gnx.Inits " + f"{root}/target/classes {d} {rest} > {work}/inits.log 2>&1", root, timeout=1800) + lines, an_s = analyze(h, root, d) + names = dict(r.split("\t") for r in (d / "inits.tsv").read_text().splitlines()) if (d / "inits.tsv").exists() else {} + rel = {} + for f in (root / "src" / "main").rglob("*.java"): + rel.setdefault(str(f.relative_to(root)).split("/java/", 1)[-1].split("/java11/", 1)[-1], str(f.relative_to(root))) + by_file: dict[str, dict[str, set[int]]] = {} + for n, c in names.items(): + src = rel.get(c.split("$")[0].replace(".", "/") + ".java") + if not src: + continue + for g, ls in lines.get(f"init-{n}", {}).items(): + by_file.setdefault(src, {}).setdefault(g, set()).update(ls) + return time.time() - t, by_file + + +def surefire_cases(root: Path) -> dict[str, dict[str, bool]]: + """class -> method -> failed, from the XML reports of the last surefire run.""" + out: dict[str, dict[str, bool]] = {} + for f in (root / "target" / "surefire-reports").glob("TEST-*.xml"): + try: + x = ET.parse(f).getroot() + except ET.ParseError: + continue + for c in x.iter("testcase"): + m = re.sub(r"[\[(].*$", "", c.get("name", "")) + bad = c.find("failure") is not None or c.find("error") is not None + cl = out.setdefault(c.get("classname", ""), {}) + cl[m] = cl.get(m, False) or bad + return out + + +def per_class(root: Path, h: dict, work: Path, pkg: str) -> tuple[dict, dict]: + d = work / "exec-class" + d.mkdir(parents=True, exist_ok=True) + classes = sorted(surefire_cases(root)) # the plain run of `measure` left its reports + empty = work / "empty" + empty.mkdir(exist_ok=True) + pom = gnx_pom(root) + t = time.time() + cases: dict[str, dict[str, bool]] = {} + for i, c in enumerate(classes): + run.sh(f"rm -rf target/surefire-reports; mvn -q -B -o -f {pom} test -Dtest={c} -Dsurefire.failIfNoSpecifiedTests=false " + f"'-Dgnx.argLine=-javaagent:{h['agent']}=destfile={d}/{i:04d}.exec,includes={pkg}.*,jmx=false' " + f"-Dgnx.cp={empty} > {work}/per-class.log 2>&1", root, timeout=1800) + cases[f"{i:04d}"] = surefire_cases(root).get(c, {}) + mvn_s = time.time() - t + (root / pom).unlink(missing_ok=True) + lines, an_s = analyze(h, root, d) + tests: dict = {} + for i, c in enumerate(classes): + ls = lines.get(f"{i:04d}", {}) + for m, failed in cases[f"{i:04d}"].items(): + tests[f"{c}#{m}"] = {"name": m, "failed": failed, "lines": ls} + return tests, {"how": "per-class", "mvn_s": round(mvn_s, 1), "analyze_s": round(an_s, 1), "map_s": round(mvn_s + an_s, 1), + "exec_bytes": bytes_of(d), "dumps": len(lines), "test_methods": len(tests), "classes": len(classes)} + + +def body_owner(root: Path, syms: list[dict]) -> dict[str, list]: + """file -> list indexed by line: the innermost function whose body holds that line (or None).""" + by_file: dict[str, list[dict]] = {} + for s in lg.fn_syms(syms): + by_file.setdefault(s["file"], []).append(s) + out = {} + for f, ss in by_file.items(): + arr = [None] * (max(s["end_line"] for s in ss) + 2) + for s in sorted(ss, key=lambda s: s["start_line"] - s["end_line"]): # widest first, inner ones overwrite + # a one-line method (`int f() { return x; }`) has no line ending in `{`: all of it is its body + b = run.body_start(root, s, "java") or s["start_line"] + for i in range(b, s["end_line"] + 1): + arr[i] = s["qualified"] + out[f] = arr + return out + + +CLASS_OF: dict[str, list[str]] = {} # bare test name -> the test classes with a method of that name +SCORE = lg.lt.score + + +def to_classes(names) -> list[str]: + return list(dict.fromkeys(c for n in names for c in CLASS_OF.get(n, ()))) + + +def score_both(order: list[str], truth: set[str]) -> dict: + """The eval's score by method name, and the same by test class: did the class with a failing test come up.""" + d = SCORE(order, truth) + d["by_class"] = SCORE(to_classes(order), set(to_classes(truth))) + return d + + +def collector(how: str, owner: str): + def collect(root: Path, syms: list[dict], work: Path) -> tuple[dict, dict]: + h = helpers(root, work) + pkg = package_of(root) + if how == "per-class": + tests, cost = per_class(root, h, work, pkg) + else: + tests, cost = per_test(root, h, work, pkg, fork=how == "per-test-fork") + CLASS_OF.clear() + for tid in sorted(tests): + CLASS_OF.setdefault(tests[tid]["name"], []).append(tid.split("#")[0].rsplit(".", 1)[-1]) + cost["names_in_several_classes"] = sum(1 for v in CLASS_OF.values() if len(v) > 1) + lg.lt.score = score_both + cost.update(junit=h["junit"], package=pkg, owner=owner) + if owner == "body": + ob = body_owner(root, syms) + lg.line_owner = lambda _syms: ob # measure books lines through line_owner: use the body-only one + else: + lg.line_owner = LINE_OWNER + return tests, cost + return collect + + +LINE_OWNER = lg.line_owner + + +def java_library(file: str) -> bool: + return file.endswith(".java") and "/src/test/" not in f"/{file}" and not file.startswith("src/test/") + + +def mean_score(res: dict, src: str, level: str, group: str = "all") -> dict: + rs = [r for r in res["functions"] if r.get("score") and (group == "all" or r["group"] == group)] + sc = [r["score"][src] if level == "method" else r["score"][src]["by_class"] for r in rs] + return {k: lg.lt.mean([x[k] for x in sc]) for k in ("recall", "precision", "top_precision", "caught")} | {"functions": len(rs)} + + +def render(by_how: dict[str, dict]) -> str: + first = next(iter(by_how.values())) + L = [f"# learn-tests на {first['repo']} (java): сводка", "", + "Все функции (pick + случайные). Полнота / точность / из 10 падают / поймано первыми 10. По методу — как истина " + "eval (имя метода теста); по классу — класс, где есть падающий метод.", "", + "| способ | единица | функций | граф (`tests-for`) | покрытие | объединение | карта, с | × обычного |", + "|---|---|---:|---|---|---|---:|---:|"] + for how, res in by_how.items(): + for level in ("method", "class"): + cells = [] + for src in lg.SOURCES: + m = mean_score(res, src, level) + cells.append(" / ".join(lg.pct(m[k]) for k in ("recall", "precision", "top_precision", "caught"))) + k = res["cost"] + L.append(f"| {how} | {'метод' if level == 'method' else 'класс'} | {mean_score(res, 'gitnext', level)['functions']} | " + + " | ".join(cells) + f" | {k['map_s']} | {k['slowdown_runs']} |") + L.append("") + for how, res in by_how.items(): + md = lg.render(res).replace("при panic в начале функции", "при throw в начале функции") + md = md.replace("# learn-tests на", f"# learn-tests ({how}) на", 1) + L.append(md) + L += ["# По классу теста", "", + "Класс засчитан, если в нём есть метод с падающим именем (имя в нескольких классах — все они).", "", + "| способ | выборка | функций | источник | полнота | точность | из 10 падают | поймано |", + "|---|---|---:|---|---:|---:|---:|---:|"] + for how, res in by_how.items(): + for g in ("pick", "random", "all"): + rs = [r for r in res["functions"] if r.get("score") and (g == "all" or r["group"] == g)] + for src in lg.SOURCES: + m = {k: lg.lt.mean([r["score"][src]["by_class"][k] for r in rs]) for k in + ("recall", "precision", "top_precision", "caught")} + L.append(f"| {how} | {g} | {len(rs)} | {src} | {lg.pct(m['recall'])} | {lg.pct(m['precision'])} | " + f"{lg.pct(m['top_precision'])} | {lg.pct(m['caught'])} |") + L += ["", "# Цена", "", "| | " + " | ".join(by_how) + " |", "|---|" + "---:|" * len(by_how)] + keys = ["plain_s", "mvn_s", "analyze_s", "map_s", "slowdown_runs", "exec_bytes", "map_sqlite_bytes", "dumps", + "test_methods", "classes", "names_in_several_classes", "map_pairs", "map_functions", "functions_with_body", "functions_never_run"] + for k in keys: + L.append(f"| {k} | " + " | ".join(str(r["cost"].get(k, "")) for r in by_how.values()) + " |") + L.append(f"\n{first['repo']} {first['commit']}\n") + return "\n".join(L) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repos", default="jsoup,json-java") + ap.add_argument("--n2", type=int, default=12, help="random functions per repo, besides the cached ones") + ap.add_argument("--seed", type=int, default=7) + ap.add_argument("--repeat", type=int, default=3, help="timed plain test runs") + ap.add_argument("--pick-from", help="truth.json as the eval left it: its functions are the `pick` group") + ap.add_argument("--how", default=",".join(HOWS), help="comma-separated: per-test, per-test-fork, per-class") + ap.add_argument("--render", action="store_true", help="only rewrite the .md files from the .json ones") + ap.add_argument("--owner", default="body", choices=("body", "line"), + help="a line belongs to the function whose body holds it, or to the one holding it at all") + a = ap.parse_args() + repos = {r["name"]: r for r in json.loads((run.HERE / "repos.json").read_text())} + plain = run.LANG["java"]["tests"] + for name in a.repos.split(","): + out = run.HERE / "results" / f"learn-tests-{name}" + suffix = "" if a.owner == "body" else f".{a.owner}" + # a way measured again replaces its old result, the other one stays + old = Path(f"{out}{suffix}.json") + by_how = json.loads(old.read_text()) if old.exists() else {} + for how in [] if a.render else a.how.split(","): + by_how[how] = lg.measure(repos[name], collector(how, a.owner), plain, java_library, a) + by_how = {h: by_how[h] for h in HOWS if h in by_how} + Path(f"{out}{suffix}.json").write_text(json.dumps(by_how, ensure_ascii=False, indent=1)) + md = render(by_how) + Path(f"{out}{suffix}.md").write_text(md) + print("\n" + md, flush=True) + + +if __name__ == "__main__": + main() diff --git a/eval/results/learn-tests-json-java.json b/eval/results/learn-tests-json-java.json new file mode 100644 index 0000000..78f6a14 --- /dev/null +++ b/eval/results/learn-tests-json-java.json @@ -0,0 +1,9061 @@ +{ + "per-test": { + "repo": "json-java", + "lang": "java", + "commit": "2cd0a8c2112a", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::json::JSONObject::numberToString", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 46, + "truth_doctests": 0, + "predicted_gitnext": 16, + "predicted_coverage": 47, + "missed_by_coverage": [], + "covered_not_failing": [ + "testSharedKeysParseUnchanged" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1, + "top_coverage": [ + "parsingErrorNumberToString", + "jsonObjectNumberToString", + "writeValues", + "jsonObjectByObjectAndNames", + "objectArrayVsIsArray", + "jsonObjectByMap", + "valuesToString", + "jsonObjectByMapWithNullValue", + "TestLenientCommas", + "shouldHandleNumericArray" + ], + "score": { + "gitnext": { + "recall": 0.1956521739130435, + "precision": 0.5625, + "predicted": 16, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 0.5555555555555556, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9787234042553191, + "predicted": 47, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8518518518518519, + "predicted": 54, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOnce", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 173, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectPutOnceNull", + "parsingErrorPutOnceDuplicateKey", + "enumPut", + "enumToString", + "jsonObjectByNames" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.028901734104046242, + "predicted": 173, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.028901734104046242, + "predicted": 173, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOpt", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectFromEnumWithNames", + "jsonObjectByObjectAndNames", + "enumToString" + ], + "score": { + "gitnext": { + "recall": 0.6666666666666666, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.017341040462427744, + "predicted": 173, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::toMap", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 2, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "toList", + "toMap" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::getMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 224, + "truth_doctests": 0, + "predicted_gitnext": 60, + "predicted_coverage": 243, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "testMaxSharedKeysConfig", + "parsingErrorNoColon", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenUnbalancedQuotesObject_testStrictModeFalse_shouldThrowJsonException", + "jsonObjectParseUTF8EncodingAssertExceptionMessage", + "parsingErrorKeyContainsCurlyBrace", + "parsingErrorKeyContainsSquareBrace", + "givenNonCompliantQuotesObject_testStrictModeTrue_shouldThrowJsonExceptionWithConcreteErrorDescription" + ], + "score": { + "gitnext": { + "recall": 0.16517857142857142, + "precision": 0.6166666666666667, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955357142857143, + "precision": 0.9176954732510288, + "predicted": 243, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955357142857143, + "precision": 0.8446969696969697, + "predicted": 264, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::isStrictMode", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 268, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 287, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "strictModeIsCloned", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "testPutNullMap", + "notEnoughHTTPException", + "testPutNullCollection" + ], + "score": { + "gitnext": { + "recall": 0.05223880597014925, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996268656716418, + "precision": 0.9303135888501742, + "predicted": 287, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996268656716418, + "precision": 0.9303135888501742, + "predicted": 287, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONPointer::queryFrom", + "file": "src/main/java/org/json/JSONPointer.java", + "truth": 70, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 70, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "queryWithNoResult", + "optQueryWithNoResult", + "jsonObjectFromEnumWithNames", + "jsonObjectByResourceBundle", + "jsonObjectToStringSuppressWarningOnCastToMap", + "jsonObjectByObjectAndNames", + "objectArrayVsIsArray", + "simpleCookieList", + "simpleCookieListWithDelimiter", + "jsonObjectByMap" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9859154929577465, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9859154929577465, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::getJsonParserConfiguration", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 290, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "emptyStr", + "parsingErrorInitialCurlyBrace", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "unclosedArray", + "parsingErrorTrailingCurlyBrace" + ], + "score": { + "gitnext": { + "recall": 0.01845018450184502, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::more", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 136, + "truth_doctests": 0, + "predicted_gitnext": 144, + "predicted_coverage": 136, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "emptyStringCookieList", + "shouldHandleEmptyXML", + "emptyXMLException", + "testNextCDATA", + "testNextBackComboWithNewLines", + "shouldHandleNonXML", + "nonXMLException", + "malFormedCookieListException", + "testMalformedXMLThrowsJSONExceptionNotClassCast", + "testToJSONObjectTypeMismatch" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9444444444444444, + "predicted": 144, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 136, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9444444444444444, + "predicted": 144, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextClean", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 290, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "emptyStr", + "parsingErrorInitialCurlyBrace", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "unclosedArray", + "parsingErrorTrailingCurlyBrace" + ], + "score": { + "gitnext": { + "recall": 0.02214022140221402, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextString", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 235, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 254, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "invalidEscapeSequence", + "parsingErrorNoColon", + "unclosedArray2", + "unclosedArray3", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenInvalidInputArrayWithNumericStrings_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "givenInvalidInputObject_testStrictModeTrue_JSONArrayUsingJSONTokener_shouldThrowJSONException", + "givenInvalidInputObject_testStrictModeTrue_JSONArrayUsingString_shouldThrowJSONException" + ], + "score": { + "gitnext": { + "recall": 0.02127659574468085, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9957446808510638, + "precision": 0.9212598425196851, + "predicted": 254, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9957446808510638, + "precision": 0.9212598425196851, + "predicted": 254, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::getMaxNumberLength", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 82, + "truth_doctests": 0, + "predicted_gitnext": 138, + "predicted_coverage": 179, + "missed_by_coverage": [], + "covered_not_failing": [ + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayIndexFailure", + "arrayIndexQuery", + "builderPreservesLiteralTokens", + "builderTest", + "contentOperations", + "emptyPointer", + "emptyPointerToString", + "escapeSequencesInObjectKeys", + "escapedTildeTokensSelectLiteralKeys" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 97, + "top_coverage": [ + "stringToValueNumbersTest", + "testOptBigIntegerVariousTypes", + "jsonObjectOptBigInteger", + "unquotedText", + "givenInvalidInputArray_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "remove", + "givenValidNestedArray_testStrictModeTrue_shouldNotThrowJsonException", + "testJSONArrayConstructor", + "test_strictModeWithInappropriateKey", + "testJSONArrayPutAll" + ], + "score": { + "gitnext": { + "recall": 0.32926829268292684, + "precision": 0.1956521739130435, + "predicted": 138, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.8, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.4581005586592179, + "predicted": 179, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.3082706766917293, + "predicted": 266, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::isKeepStrings", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 34, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 34, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "maxNestingDepthIsCloned", + "test_keepStringBehavior", + "noCloseEndTagException", + "testValidXMLStillWorks", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseStartTagException", + "testValidXMLToJSONObjectStillWorks", + "invalidBangInTagException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XML::noSpace", + "file": "src/main/java/org/json/XML.java", + "truth": 12, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 12, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testJSONMLToStringEscapesLiteralNewlines", + "spaceInTagException", + "emptyTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility3", + "commentsInXML", + "testToJSONArray_reversibility2", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONObject", + "toJSONArray" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getForceList", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 64, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 64, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleNoCloseStartTag", + "testValidWhitespaceNumericEntityToJSONObject", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testMaxNumberLengthXML", + "testRejectsC1ControlCharacterReference", + "testValidDecimalEntity", + "testValidHexEntity", + "testValidUppercaseHexEntity" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 64, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getXsiTypeMap", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testXSITypeMapNotModifiable", + "shouldHandleInvalidSlashInTag", + "shouldHandleNoCloseStartTag", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testToJsonWithNullWhenNilConversionDisabled", + "shouldHandleNoStartTag", + "testToJSONArray_reversibility", + "testToJsonWithTypeWhenTypeConversionEnabled" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.27848101265822783, + "predicted": 79, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.27848101265822783, + "predicted": 79, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::isConvertNilAttributeToNull", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleInvalidSlashInTag", + "shouldHandleNoCloseStartTag", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testToJsonWithNullWhenNilConversionDisabled", + "testToJsonWithNullWhenNilConversionEnabled", + "shouldHandleNoStartTag", + "testToJSONArray_reversibility", + "testToJsonWithTypeWhenTypeConversionEnabled" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.28205128205128205, + "predicted": 78, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.28205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::shouldTrimWhiteSpace", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 93, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 93, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testNextContentWithText", + "testIssue484InfinteLoop1", + "testIssue484InfinteLoop2", + "testMaxNestingDepthIsRespectedWithValidXML", + "noCloseEndTagException", + "shouldHandleInvalidSlashInTag", + "testValidXMLStillWorks", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7815126050420168, + "predicted": 119, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 93, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7815126050420168, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::withKeepBooleanAsString", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 2, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "test_keepStringBehavior", + "testToJSONArray_jsonOutput_withKeepBooleanAsString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.02531645569620253, + "predicted": 79, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.02531645569620253, + "predicted": 79, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextCDATA", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 118, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testNextCDATA", + "shouldPreserveCdataWhenKeepingStrings", + "shouldHandleCommentsInXML", + "toJSONObjectToJSONArray", + "contentOperations" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.0423728813559322, + "predicted": 118, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0423728813559322, + "predicted": 118, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextMeta", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 6, + "truth_doctests": 0, + "predicted_gitnext": 117, + "predicted_coverage": 6, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testIssue484InfinteLoop1", + "testIssue484InfinteLoop2", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.05128205128205128, + "predicted": 117, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.05128205128205128, + "predicted": 117, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::skipPast", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 88, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 88, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testSkipPast", + "testNextCDATA", + "shouldHandleNonXML", + "testEmptyNumericEntityThrowsJSONException", + "testEmptyHexEntityThrowsJSONException", + "testInvalidDecimalEntityThrowsJSONException", + "testInvalidHexEntityThrowsJSONException", + "testOutOfRangeHexEntityInAttributeThrowsJSONException", + "testOutOfRangeDecimalEntityThrowsJSONException", + "testOutOfRangeHexEntityThrowsJSONException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7394957983193278, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 88, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7394957983193278, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optJSONObject", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectByResourceBundle", + "jsonObjectOptNoKey", + "jsonObjectOptDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONParserConfiguration::withMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testMaxSharedKeysConfig", + "testSharedKeysLimit", + "testSharedKeysDisabled", + "testSharedKeysUnlimited", + "testSharedKeysParseUnchanged" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XMLTokener::parseHexEntity", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 16, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testIssue537CaseSensitiveHexUnEscapeDirect", + "testValidWhitespaceNumericEntityUnescape", + "testEmptyHexEntityThrowsJSONException", + "testInvalidHexEntityThrowsJSONException", + "testOutOfRangeHexEntityInAttributeThrowsJSONException", + "testOutOfRangeHexEntityThrowsJSONException", + "testSurrogateHexEntityThrowsJSONException", + "testValidHexEntity", + "testValidUppercaseHexEntity", + "shouldPreserveEscapedInvalidCharacterReferences" + ], + "score": { + "gitnext": { + "recall": 0.1875, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.6666666666666666, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 16, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::checkKeyForStrictMode", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 223, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 242, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "parsingErrorNoColon", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenUnbalancedQuotesObject_testStrictModeFalse_shouldThrowJsonException", + "jsonObjectParseUTF8EncodingAssertExceptionMessage", + "parsingErrorKeyContainsCurlyBrace", + "parsingErrorKeyContainsSquareBrace", + "givenNonCompliantQuotesObject_testStrictModeTrue_shouldThrowJsonExceptionWithConcreteErrorDescription", + "equals" + ], + "score": { + "gitnext": { + "recall": 0.29596412556053814, + "precision": 0.38596491228070173, + "predicted": 171, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955156950672646, + "precision": 0.9173553719008265, + "predicted": 242, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955156950672646, + "precision": 0.6397694524495677, + "predicted": 347, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONML::toString", + "file": "src/main/java/org/json/JSONML.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 43, + "predicted_coverage": 14, + "missed_by_coverage": [], + "covered_not_failing": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "nullJSONXMLException", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray", + "spaceInTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility2", + "testToJSONArray_reversibility3", + "toJSONArray" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 10, + "top_coverage": [ + "nullJSONXMLException", + "emptyJSONXMLException", + "testJSONMLToStringEscapesLiteralNewlines", + "spaceInTagException", + "emptyTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility3", + "commentsInXML", + "testToJSONArray_reversibility2", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.09302325581395349, + "predicted": 43, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 14, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.09302325581395349, + "predicted": 43, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONWriter::array", + "file": "src/main/java/org/json/JSONWriter.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 86, + "predicted_coverage": 4, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "missplacedEndObjectException", + "missplacedArrayException", + "simpleArrayString", + "complexObjectString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.046511627906976744, + "predicted": 86, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.046511627906976744, + "predicted": 86, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::CDL::rowToJSONObject", + "file": "src/main/java/org/json/CDL.java", + "truth": 13, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 10, + "missed_by_coverage": [ + "badEscapedQuote", + "unbalancedEscapedQuote", + "unbalancedQuoteInValue" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 3, + "covered_not_failing_n": 0, + "top_coverage": [ + "onlyColumnNames", + "csvParsingCatchesLastRow", + "emptyLinesToJSONArray", + "singleEscapedQuoteMiddleString", + "singleEscapedQuote", + "textToJSONArray", + "textToJSONArrayPipeDelimited", + "jsonArrayToJSONArray", + "textToJSONArrayAndBackToString", + "textToJSONArrayAndBackToStringCustomDelimiter" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7647058823529411, + "predicted": 17, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.7692307692307693, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7647058823529411, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONArray::getJSONObject", + "file": "src/main/java/org/json/JSONArray.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 36, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonArrayByListWithNestedNullValue", + "jsonObjectByMapWithNestedNullValueAndParserConfiguration", + "checkSpecialChars", + "emptyLinesToJSONArray", + "testToJSONArrayPreservesChildNodeAttributes", + "singleEscapedQuote", + "explicitConfigurationOverridesTokenerStrictMode", + "testSharedKeysLimit", + "testSharedKeysLenientSyntax", + "testSharedKeysByDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.6111111111111112, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.6111111111111112, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XML::indent", + "file": "src/main/java/org/json/XML.java", + "truth": 27, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 27, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleNullJSONXML", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyArray", + "testEscapesLiteralWhitespaceControlCharsOnSerialization", + "shouldHandleNullNodeValue", + "shouldHandleIllegalJSONNodeNames", + "shouldHandleEmptyMultiArray", + "shouldHandleMultiArray", + "shouldHandleNonEmptyArray", + "toStringRejectsElementInjectionInKey" + ], + "score": { + "gitnext": { + "recall": 0.14814814814814814, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 27, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9310344827586207, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optEnum", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "enumAPI", + "jsonObjectOptNoKey", + "jsonObjectOptDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.01744186046511628, + "predicted": 172, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.01744186046511628, + "predicted": 172, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::getKeyNameFromMethod", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 32, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 32, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "enumsShouldNotIncludeEnumMethods", + "javaLibraryClassesShouldNotIncludeTheirMethods", + "jsonObjectParseNullFieldsWithoutParserConfiguration", + "genericBeanWithGetAndIsMethodsShouldNotBeIncluded", + "issue713BeanConstructorWithNonFiniteNumbers", + "numberSubclassesShouldNotIncludeNumberMethods", + "jsonObjectByLocaleBean", + "jsonObjectParseNullFieldsWithParserConfiguration", + "testLongRepeatObjectNotRecursive", + "testNotRecursiveEquals" + ], + "score": { + "gitnext": { + "recall": 0.75, + "precision": 0.14035087719298245, + "predicted": 171, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.25, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1787709497206704, + "predicted": 179, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::isDecimalNotation", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 87, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 156, + "missed_by_coverage": [], + "covered_not_failing": [ + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayIndexFailure", + "arrayIndexQuery", + "builderPreservesLiteralTokens", + "builderTest", + "emptyPointer", + "emptyPointerToString", + "escapeSequencesInObjectKeys", + "escapedTildeTokensSelectLiteralKeys", + "explicitConfigurationAppliesToNestedDuplicateKeys" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 69, + "top_coverage": [ + "stringToValueNumbersTest", + "testOptBigIntegerVariousTypes", + "testLongFromString", + "unquotedText", + "givenInvalidInputArray_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "remove", + "givenValidNestedArray_testStrictModeTrue_shouldNotThrowJsonException", + "testJSONArrayConstructor", + "test_strictModeWithInappropriateKey", + "testJSONArrayPutAll" + ], + "score": { + "gitnext": { + "recall": 0.28735632183908044, + "precision": 0.8064516129032258, + "predicted": 31, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.4, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5576923076923077, + "predicted": 156, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.5403726708074534, + "predicted": 161, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.703, + "precision": 0.524, + "top_precision": 0.655, + "caught": 1.0 + }, + "coverage": { + "recall": 0.999, + "precision": 0.958, + "top_precision": 0.982, + "caught": 1.0 + }, + "union": { + "recall": 0.999, + "precision": 0.54, + "top_precision": 0.832, + "caught": 1.0 + }, + "functions": 22 + }, + "random": { + "gitnext": { + "recall": 0.722, + "precision": 0.415, + "top_precision": 0.525, + "caught": 0.917 + }, + "coverage": { + "recall": 0.98, + "precision": 0.897, + "top_precision": 0.892, + "caught": 1.0 + }, + "union": { + "recall": 1.0, + "precision": 0.47, + "top_precision": 0.758, + "caught": 1.0 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.71, + "precision": 0.485, + "top_precision": 0.609, + "caught": 0.971 + }, + "coverage": { + "recall": 0.992, + "precision": 0.936, + "top_precision": 0.95, + "caught": 1.0 + }, + "union": { + "recall": 0.999, + "precision": 0.515, + "top_precision": 0.806, + "caught": 1.0 + }, + "functions": 34 + } + }, + "cost": { + "how": "per-test", + "class_inits_s": 0.5, + "class_inits_booked": true, + "files_with_init_lines": 5, + "mvn_s": 6.1, + "analyze_s": 3.8, + "map_s": 10.5, + "exec_bytes": 503756, + "dumps": 1686, + "test_methods": 634, + "classes": 22, + "classes_with_setup_lines": 1, + "run_failed": false, + "names_in_several_classes": 46, + "junit": 4, + "package": "org.json", + "owner": "body", + "repeat": 3, + "plain_s": 5.1, + "tests": 634, + "map_pairs": 18211, + "map_functions": 311, + "map_sqlite_bytes": 233472, + "functions_with_body": 510, + "functions_never_run": 16, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 2.1 + } + }, + "per-test-fork": { + "repo": "json-java", + "lang": "java", + "commit": "2cd0a8c2112a", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::json::JSONObject::numberToString", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 46, + "truth_doctests": 0, + "predicted_gitnext": 16, + "predicted_coverage": 47, + "missed_by_coverage": [], + "covered_not_failing": [ + "testSharedKeysParseUnchanged" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1, + "top_coverage": [ + "parsingErrorNumberToString", + "jsonObjectNumberToString", + "writeValues", + "jsonObjectByObjectAndNames", + "objectArrayVsIsArray", + "jsonObjectByMap", + "valuesToString", + "jsonObjectByMapWithNullValue", + "TestLenientCommas", + "shouldHandleNumericArray" + ], + "score": { + "gitnext": { + "recall": 0.1956521739130435, + "precision": 0.5625, + "predicted": 16, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 0.5555555555555556, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9787234042553191, + "predicted": 47, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8518518518518519, + "predicted": 54, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOnce", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 173, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectPutOnceNull", + "parsingErrorPutOnceDuplicateKey", + "enumPut", + "enumToString", + "jsonObjectByNames" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.028901734104046242, + "predicted": 173, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.028901734104046242, + "predicted": 173, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOpt", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectFromEnumWithNames", + "jsonObjectByObjectAndNames", + "enumToString" + ], + "score": { + "gitnext": { + "recall": 0.6666666666666666, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.017341040462427744, + "predicted": 173, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::toMap", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 2, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "toList", + "toMap" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::getMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 224, + "truth_doctests": 0, + "predicted_gitnext": 60, + "predicted_coverage": 243, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "testMaxSharedKeysConfig", + "parsingErrorNoColon", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenUnbalancedQuotesObject_testStrictModeFalse_shouldThrowJsonException", + "jsonObjectParseUTF8EncodingAssertExceptionMessage", + "parsingErrorKeyContainsCurlyBrace", + "parsingErrorKeyContainsSquareBrace", + "givenNonCompliantQuotesObject_testStrictModeTrue_shouldThrowJsonExceptionWithConcreteErrorDescription" + ], + "score": { + "gitnext": { + "recall": 0.16517857142857142, + "precision": 0.6166666666666667, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955357142857143, + "precision": 0.9176954732510288, + "predicted": 243, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955357142857143, + "precision": 0.8446969696969697, + "predicted": 264, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::isStrictMode", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 268, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 287, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "strictModeIsCloned", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "testPutNullMap", + "notEnoughHTTPException", + "testPutNullCollection" + ], + "score": { + "gitnext": { + "recall": 0.05223880597014925, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996268656716418, + "precision": 0.9303135888501742, + "predicted": 287, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996268656716418, + "precision": 0.9303135888501742, + "predicted": 287, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONPointer::queryFrom", + "file": "src/main/java/org/json/JSONPointer.java", + "truth": 70, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 70, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "queryWithNoResult", + "optQueryWithNoResult", + "jsonObjectFromEnumWithNames", + "jsonObjectByResourceBundle", + "jsonObjectToStringSuppressWarningOnCastToMap", + "jsonObjectByObjectAndNames", + "objectArrayVsIsArray", + "simpleCookieList", + "simpleCookieListWithDelimiter", + "jsonObjectByMap" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9859154929577465, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9859154929577465, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::getJsonParserConfiguration", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 290, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "emptyStr", + "parsingErrorInitialCurlyBrace", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "unclosedArray", + "parsingErrorTrailingCurlyBrace" + ], + "score": { + "gitnext": { + "recall": 0.01845018450184502, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::more", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 136, + "truth_doctests": 0, + "predicted_gitnext": 144, + "predicted_coverage": 136, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "emptyStringCookieList", + "shouldHandleEmptyXML", + "emptyXMLException", + "testNextCDATA", + "testNextBackComboWithNewLines", + "shouldHandleNonXML", + "nonXMLException", + "malFormedCookieListException", + "testMalformedXMLThrowsJSONExceptionNotClassCast", + "testToJSONObjectTypeMismatch" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9444444444444444, + "predicted": 144, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 136, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9444444444444444, + "predicted": 144, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextClean", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 290, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "emptyStr", + "parsingErrorInitialCurlyBrace", + "testPutNullBoolean", + "testPutNullDouble", + "testPutNullFloat", + "testPutNullInt", + "testPutNullLong", + "testPutNullObject", + "unclosedArray", + "parsingErrorTrailingCurlyBrace" + ], + "score": { + "gitnext": { + "recall": 0.02214022140221402, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.9310344827586207, + "predicted": 290, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextString", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 235, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 254, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "invalidEscapeSequence", + "parsingErrorNoColon", + "unclosedArray2", + "unclosedArray3", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenInvalidInputArrayWithNumericStrings_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "givenInvalidInputObject_testStrictModeTrue_JSONArrayUsingJSONTokener_shouldThrowJSONException", + "givenInvalidInputObject_testStrictModeTrue_JSONArrayUsingString_shouldThrowJSONException" + ], + "score": { + "gitnext": { + "recall": 0.02127659574468085, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9957446808510638, + "precision": 0.9212598425196851, + "predicted": 254, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9957446808510638, + "precision": 0.9212598425196851, + "predicted": 254, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::getMaxNumberLength", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 82, + "truth_doctests": 0, + "predicted_gitnext": 138, + "predicted_coverage": 179, + "missed_by_coverage": [], + "covered_not_failing": [ + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayIndexFailure", + "arrayIndexQuery", + "builderPreservesLiteralTokens", + "builderTest", + "contentOperations", + "emptyPointer", + "emptyPointerToString", + "escapeSequencesInObjectKeys", + "escapedTildeTokensSelectLiteralKeys" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 97, + "top_coverage": [ + "stringToValueNumbersTest", + "testOptBigIntegerVariousTypes", + "jsonObjectOptBigInteger", + "unquotedText", + "givenInvalidInputArray_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "remove", + "givenValidNestedArray_testStrictModeTrue_shouldNotThrowJsonException", + "testJSONArrayConstructor", + "test_strictModeWithInappropriateKey", + "testJSONArrayPutAll" + ], + "score": { + "gitnext": { + "recall": 0.32926829268292684, + "precision": 0.1956521739130435, + "predicted": 138, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.8, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.4581005586592179, + "predicted": 179, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.3082706766917293, + "predicted": 266, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::isKeepStrings", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 34, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 34, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "maxNestingDepthIsCloned", + "test_keepStringBehavior", + "noCloseEndTagException", + "testValidXMLStillWorks", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseStartTagException", + "testValidXMLToJSONObjectStillWorks", + "invalidBangInTagException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XML::noSpace", + "file": "src/main/java/org/json/XML.java", + "truth": 12, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 12, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testJSONMLToStringEscapesLiteralNewlines", + "spaceInTagException", + "emptyTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility3", + "commentsInXML", + "testToJSONArray_reversibility2", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONObject", + "toJSONArray" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getForceList", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 64, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 64, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleNoCloseStartTag", + "testValidWhitespaceNumericEntityToJSONObject", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testMaxNumberLengthXML", + "testRejectsC1ControlCharacterReference", + "testValidDecimalEntity", + "testValidHexEntity", + "testValidUppercaseHexEntity" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 64, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getXsiTypeMap", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testXSITypeMapNotModifiable", + "shouldHandleInvalidSlashInTag", + "shouldHandleNoCloseStartTag", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testToJsonWithNullWhenNilConversionDisabled", + "shouldHandleNoStartTag", + "testToJSONArray_reversibility", + "testToJsonWithTypeWhenTypeConversionEnabled" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.27848101265822783, + "predicted": 79, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.27848101265822783, + "predicted": 79, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::isConvertNilAttributeToNull", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleInvalidSlashInTag", + "shouldHandleNoCloseStartTag", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag", + "shouldHandleInvalidCDATABangInTag", + "testToJsonWithNullWhenNilConversionDisabled", + "testToJsonWithNullWhenNilConversionEnabled", + "shouldHandleNoStartTag", + "testToJSONArray_reversibility", + "testToJsonWithTypeWhenTypeConversionEnabled" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.28205128205128205, + "predicted": 78, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.28205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::shouldTrimWhiteSpace", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 93, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 93, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testNextContentWithText", + "testIssue484InfinteLoop1", + "testIssue484InfinteLoop2", + "testMaxNestingDepthIsRespectedWithValidXML", + "noCloseEndTagException", + "shouldHandleInvalidSlashInTag", + "testValidXMLStillWorks", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7815126050420168, + "predicted": 119, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 93, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7815126050420168, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::withKeepBooleanAsString", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 2, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "test_keepStringBehavior", + "testToJSONArray_jsonOutput_withKeepBooleanAsString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.02531645569620253, + "predicted": 79, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.02531645569620253, + "predicted": 79, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextCDATA", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 118, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testNextCDATA", + "shouldPreserveCdataWhenKeepingStrings", + "shouldHandleCommentsInXML", + "toJSONObjectToJSONArray", + "contentOperations" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.0423728813559322, + "predicted": 118, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0423728813559322, + "predicted": 118, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextMeta", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 6, + "truth_doctests": 0, + "predicted_gitnext": 117, + "predicted_coverage": 6, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testIssue484InfinteLoop1", + "testIssue484InfinteLoop2", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "shouldHandleInvalidBangInTag", + "shouldHandleInvalidBangNoCloseInTag" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.05128205128205128, + "predicted": 117, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.05128205128205128, + "predicted": 117, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::skipPast", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 88, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 88, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testSkipPast", + "testNextCDATA", + "shouldHandleNonXML", + "testEmptyNumericEntityThrowsJSONException", + "testEmptyHexEntityThrowsJSONException", + "testInvalidDecimalEntityThrowsJSONException", + "testInvalidHexEntityThrowsJSONException", + "testOutOfRangeHexEntityInAttributeThrowsJSONException", + "testOutOfRangeDecimalEntityThrowsJSONException", + "testOutOfRangeHexEntityThrowsJSONException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7394957983193278, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 88, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7394957983193278, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optJSONObject", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonObjectByResourceBundle", + "jsonObjectOptNoKey", + "jsonObjectOptDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONParserConfiguration::withMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 5, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testMaxSharedKeysConfig", + "testSharedKeysLimit", + "testSharedKeysDisabled", + "testSharedKeysUnlimited", + "testSharedKeysParseUnchanged" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XMLTokener::parseHexEntity", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 16, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testIssue537CaseSensitiveHexUnEscapeDirect", + "testValidWhitespaceNumericEntityUnescape", + "testEmptyHexEntityThrowsJSONException", + "testInvalidHexEntityThrowsJSONException", + "testOutOfRangeHexEntityInAttributeThrowsJSONException", + "testOutOfRangeHexEntityThrowsJSONException", + "testSurrogateHexEntityThrowsJSONException", + "testValidHexEntity", + "testValidUppercaseHexEntity", + "shouldPreserveEscapedInvalidCharacterReferences" + ], + "score": { + "gitnext": { + "recall": 0.1875, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.6666666666666666, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 16, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8, + "predicted": 20, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::checkKeyForStrictMode", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 223, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 242, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "givenInvalidInputObjects_testStrictModeTrue_shouldThrowJsonException", + "givenInvalidInput_testStrictModeTrue_shouldThrowJsonException", + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 20, + "top_coverage": [ + "parsingErrorNoColon", + "jsonObjectParseControlCharacterEOFAssertExceptionMessage", + "jsonObjectParseControlCharacterNewLineAssertExceptionMessage", + "jsonObjectParseIllegalEscapeAssertExceptionMessage", + "givenUnbalancedQuotesObject_testStrictModeFalse_shouldThrowJsonException", + "jsonObjectParseUTF8EncodingAssertExceptionMessage", + "parsingErrorKeyContainsCurlyBrace", + "parsingErrorKeyContainsSquareBrace", + "givenNonCompliantQuotesObject_testStrictModeTrue_shouldThrowJsonExceptionWithConcreteErrorDescription", + "equals" + ], + "score": { + "gitnext": { + "recall": 0.29596412556053814, + "precision": 0.38596491228070173, + "predicted": 171, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955156950672646, + "precision": 0.9173553719008265, + "predicted": 242, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955156950672646, + "precision": 0.6397694524495677, + "predicted": 347, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONML::toString", + "file": "src/main/java/org/json/JSONML.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 43, + "predicted_coverage": 14, + "missed_by_coverage": [], + "covered_not_failing": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "nullJSONXMLException", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray", + "spaceInTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility2", + "testToJSONArray_reversibility3", + "toJSONArray" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 10, + "top_coverage": [ + "nullJSONXMLException", + "emptyJSONXMLException", + "testJSONMLToStringEscapesLiteralNewlines", + "spaceInTagException", + "emptyTagException", + "testToJSONArray_reversibility", + "testToJSONArray_reversibility3", + "commentsInXML", + "testToJSONArray_reversibility2", + "shouldDecodeEntitiesOnceWhenKeepingStringsInJSONArray" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.09302325581395349, + "predicted": 43, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 14, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.09302325581395349, + "predicted": 43, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONWriter::array", + "file": "src/main/java/org/json/JSONWriter.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 86, + "predicted_coverage": 4, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "missplacedEndObjectException", + "missplacedArrayException", + "simpleArrayString", + "complexObjectString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.046511627906976744, + "predicted": 86, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.046511627906976744, + "predicted": 86, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::CDL::rowToJSONObject", + "file": "src/main/java/org/json/CDL.java", + "truth": 13, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 10, + "missed_by_coverage": [ + "badEscapedQuote", + "unbalancedEscapedQuote", + "unbalancedQuoteInValue" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 3, + "covered_not_failing_n": 0, + "top_coverage": [ + "onlyColumnNames", + "csvParsingCatchesLastRow", + "emptyLinesToJSONArray", + "singleEscapedQuoteMiddleString", + "singleEscapedQuote", + "textToJSONArray", + "textToJSONArrayPipeDelimited", + "jsonArrayToJSONArray", + "textToJSONArrayAndBackToString", + "textToJSONArrayAndBackToStringCustomDelimiter" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7647058823529411, + "predicted": 17, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.7692307692307693, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7647058823529411, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONArray::getJSONObject", + "file": "src/main/java/org/json/JSONArray.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 36, + "predicted_coverage": 22, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "jsonArrayByListWithNestedNullValue", + "jsonObjectByMapWithNestedNullValueAndParserConfiguration", + "checkSpecialChars", + "emptyLinesToJSONArray", + "testToJSONArrayPreservesChildNodeAttributes", + "singleEscapedQuote", + "explicitConfigurationOverridesTokenerStrictMode", + "testSharedKeysLimit", + "testSharedKeysLenientSyntax", + "testSharedKeysByDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.6111111111111112, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 22, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.6111111111111112, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XML::indent", + "file": "src/main/java/org/json/XML.java", + "truth": 27, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 27, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "shouldHandleNullJSONXML", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyArray", + "testEscapesLiteralWhitespaceControlCharsOnSerialization", + "shouldHandleNullNodeValue", + "shouldHandleIllegalJSONNodeNames", + "shouldHandleEmptyMultiArray", + "shouldHandleMultiArray", + "shouldHandleNonEmptyArray", + "toStringRejectsElementInjectionInKey" + ], + "score": { + "gitnext": { + "recall": 0.14814814814814814, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 27, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9310344827586207, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optEnum", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 3, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "enumAPI", + "jsonObjectOptNoKey", + "jsonObjectOptDefault" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.01744186046511628, + "predicted": 172, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.01744186046511628, + "predicted": 172, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::getKeyNameFromMethod", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 32, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 32, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "enumsShouldNotIncludeEnumMethods", + "javaLibraryClassesShouldNotIncludeTheirMethods", + "jsonObjectParseNullFieldsWithoutParserConfiguration", + "genericBeanWithGetAndIsMethodsShouldNotBeIncluded", + "issue713BeanConstructorWithNonFiniteNumbers", + "numberSubclassesShouldNotIncludeNumberMethods", + "jsonObjectByLocaleBean", + "jsonObjectParseNullFieldsWithParserConfiguration", + "testLongRepeatObjectNotRecursive", + "testNotRecursiveEquals" + ], + "score": { + "gitnext": { + "recall": 0.75, + "precision": 0.14035087719298245, + "predicted": 171, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.25, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1787709497206704, + "predicted": 179, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::isDecimalNotation", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 87, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 156, + "missed_by_coverage": [], + "covered_not_failing": [ + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayIndexFailure", + "arrayIndexQuery", + "builderPreservesLiteralTokens", + "builderTest", + "emptyPointer", + "emptyPointerToString", + "escapeSequencesInObjectKeys", + "escapedTildeTokensSelectLiteralKeys", + "explicitConfigurationAppliesToNestedDuplicateKeys" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 69, + "top_coverage": [ + "stringToValueNumbersTest", + "testOptBigIntegerVariousTypes", + "testLongFromString", + "unquotedText", + "givenInvalidInputArray_testStrictModeTrue_shouldThrowInvalidCharacterErrorMessage", + "remove", + "givenValidNestedArray_testStrictModeTrue_shouldNotThrowJsonException", + "testJSONArrayConstructor", + "test_strictModeWithInappropriateKey", + "testJSONArrayPutAll" + ], + "score": { + "gitnext": { + "recall": 0.28735632183908044, + "precision": 0.8064516129032258, + "predicted": 31, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.4, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5576923076923077, + "predicted": 156, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.5403726708074534, + "predicted": 161, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.703, + "precision": 0.524, + "top_precision": 0.655, + "caught": 1.0 + }, + "coverage": { + "recall": 0.999, + "precision": 0.958, + "top_precision": 0.982, + "caught": 1.0 + }, + "union": { + "recall": 0.999, + "precision": 0.54, + "top_precision": 0.832, + "caught": 1.0 + }, + "functions": 22 + }, + "random": { + "gitnext": { + "recall": 0.722, + "precision": 0.415, + "top_precision": 0.525, + "caught": 0.917 + }, + "coverage": { + "recall": 0.98, + "precision": 0.897, + "top_precision": 0.892, + "caught": 1.0 + }, + "union": { + "recall": 1.0, + "precision": 0.47, + "top_precision": 0.758, + "caught": 1.0 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.71, + "precision": 0.485, + "top_precision": 0.609, + "caught": 0.971 + }, + "coverage": { + "recall": 0.992, + "precision": 0.936, + "top_precision": 0.95, + "caught": 1.0 + }, + "union": { + "recall": 0.999, + "precision": 0.515, + "top_precision": 0.806, + "caught": 1.0 + }, + "functions": 34 + } + }, + "cost": { + "how": "per-test-fork", + "class_inits_s": 0.5, + "class_inits_booked": true, + "files_with_init_lines": 5, + "mvn_s": 16.6, + "analyze_s": 3.1, + "map_s": 20.2, + "exec_bytes": 504392, + "dumps": 1686, + "test_methods": 634, + "classes": 22, + "classes_with_setup_lines": 1, + "run_failed": false, + "names_in_several_classes": 46, + "junit": 4, + "package": "org.json", + "owner": "body", + "repeat": 3, + "plain_s": 4.9, + "tests": 634, + "map_pairs": 18214, + "map_functions": 311, + "map_sqlite_bytes": 233472, + "functions_with_body": 510, + "functions_never_run": 16, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 4.1 + } + }, + "per-class": { + "repo": "json-java", + "lang": "java", + "commit": "2cd0a8c2112a", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::json::JSONObject::numberToString", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 46, + "truth_doctests": 0, + "predicted_gitnext": 16, + "predicted_coverage": 388, + "missed_by_coverage": [], + "covered_not_failing": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "badObject", + "clarifyCurrentBehavior", + "commentsInXML", + "contentOperations", + "emptyJSONXMLException", + "emptyJsonObject" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 342, + "top_coverage": [ + "testEnumJSONString", + "testJSONNullStringValue", + "testJSONStringExceptionValue", + "testJSONStringValue", + "testNullStringValue", + "testStringValue", + "valuesToString", + "writeValues", + "enumAPI", + "enumPut" + ], + "score": { + "gitnext": { + "recall": 0.1956521739130435, + "precision": 0.5625, + "predicted": 16, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 0.5555555555555556, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.11855670103092783, + "predicted": 388, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.11855670103092783, + "predicted": 388, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOnce", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 173, + "predicted_coverage": 176, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "enumAPI", + "enumValueToString", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 171, + "top_coverage": [ + "enumAPI", + "enumPut", + "enumToString", + "enumValueToString", + "jsonObjectFromEnum", + "jsonObjectFromEnumWithNames", + "verifyEnumConsistency", + "wrap", + "jsonObjectPutOnceNull", + "parsingErrorPutOnceDuplicateKey" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.028901734104046242, + "predicted": 173, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.028409090909090908, + "predicted": 176, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.027932960893854747, + "predicted": 179, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::putOpt", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 176, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "enumAPI", + "enumPut", + "enumValueToString", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 173, + "top_coverage": [ + "enumAPI", + "enumPut", + "enumToString", + "enumValueToString", + "jsonObjectFromEnum", + "jsonObjectFromEnumWithNames", + "verifyEnumConsistency", + "wrap", + "bigNumberOperations", + "clarifyCurrentBehavior" + ], + "score": { + "gitnext": { + "recall": 0.6666666666666666, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.017045454545454544, + "predicted": 176, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.01675977653631285, + "predicted": 179, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONObject::toMap", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 207, + "missed_by_coverage": [], + "covered_not_failing": [ + "TestLenientCommas", + "badObject", + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "emptyStr", + "equals", + "failedGetArrayValues", + "getArrayValues" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 205, + "top_coverage": [ + "TestLenientCommas", + "badObject", + "emptyStr", + "failedGetArrayValues", + "getArrayValues", + "issue654StackOverflowInputWellFormed", + "iteratorTest", + "join", + "jsonArrayByListWithNestedNullValue", + "jsonArrayClearMethodTest" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.011627906976744186, + "predicted": 172, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.00966183574879227, + "predicted": 207, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.009523809523809525, + "predicted": 210, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::getMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 224, + "truth_doctests": 0, + "predicted_gitnext": 60, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "TestLenientCommas", + "allowNullArrayInStrictMode", + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "commentsInXML", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 305, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.16517857142857142, + "precision": 0.6166666666666667, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955357142857143, + "precision": 0.42234848484848486, + "predicted": 528, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955357142857143, + "precision": 0.42234848484848486, + "predicted": 528, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONParserConfiguration::isStrictMode", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 268, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 261, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.05223880597014925, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996268656716418, + "precision": 0.5056818181818182, + "predicted": 528, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996268656716418, + "precision": 0.5056818181818182, + "predicted": 528, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONPointer::queryFrom", + "file": "src/main/java/org/json/JSONPointer.java", + "truth": 70, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 276, + "missed_by_coverage": [], + "covered_not_failing": [ + "TestLenientCommas", + "badObject", + "bigNumberOperations", + "clarifyCurrentBehavior", + "convertCookieListWithNullValueToString", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "emptyPointerToString", + "emptyStr", + "emptyStringCookieList" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 206, + "top_coverage": [ + "convertCookieListToString", + "convertCookieListWithNullValueToString", + "convertEncodedCookieListToString", + "emptyStringCookieList", + "malFormedCookieListException", + "multiPartCookieList", + "nullCookieListException", + "simpleCookieList", + "simpleCookieListWithDelimiter", + "enumAPI" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9859154929577465, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2536231884057971, + "predicted": 276, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 8, + "top_hits": 6, + "top_n": 8, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2536231884057971, + "predicted": 276, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 8, + "top_hits": 6, + "top_n": 8, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::getJsonParserConfiguration", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 258, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.01845018450184502, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.5113636363636364, + "predicted": 528, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.5113636363636364, + "predicted": 528, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::more", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 136, + "truth_doctests": 0, + "predicted_gitnext": 144, + "predicted_coverage": 353, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "convertCookieListWithNullValueToString", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "emptyNameCookieException", + "emptyStringCookieException", + "emptyTagException", + "equals", + "escapeString" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 217, + "top_coverage": [ + "testNextCDATA", + "testNextContentWithTag", + "testNextContentWithText", + "testNextEntityKnown", + "testNextEntityUnknown", + "testSkipPast", + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9444444444444444, + "predicted": 144, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 9, + "top_hits": 9, + "top_n": 9, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.38526912181303113, + "predicted": 353, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9, + "predicted": 10, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.38526912181303113, + "predicted": 353, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9, + "predicted": 10, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextClean", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 271, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 258, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.02214022140221402, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.996309963099631, + "precision": 0.5113636363636364, + "predicted": 528, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.996309963099631, + "precision": 0.5113636363636364, + "predicted": 528, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::JSONTokener::nextString", + "file": "src/main/java/org/json/JSONTokener.java", + "truth": 235, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "TestLenientCommas", + "allowNullArrayInStrictMode", + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyJSONXMLException" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 294, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.02127659574468085, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9957446808510638, + "precision": 0.4431818181818182, + "predicted": 528, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9957446808510638, + "precision": 0.4431818181818182, + "predicted": 528, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::getMaxNumberLength", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 82, + "truth_doctests": 0, + "predicted_gitnext": 138, + "predicted_coverage": 487, + "missed_by_coverage": [], + "covered_not_failing": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "arrayIndexFailure", + "arrayIndexQuery", + "badObject", + "builderPreservesLiteralTokens", + "builderTest", + "commentsInXML" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 405, + "top_coverage": [ + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal", + "testOptBigInteger", + "testOptDouble" + ], + "score": { + "gitnext": { + "recall": 0.32926829268292684, + "precision": 0.1956521739130435, + "predicted": 138, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.8, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.16837782340862423, + "predicted": 487, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.16768916155419222, + "predicted": 489, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::ParserConfiguration::isKeepStrings", + "file": "src/main/java/org/json/ParserConfiguration.java", + "truth": 34, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 202, + "missed_by_coverage": [], + "covered_not_failing": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "clarifyCurrentBehavior", + "contentOperations", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "explicitConfigurationAppliesToNestedDuplicateKeys" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 168, + "top_coverage": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "explicitConfigurationAppliesToNestedDuplicateKeys", + "explicitConfigurationOverridesTokenerStrictMode", + "givenCompliantJSONArrayFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenCompliantJSONObjectFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenEmptyArray_testStrictModeTrue_shouldNotThrowJsonException", + "givenEmptyObject_testStrictModeTrue_shouldNotThrowJsonException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.2857142857142857, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.16831683168316833, + "predicted": 202, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.16748768472906403, + "predicted": 203, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XML::noSpace", + "file": "src/main/java/org/json/XML.java", + "truth": 12, + "truth_doctests": 0, + "predicted_gitnext": 14, + "predicted_coverage": 118, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "contentOperations", + "emptyJSONXMLException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 106, + "top_coverage": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8571428571428571, + "predicted": 14, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.1016949152542373, + "predicted": 118, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1016949152542373, + "predicted": 118, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getForceList", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 64, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 266, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence", + "issue654IncorrectNestingNoKey1", + "issue654IncorrectNestingNoKey2", + "issue654StackOverflowInput" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 202, + "top_coverage": [ + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyXML" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.8205128205128205, + "predicted": 78, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.24060150375939848, + "predicted": 266, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.24060150375939848, + "predicted": 266, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::getXsiTypeMap", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 99, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "contentOperations", + "shouldCreateExplicitEndTagWithEmptyValueWhenConfigured", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 77, + "top_coverage": [ + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyXML" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.27848101265822783, + "predicted": 79, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2222222222222222, + "predicted": 99, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.22, + "predicted": 100, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::isConvertNilAttributeToNull", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 78, + "predicted_coverage": 99, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "contentOperations", + "shouldCreateExplicitEndTagWithEmptyValueWhenConfigured", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 77, + "top_coverage": [ + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyXML" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.28205128205128205, + "predicted": 78, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2222222222222222, + "predicted": 99, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.22, + "predicted": 100, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::shouldTrimWhiteSpace", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 93, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 314, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "emptyTagException", + "emptyXMLException", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 221, + "top_coverage": [ + "testNextCDATA", + "testNextContentWithTag", + "testNextContentWithText", + "testNextEntityKnown", + "testNextEntityUnknown", + "testSkipPast", + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7815126050420168, + "predicted": 119, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2961783439490446, + "predicted": 314, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8333333333333334, + "predicted": 6, + "top_hits": 5, + "top_n": 6, + "top_precision": 0.8333333333333334, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2961783439490446, + "predicted": 314, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8333333333333334, + "predicted": 6, + "top_hits": 5, + "top_n": 6, + "top_precision": 0.8333333333333334, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLParserConfiguration::withKeepBooleanAsString", + "file": "src/main/java/org/json/XMLParserConfiguration.java", + "truth": 2, + "truth_doctests": 0, + "predicted_gitnext": 79, + "predicted_coverage": 53, + "missed_by_coverage": [], + "covered_not_failing": [ + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyXML" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 51, + "top_coverage": [ + "testToJSONArray_jsonOutput_withKeepBooleanAsString", + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.02531645569620253, + "predicted": 79, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.03773584905660377, + "predicted": 53, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.3333333333333333, + "predicted": 3, + "top_hits": 1, + "top_n": 3, + "top_precision": 0.3333333333333333, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.021505376344086023, + "predicted": 93, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextCDATA", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 118, + "predicted_coverage": 148, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 143, + "top_coverage": [ + "testNextCDATA", + "testNextContentWithTag", + "testNextContentWithText", + "testNextEntityKnown", + "testNextEntityUnknown", + "testSkipPast", + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.0423728813559322, + "predicted": 118, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.033783783783783786, + "predicted": 148, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.03355704697986577, + "predicted": 149, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::nextMeta", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 6, + "truth_doctests": 0, + "predicted_gitnext": 117, + "predicted_coverage": 142, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "commentsInXML", + "contentOperations", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 136, + "top_coverage": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.05128205128205128, + "predicted": 117, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.04225352112676056, + "predicted": 142, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.04195804195804196, + "predicted": 143, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::json::XMLTokener::skipPast", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 88, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 314, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "emptyJSONXMLException", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "emptyTagException", + "emptyXMLException", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 226, + "top_coverage": [ + "testSkipPast", + "testNextCDATA", + "testNextContentWithTag", + "testNextContentWithText", + "testNextEntityKnown", + "testNextEntityUnknown", + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7394957983193278, + "predicted": 119, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.2802547770700637, + "predicted": 314, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8333333333333334, + "predicted": 6, + "top_hits": 5, + "top_n": 6, + "top_precision": 0.8333333333333334, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2802547770700637, + "predicted": 314, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8333333333333334, + "predicted": 6, + "top_hits": 5, + "top_n": 6, + "top_precision": 0.8333333333333334, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optJSONObject", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 168, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence", + "issue654IncorrectNestingNoKey1", + "issue654IncorrectNestingNoKey2" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 165, + "top_coverage": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence", + "issue654IncorrectNestingNoKey1", + "issue654IncorrectNestingNoKey2" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.017857142857142856, + "predicted": 168, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.017543859649122806, + "predicted": 171, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONParserConfiguration::withMaxSharedKeys", + "file": "src/main/java/org/json/JSONParserConfiguration.java", + "truth": 5, + "truth_doctests": 0, + "predicted_gitnext": 5, + "predicted_coverage": 60, + "missed_by_coverage": [], + "covered_not_failing": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "explicitConfigurationAppliesToNestedDuplicateKeys", + "explicitConfigurationOverridesTokenerStrictMode", + "givenCompliantJSONArrayFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenCompliantJSONObjectFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenEmptyArray_testStrictModeTrue_shouldNotThrowJsonException", + "givenEmptyObject_testStrictModeTrue_shouldNotThrowJsonException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 55, + "top_coverage": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "explicitConfigurationAppliesToNestedDuplicateKeys", + "explicitConfigurationOverridesTokenerStrictMode", + "givenCompliantJSONArrayFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenCompliantJSONObjectFile_testStrictModeTrue_shouldNotThrowAnyException", + "givenEmptyArray_testStrictModeTrue_shouldNotThrowJsonException", + "givenEmptyObject_testStrictModeTrue_shouldNotThrowJsonException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.08333333333333333, + "predicted": 60, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.08333333333333333, + "predicted": 60, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XMLTokener::parseHexEntity", + "file": "src/main/java/org/json/XMLTokener.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 142, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "commentsInXML", + "contentOperations", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 126, + "top_coverage": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "score": { + "gitnext": { + "recall": 0.1875, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.6666666666666666, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.11267605633802817, + "predicted": 142, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.75, + "predicted": 4, + "top_hits": 3, + "top_n": 4, + "top_precision": 0.75, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1095890410958904, + "predicted": 146, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6, + "predicted": 5, + "top_hits": 3, + "top_n": 5, + "top_precision": 0.6, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::checkKeyForStrictMode", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 223, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 528, + "missed_by_coverage": [ + "setJsonObject" + ], + "covered_not_failing": [ + "TestLenientCommas", + "allowNullArrayInStrictMode", + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "booleanAttribute", + "checkSpecialChars", + "commentsInXML", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString" + ], + "missed_by_coverage_n": 1, + "covered_not_failing_n": 306, + "top_coverage": [ + "convertHTTPRequestToString", + "convertHTTPResponseToString", + "emptyStringHTTPRequest", + "extendedHTTPRequest", + "extendedHTTPResponse", + "notEnoughHTTPException", + "nullHTTPException", + "simpleHTTPRequest", + "simpleHTTPResponse", + "booleanAttribute" + ], + "score": { + "gitnext": { + "recall": 0.29596412556053814, + "precision": 0.38596491228070173, + "predicted": 171, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.3333333333333333, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9955156950672646, + "precision": 0.42045454545454547, + "predicted": 528, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9955156950672646, + "precision": 0.4180790960451977, + "predicted": 531, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9230769230769231, + "predicted": 13, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONML::toString", + "file": "src/main/java/org/json/JSONML.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 43, + "predicted_coverage": 118, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "commentsInXML", + "contentOperations", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 114, + "top_coverage": [ + "commentsInXML", + "emptyJSONXMLException", + "emptyTagException", + "emptyXMLException", + "invalidBangInTagException", + "invalidBangNoCloseInTagException", + "invalidCDATABangInTagException", + "invalidSlashInTagException", + "noCloseEndBraceException", + "noCloseEndTagException" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.09302325581395349, + "predicted": 43, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.03389830508474576, + "predicted": 118, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 4, + "top_hits": 2, + "top_n": 4, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.027586206896551724, + "predicted": 145, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 12, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONWriter::array", + "file": "src/main/java/org/json/JSONWriter.java", + "truth": 4, + "truth_doctests": 0, + "predicted_gitnext": 86, + "predicted_coverage": 10, + "missed_by_coverage": [], + "covered_not_failing": [ + "exceedNestDepthException", + "missplacedEndArrayException", + "missplacedObjectException", + "nullKeyException", + "outOfSequenceException", + "simpleObjectString" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 6, + "top_coverage": [ + "missplacedArrayException", + "missplacedEndArrayException", + "simpleArrayString", + "complexObjectString", + "exceedNestDepthException", + "missplacedEndObjectException", + "missplacedObjectException", + "nullKeyException", + "outOfSequenceException", + "simpleObjectString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.046511627906976744, + "predicted": 86, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.4, + "predicted": 10, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.04395604395604396, + "predicted": 91, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.09090909090909091, + "predicted": 11, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::CDL::rowToJSONObject", + "file": "src/main/java/org/json/CDL.java", + "truth": 13, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 21, + "missed_by_coverage": [], + "covered_not_failing": [ + "checkSpecialChars", + "emptyJSONArrayToString", + "emptyString", + "exceptionOnNullString", + "nullInName", + "nullJSONArrayToString", + "nullJSONArraysToString", + "unbalancedQuoteInName" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 8, + "top_coverage": [ + "badEscapedQuote", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyLinesToJSONArray", + "emptyString", + "exceptionOnNullString", + "jsonArrayToJSONArray", + "nullInName", + "nullJSONArrayToString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.7647058823529411, + "predicted": 17, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.6190476190476191, + "predicted": 21, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.6190476190476191, + "predicted": 21, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONArray::getJSONObject", + "file": "src/main/java/org/json/JSONArray.java", + "truth": 22, + "truth_doctests": 0, + "predicted_gitnext": 36, + "predicted_coverage": 332, + "missed_by_coverage": [], + "covered_not_failing": [ + "TestLenientCommas", + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "badEscapedQuote", + "badObject", + "bigNumberOperations", + "clarifyCurrentBehavior", + "commentsInXML" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 310, + "top_coverage": [ + "badEscapedQuote", + "checkSpecialChars", + "csvParsingCatchesLastRow", + "emptyJSONArrayToString", + "emptyLinesToJSONArray", + "emptyString", + "exceptionOnNullString", + "jsonArrayToJSONArray", + "nullInName", + "nullJSONArrayToString" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.6111111111111112, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.06626506024096386, + "predicted": 332, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7142857142857143, + "predicted": 7, + "top_hits": 5, + "top_n": 7, + "top_precision": 0.7142857142857143, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.06626506024096386, + "predicted": 332, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7142857142857143, + "predicted": 7, + "top_hits": 5, + "top_n": 7, + "top_precision": 0.7142857142857143, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::XML::indent", + "file": "src/main/java/org/json/XML.java", + "truth": 27, + "truth_doctests": 0, + "predicted_gitnext": 6, + "predicted_coverage": 266, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence", + "issue654IncorrectNestingNoKey1", + "issue654IncorrectNestingNoKey2" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 239, + "top_coverage": [ + "contentOperations", + "shouldHandleArraytoString", + "shouldHandleCommentsInXML", + "shouldHandleContentArraytoString", + "shouldHandleContentNoArraytoString", + "shouldHandleEmptyArray", + "shouldHandleEmptyJSONXML", + "shouldHandleEmptyMultiArray", + "shouldHandleEmptyNodeValue", + "shouldHandleEmptyXML" + ], + "score": { + "gitnext": { + "recall": 0.14814814814814814, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.10150375939849623, + "predicted": 266, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.10112359550561797, + "predicted": 267, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8, + "predicted": 5, + "top_hits": 4, + "top_n": 5, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::optEnum", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 3, + "truth_doctests": 0, + "predicted_gitnext": 172, + "predicted_coverage": 176, + "missed_by_coverage": [], + "covered_not_failing": [ + "bigNumberOperations", + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "enumPut", + "enumToString", + "enumValueToString", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 173, + "top_coverage": [ + "enumAPI", + "enumPut", + "enumToString", + "enumValueToString", + "jsonObjectFromEnum", + "jsonObjectFromEnumWithNames", + "verifyEnumConsistency", + "wrap", + "bigNumberOperations", + "clarifyCurrentBehavior" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.01744186046511628, + "predicted": 172, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.017045454545454544, + "predicted": 176, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.01675977653631285, + "predicted": 179, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4, + "predicted": 5, + "top_hits": 2, + "top_n": 5, + "top_precision": 0.4, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::getKeyNameFromMethod", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 32, + "truth_doctests": 0, + "predicted_gitnext": 171, + "predicted_coverage": 185, + "missed_by_coverage": [], + "covered_not_failing": [ + "clarifyCurrentBehavior", + "emptyJsonObject", + "emptyJsonObjectNamesToJsonAray", + "enumAPI", + "enumPut", + "enumValueToString", + "equals", + "getBigIntegerHonorsMaxNumberLengthConfig", + "getBigIntegerHugeExponentReturnsDefault", + "invalidEscapeSequence" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 153, + "top_coverage": [ + "enumsShouldNotIncludeEnumMethods", + "genericBeanWithGetAndIsMethodsShouldNotBeIncluded", + "javaLibraryClassesShouldNotIncludeTheirMethods", + "jsonObjectByRecord", + "methodsStartingWithUppercaseShouldNotBeIncluded", + "mixedGettersAndRecordStyleAccessors", + "numberSubclassesShouldNotIncludeNumberMethods", + "recordStyleClassShouldNotIncludeObjectMethods", + "jsonObjectByLocaleBean", + "enumAPI" + ], + "score": { + "gitnext": { + "recall": 0.75, + "precision": 0.14035087719298245, + "predicted": 171, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.25, + "precision": 0.25, + "predicted": 4, + "top_hits": 1, + "top_n": 4, + "top_precision": 0.25, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.17297297297297298, + "predicted": 185, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1702127659574468, + "predicted": 188, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::json::JSONObject::isDecimalNotation", + "file": "src/main/java/org/json/JSONObject.java", + "truth": 87, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 487, + "missed_by_coverage": [], + "covered_not_failing": [ + "allowNullArrayInStrictMode", + "allowNullObjectInStrictMode", + "arrayConstructorAppliesExplicitStrictModeToValues", + "arrayConstructorRestoresTokenerConfiguration", + "arrayIndexFailure", + "arrayIndexQuery", + "badObject", + "builderPreservesLiteralTokens", + "builderTest", + "commentsInXML" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 400, + "top_coverage": [ + "testGetBigDecimal", + "testGetBigInteger", + "testGetDouble", + "testGetFloat", + "testGetInt", + "testGetLong", + "testGetNumber", + "testOptBigDecimal", + "testOptBigInteger", + "testOptDouble" + ], + "score": { + "gitnext": { + "recall": 0.28735632183908044, + "precision": 0.8064516129032258, + "predicted": 31, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.4, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.17864476386036962, + "predicted": 487, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.17791411042944785, + "predicted": 489, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 10, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.703, + "precision": 0.524, + "top_precision": 0.655, + "caught": 1.0 + }, + "coverage": { + "recall": 0.999, + "precision": 0.228, + "top_precision": 0.332, + "caught": 0.818 + }, + "union": { + "recall": 0.999, + "precision": 0.227, + "top_precision": 0.668, + "caught": 1.0 + }, + "functions": 22 + }, + "random": { + "gitnext": { + "recall": 0.722, + "precision": 0.415, + "top_precision": 0.525, + "caught": 0.917 + }, + "coverage": { + "recall": 1.0, + "precision": 0.185, + "top_precision": 0.308, + "caught": 0.667 + }, + "union": { + "recall": 1.0, + "precision": 0.154, + "top_precision": 0.483, + "caught": 0.917 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.71, + "precision": 0.485, + "top_precision": 0.609, + "caught": 0.971 + }, + "coverage": { + "recall": 0.999, + "precision": 0.213, + "top_precision": 0.324, + "caught": 0.765 + }, + "union": { + "recall": 0.999, + "precision": 0.201, + "top_precision": 0.603, + "caught": 0.971 + }, + "functions": 34 + } + }, + "cost": { + "how": "per-class", + "mvn_s": 72.2, + "analyze_s": 0.4, + "map_s": 72.6, + "exec_bytes": 16999, + "dumps": 22, + "test_methods": 640, + "classes": 22, + "names_in_several_classes": 47, + "junit": 4, + "package": "org.json", + "owner": "body", + "repeat": 3, + "plain_s": 4.5, + "tests": 640, + "map_pairs": 75187, + "map_functions": 311, + "map_sqlite_bytes": 774144, + "functions_with_body": 510, + "functions_never_run": 16, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 16.1 + } + } +} \ No newline at end of file diff --git a/eval/results/learn-tests-json-java.md b/eval/results/learn-tests-json-java.md new file mode 100644 index 0000000..c8b6d3e --- /dev/null +++ b/eval/results/learn-tests-json-java.md @@ -0,0 +1,318 @@ +# learn-tests на json-java (java): сводка + +Все функции (pick + случайные). Полнота / точность / из 10 падают / поймано первыми 10. По методу — как истина eval (имя метода теста); по классу — класс, где есть падающий метод. + +| способ | единица | функций | граф (`tests-for`) | покрытие | объединение | карта, с | × обычного | +|---|---|---:|---|---|---|---:|---:| +| per-test | метод | 34 | 71% / 48% / 61% / 97% | 99% / 94% / 95% / 100% | 100% / 52% / 81% / 100% | 10.5 | 2.1 | +| per-test | класс | 34 | 78% / 77% / 77% / 100% | 100% / 97% / 97% / 100% | 100% / 77% / 77% / 100% | 10.5 | 2.1 | +| per-test-fork | метод | 34 | 71% / 48% / 61% / 97% | 99% / 94% / 95% / 100% | 100% / 52% / 81% / 100% | 20.2 | 4.1 | +| per-test-fork | класс | 34 | 78% / 77% / 77% / 100% | 100% / 97% / 97% / 100% | 100% / 77% / 77% / 100% | 20.2 | 4.1 | +| per-class | метод | 34 | 71% / 48% / 61% / 97% | 100% / 21% / 32% / 76% | 100% / 20% / 60% / 97% | 72.6 | 16.1 | +| per-class | класс | 34 | 78% / 77% / 77% / 100% | 100% / 77% / 77% / 100% | 100% / 73% / 73% / 100% | 72.6 | 16.1 | + +# learn-tests (per-test) на json-java (java): покрытие по тестам против `tests-for` +json-java 2cd0a8c2112a, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 22 | gitnext | 70% | 52% | 66% | 100% | +| pick (кэш eval) | 22 | coverage | 100% | 96% | 98% | 100% | +| pick (кэш eval) | 22 | union | 100% | 54% | 83% | 100% | +| случайные | 12 | gitnext | 72% | 42% | 52% | 92% | +| случайные | 12 | coverage | 98% | 90% | 89% | 100% | +| случайные | 12 | union | 100% | 47% | 76% | 100% | +| все | 34 | gitnext | 71% | 48% | 61% | 97% | +| все | 34 | coverage | 99% | 94% | 95% | 100% | +| все | 34 | union | 100% | 52% | 81% | 100% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::json::JSONObject::numberToString` | 46 | 20% / 50% | 100% / 98% / 100% | | +| pick | `org::json::JSONObject::putOnce` | 5 | 100% / 40% | 100% / 100% / 100% | | +| pick | `org::json::JSONObject::putOpt` | 3 | 67% / 10% | 100% / 100% / 100% | | +| pick | `org::json::JSONObject::toMap` | 2 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::JSONParserConfiguration::getMaxSharedKeys` | 224 | 17% / 100% | 100% / 92% / 100% | setJsonObject | +| pick | `org::json::JSONParserConfiguration::isStrictMode` | 268 | 5% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONPointer::queryFrom` | 70 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::json::JSONTokener::getJsonParserConfiguration` | 271 | 2% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONTokener::more` | 136 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::JSONTokener::nextClean` | 271 | 2% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONTokener::nextString` | 235 | 2% / 100% | 100% / 92% / 100% | setJsonObject | +| pick | `org::json::ParserConfiguration::getMaxNumberLength` | 82 | 33% / 70% | 100% / 46% / 60% | | +| pick | `org::json::ParserConfiguration::isKeepStrings` | 34 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::XML::noSpace` | 12 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::getForceList` | 64 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::getXsiTypeMap` | 22 | 100% / 60% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::isConvertNilAttributeToNull` | 22 | 100% / 50% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::shouldTrimWhiteSpace` | 93 | 100% / 60% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::withKeepBooleanAsString` | 2 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::nextCDATA` | 5 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::nextMeta` | 6 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::skipPast` | 88 | 100% / 80% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::optJSONObject` | 3 | 100% / 20% | 100% / 100% / 100% | | +| random | `org::json::JSONParserConfiguration::withMaxSharedKeys` | 5 | 100% / 100% | 100% / 100% / 100% | | +| random | `org::json::XMLTokener::parseHexEntity` | 16 | 19% / 43% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::checkKeyForStrictMode` | 223 | 30% / 60% | 100% / 92% / 100% | setJsonObject | +| random | `org::json::JSONML::toString` | 4 | 100% / 20% | 100% / 29% / 10% | | +| random | `org::json::JSONWriter::array` | 4 | 100% / 40% | 100% / 100% / 100% | | +| random | `org::json::CDL::rowToJSONObject` | 13 | 100% / 80% | 77% / 100% / 100% | badEscapedQuote, unbalancedEscapedQuote, unbalancedQuoteInValue | +| random | `org::json::JSONArray::getJSONObject` | 22 | 100% / 100% | 100% / 100% / 100% | | +| random | `org::json::XML::indent` | 27 | 15% / 67% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::optEnum` | 3 | 100% / 30% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::getKeyNameFromMethod` | 32 | 75% / 0% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::isDecimalNotation` | 87 | 29% / 70% | 100% / 56% / 60% | | + +## Цена + +- `how`: per-test +- `class_inits_s`: 0.5 +- `class_inits_booked`: True +- `files_with_init_lines`: 5 +- `mvn_s`: 6.1 +- `analyze_s`: 3.8 +- `map_s`: 10.5 +- `exec_bytes`: 503756 +- `dumps`: 1686 +- `test_methods`: 634 +- `classes`: 22 +- `classes_with_setup_lines`: 1 +- `run_failed`: False +- `names_in_several_classes`: 46 +- `junit`: 4 +- `package`: org.json +- `owner`: body +- `repeat`: 3 +- `plain_s`: 5.1 +- `tests`: 634 +- `map_pairs`: 18211 +- `map_functions`: 311 +- `map_sqlite_bytes`: 233472 +- `functions_with_body`: 510 +- `functions_never_run`: 16 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 2.1 + +# learn-tests (per-test-fork) на json-java (java): покрытие по тестам против `tests-for` +json-java 2cd0a8c2112a, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 22 | gitnext | 70% | 52% | 66% | 100% | +| pick (кэш eval) | 22 | coverage | 100% | 96% | 98% | 100% | +| pick (кэш eval) | 22 | union | 100% | 54% | 83% | 100% | +| случайные | 12 | gitnext | 72% | 42% | 52% | 92% | +| случайные | 12 | coverage | 98% | 90% | 89% | 100% | +| случайные | 12 | union | 100% | 47% | 76% | 100% | +| все | 34 | gitnext | 71% | 48% | 61% | 97% | +| все | 34 | coverage | 99% | 94% | 95% | 100% | +| все | 34 | union | 100% | 52% | 81% | 100% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::json::JSONObject::numberToString` | 46 | 20% / 50% | 100% / 98% / 100% | | +| pick | `org::json::JSONObject::putOnce` | 5 | 100% / 40% | 100% / 100% / 100% | | +| pick | `org::json::JSONObject::putOpt` | 3 | 67% / 10% | 100% / 100% / 100% | | +| pick | `org::json::JSONObject::toMap` | 2 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::JSONParserConfiguration::getMaxSharedKeys` | 224 | 17% / 100% | 100% / 92% / 100% | setJsonObject | +| pick | `org::json::JSONParserConfiguration::isStrictMode` | 268 | 5% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONPointer::queryFrom` | 70 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::json::JSONTokener::getJsonParserConfiguration` | 271 | 2% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONTokener::more` | 136 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::JSONTokener::nextClean` | 271 | 2% / 100% | 100% / 93% / 100% | setJsonObject | +| pick | `org::json::JSONTokener::nextString` | 235 | 2% / 100% | 100% / 92% / 100% | setJsonObject | +| pick | `org::json::ParserConfiguration::getMaxNumberLength` | 82 | 33% / 70% | 100% / 46% / 60% | | +| pick | `org::json::ParserConfiguration::isKeepStrings` | 34 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::XML::noSpace` | 12 | 100% / 80% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::getForceList` | 64 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::getXsiTypeMap` | 22 | 100% / 60% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::isConvertNilAttributeToNull` | 22 | 100% / 50% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::shouldTrimWhiteSpace` | 93 | 100% / 60% | 100% / 100% / 100% | | +| pick | `org::json::XMLParserConfiguration::withKeepBooleanAsString` | 2 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::nextCDATA` | 5 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::nextMeta` | 6 | 100% / 20% | 100% / 100% / 100% | | +| pick | `org::json::XMLTokener::skipPast` | 88 | 100% / 80% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::optJSONObject` | 3 | 100% / 20% | 100% / 100% / 100% | | +| random | `org::json::JSONParserConfiguration::withMaxSharedKeys` | 5 | 100% / 100% | 100% / 100% / 100% | | +| random | `org::json::XMLTokener::parseHexEntity` | 16 | 19% / 43% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::checkKeyForStrictMode` | 223 | 30% / 60% | 100% / 92% / 100% | setJsonObject | +| random | `org::json::JSONML::toString` | 4 | 100% / 20% | 100% / 29% / 10% | | +| random | `org::json::JSONWriter::array` | 4 | 100% / 40% | 100% / 100% / 100% | | +| random | `org::json::CDL::rowToJSONObject` | 13 | 100% / 80% | 77% / 100% / 100% | badEscapedQuote, unbalancedEscapedQuote, unbalancedQuoteInValue | +| random | `org::json::JSONArray::getJSONObject` | 22 | 100% / 100% | 100% / 100% / 100% | | +| random | `org::json::XML::indent` | 27 | 15% / 67% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::optEnum` | 3 | 100% / 30% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::getKeyNameFromMethod` | 32 | 75% / 0% | 100% / 100% / 100% | | +| random | `org::json::JSONObject::isDecimalNotation` | 87 | 29% / 70% | 100% / 56% / 60% | | + +## Цена + +- `how`: per-test-fork +- `class_inits_s`: 0.5 +- `class_inits_booked`: True +- `files_with_init_lines`: 5 +- `mvn_s`: 16.6 +- `analyze_s`: 3.1 +- `map_s`: 20.2 +- `exec_bytes`: 504392 +- `dumps`: 1686 +- `test_methods`: 634 +- `classes`: 22 +- `classes_with_setup_lines`: 1 +- `run_failed`: False +- `names_in_several_classes`: 46 +- `junit`: 4 +- `package`: org.json +- `owner`: body +- `repeat`: 3 +- `plain_s`: 4.9 +- `tests`: 634 +- `map_pairs`: 18214 +- `map_functions`: 311 +- `map_sqlite_bytes`: 233472 +- `functions_with_body`: 510 +- `functions_never_run`: 16 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 4.1 + +# learn-tests (per-class) на json-java (java): покрытие по тестам против `tests-for` +json-java 2cd0a8c2112a, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 22 | gitnext | 70% | 52% | 66% | 100% | +| pick (кэш eval) | 22 | coverage | 100% | 23% | 33% | 82% | +| pick (кэш eval) | 22 | union | 100% | 23% | 67% | 100% | +| случайные | 12 | gitnext | 72% | 42% | 52% | 92% | +| случайные | 12 | coverage | 100% | 18% | 31% | 67% | +| случайные | 12 | union | 100% | 15% | 48% | 92% | +| все | 34 | gitnext | 71% | 48% | 61% | 97% | +| все | 34 | coverage | 100% | 21% | 32% | 76% | +| все | 34 | union | 100% | 20% | 60% | 97% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::json::JSONObject::numberToString` | 46 | 20% / 50% | 100% / 12% / 20% | | +| pick | `org::json::JSONObject::putOnce` | 5 | 100% / 40% | 100% / 3% / 40% | | +| pick | `org::json::JSONObject::putOpt` | 3 | 67% / 10% | 100% / 2% / 20% | | +| pick | `org::json::JSONObject::toMap` | 2 | 100% / 20% | 100% / 1% / 0% | | +| pick | `org::json::JSONParserConfiguration::getMaxSharedKeys` | 224 | 17% / 100% | 100% / 42% / 70% | setJsonObject | +| pick | `org::json::JSONParserConfiguration::isStrictMode` | 268 | 5% / 100% | 100% / 51% / 80% | setJsonObject | +| pick | `org::json::JSONPointer::queryFrom` | 70 | 100% / 100% | 100% / 25% / 50% | | +| pick | `org::json::JSONTokener::getJsonParserConfiguration` | 271 | 2% / 100% | 100% / 51% / 80% | setJsonObject | +| pick | `org::json::JSONTokener::more` | 136 | 100% / 80% | 100% / 39% / 50% | | +| pick | `org::json::JSONTokener::nextClean` | 271 | 2% / 100% | 100% / 51% / 80% | setJsonObject | +| pick | `org::json::JSONTokener::nextString` | 235 | 2% / 100% | 100% / 44% / 70% | setJsonObject | +| pick | `org::json::ParserConfiguration::getMaxNumberLength` | 82 | 33% / 70% | 100% / 17% / 40% | | +| pick | `org::json::ParserConfiguration::isKeepStrings` | 34 | 100% / 80% | 100% / 17% / 0% | | +| pick | `org::json::XML::noSpace` | 12 | 100% / 80% | 100% / 10% / 20% | | +| pick | `org::json::XMLParserConfiguration::getForceList` | 64 | 100% / 100% | 100% / 24% / 20% | | +| pick | `org::json::XMLParserConfiguration::getXsiTypeMap` | 22 | 100% / 60% | 100% / 22% / 0% | | +| pick | `org::json::XMLParserConfiguration::isConvertNilAttributeToNull` | 22 | 100% / 50% | 100% / 22% / 0% | | +| pick | `org::json::XMLParserConfiguration::shouldTrimWhiteSpace` | 93 | 100% / 60% | 100% / 30% / 20% | | +| pick | `org::json::XMLParserConfiguration::withKeepBooleanAsString` | 2 | 100% / 20% | 100% / 4% / 10% | | +| pick | `org::json::XMLTokener::nextCDATA` | 5 | 100% / 20% | 100% / 3% / 10% | | +| pick | `org::json::XMLTokener::nextMeta` | 6 | 100% / 20% | 100% / 4% / 20% | | +| pick | `org::json::XMLTokener::skipPast` | 88 | 100% / 80% | 100% / 28% / 30% | | +| random | `org::json::JSONObject::optJSONObject` | 3 | 100% / 20% | 100% / 2% / 0% | | +| random | `org::json::JSONParserConfiguration::withMaxSharedKeys` | 5 | 100% / 100% | 100% / 8% / 0% | | +| random | `org::json::XMLTokener::parseHexEntity` | 16 | 19% / 43% | 100% / 11% / 0% | | +| random | `org::json::JSONObject::checkKeyForStrictMode` | 223 | 30% / 60% | 100% / 42% / 70% | setJsonObject | +| random | `org::json::JSONML::toString` | 4 | 100% / 20% | 100% / 3% / 0% | | +| random | `org::json::JSONWriter::array` | 4 | 100% / 40% | 100% / 40% / 40% | | +| random | `org::json::CDL::rowToJSONObject` | 13 | 100% / 80% | 100% / 62% / 40% | | +| random | `org::json::JSONArray::getJSONObject` | 22 | 100% / 100% | 100% / 7% / 30% | | +| random | `org::json::XML::indent` | 27 | 15% / 67% | 100% / 10% / 70% | | +| random | `org::json::JSONObject::optEnum` | 3 | 100% / 30% | 100% / 2% / 10% | | +| random | `org::json::JSONObject::getKeyNameFromMethod` | 32 | 75% / 0% | 100% / 17% / 50% | | +| random | `org::json::JSONObject::isDecimalNotation` | 87 | 29% / 70% | 100% / 18% / 60% | | + +## Цена + +- `how`: per-class +- `mvn_s`: 72.2 +- `analyze_s`: 0.4 +- `map_s`: 72.6 +- `exec_bytes`: 16999 +- `dumps`: 22 +- `test_methods`: 640 +- `classes`: 22 +- `names_in_several_classes`: 47 +- `junit`: 4 +- `package`: org.json +- `owner`: body +- `repeat`: 3 +- `plain_s`: 4.5 +- `tests`: 640 +- `map_pairs`: 75187 +- `map_functions`: 311 +- `map_sqlite_bytes`: 774144 +- `functions_with_body`: 510 +- `functions_never_run`: 16 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 16.1 + +# По классу теста + +Класс засчитан, если в нём есть метод с падающим именем (имя в нескольких классах — все они). + +| способ | выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---|---:|---|---:|---:|---:|---:| +| per-test | pick | 22 | gitnext | 79% | 84% | 84% | 100% | +| per-test | pick | 22 | coverage | 100% | 98% | 98% | 100% | +| per-test | pick | 22 | union | 100% | 83% | 82% | 100% | +| per-test | random | 12 | gitnext | 76% | 62% | 63% | 100% | +| per-test | random | 12 | coverage | 100% | 95% | 95% | 100% | +| per-test | random | 12 | union | 100% | 66% | 66% | 100% | +| per-test | all | 34 | gitnext | 78% | 77% | 77% | 100% | +| per-test | all | 34 | coverage | 100% | 97% | 97% | 100% | +| per-test | all | 34 | union | 100% | 77% | 77% | 100% | +| per-test-fork | pick | 22 | gitnext | 79% | 84% | 84% | 100% | +| per-test-fork | pick | 22 | coverage | 100% | 98% | 98% | 100% | +| per-test-fork | pick | 22 | union | 100% | 83% | 82% | 100% | +| per-test-fork | random | 12 | gitnext | 76% | 62% | 63% | 100% | +| per-test-fork | random | 12 | coverage | 100% | 95% | 95% | 100% | +| per-test-fork | random | 12 | union | 100% | 66% | 66% | 100% | +| per-test-fork | all | 34 | gitnext | 78% | 77% | 77% | 100% | +| per-test-fork | all | 34 | coverage | 100% | 97% | 97% | 100% | +| per-test-fork | all | 34 | union | 100% | 77% | 77% | 100% | +| per-class | pick | 22 | gitnext | 79% | 84% | 84% | 100% | +| per-class | pick | 22 | coverage | 100% | 79% | 79% | 100% | +| per-class | pick | 22 | union | 100% | 79% | 78% | 100% | +| per-class | random | 12 | gitnext | 76% | 62% | 63% | 100% | +| per-class | random | 12 | coverage | 100% | 74% | 74% | 100% | +| per-class | random | 12 | union | 100% | 63% | 63% | 100% | +| per-class | all | 34 | gitnext | 78% | 77% | 77% | 100% | +| per-class | all | 34 | coverage | 100% | 77% | 77% | 100% | +| per-class | all | 34 | union | 100% | 73% | 73% | 100% | + +# Цена + +| | per-test | per-test-fork | per-class | +|---|---:|---:|---:| +| plain_s | 5.1 | 4.9 | 4.5 | +| mvn_s | 6.1 | 16.6 | 72.2 | +| analyze_s | 3.8 | 3.1 | 0.4 | +| map_s | 10.5 | 20.2 | 72.6 | +| slowdown_runs | 2.1 | 4.1 | 16.1 | +| exec_bytes | 503756 | 504392 | 16999 | +| map_sqlite_bytes | 233472 | 233472 | 774144 | +| dumps | 1686 | 1686 | 22 | +| test_methods | 634 | 634 | 640 | +| classes | 22 | 22 | 22 | +| names_in_several_classes | 46 | 46 | 47 | +| map_pairs | 18211 | 18214 | 75187 | +| map_functions | 311 | 311 | 311 | +| functions_with_body | 510 | 510 | 510 | +| functions_never_run | 16 | 16 | 16 | + +json-java 2cd0a8c2112a diff --git a/eval/results/learn-tests-jsoup.json b/eval/results/learn-tests-jsoup.json new file mode 100644 index 0000000..233e86b --- /dev/null +++ b/eval/results/learn-tests-jsoup.json @@ -0,0 +1,8979 @@ +{ + "per-test": { + "repo": "jsoup", + "lang": "java", + "commit": "d90d57521d93", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::jsoup::internal::LegacyMatching::cssAttributeMatches", + "file": "src/main/java/org/jsoup/internal/LegacyMatching.java", + "truth": 7, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 7, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "qualifiedAttributeNamesUseTheSameLiteralAndRegexParsing", + "parsedAttributeValuesRoundTrip", + "aLoneOperandIsNotAnExplicitFlag", + "absoluteAttributeLookupIgnoresKeyCase", + "hyphenatedAttributeValueAtBufferBoundary", + "explicitCompilationCapturesTheWholeQuery", + "explicitFlagsOverrideDefaultsInBothModes" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::Normalizer::lowerCase", + "file": "src/main/java/org/jsoup/internal/Normalizer.java", + "truth": 1689, + "truth_doctests": 0, + "predicted_gitnext": 1183, + "predicted_coverage": 304, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 1385, + "covered_not_failing_n": 0, + "top_coverage": [ + "separatesAsciiAndUnicodeCase", + "testContainsDataToString", + "testAttributeWithValueContainingToString", + "testAttributeWithValueEndingToString", + "testAttributeWithValueNotToString", + "testAttributeWithValueStartingToString", + "testAttributeWithValueToString", + "testContainsOwnTextToString", + "testContainsTextToString", + "caseInsensitiveHeaders" + ], + "score": { + "gitnext": { + "recall": 0.6956779159265838, + "precision": 0.9932375316990701, + "predicted": 1183, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.17998815867377146, + "precision": 1.0, + "predicted": 304, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5735294117647058, + "precision": 1.0, + "predicted": 39, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.7927767910005921, + "precision": 0.994060876020787, + "predicted": 1347, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9264705882352942, + "precision": 1.0, + "predicted": 63, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::SoftPool::release", + "file": "src/main/java/org/jsoup/internal/SoftPool.java", + "truth": 1755, + "truth_doctests": 0, + "predicted_gitnext": 834, + "predicted_coverage": 1730, + "missed_by_coverage": [ + "adoptionDoesNotAddSecondDocumentElement", + "canCreateEmptyConnection", + "caseInsensitiveHeaders", + "downgradesSocksProxyToUrlConnectionExecutor", + "emptyCookieJar", + "fixesHeaderEncodingIfRequired", + "getsHttpClient", + "getsHttpClientByDefault", + "handlesHeaderEncodingOnRequest", + "ignoresEmptySetCookies" + ], + "covered_not_failing": [ + "extendedValuesStillRejectMalformedQuotedSyntax", + "testSoftLocalPool", + "unknownPseudoNodeSelectError" + ], + "missed_by_coverage_n": 28, + "covered_not_failing_n": 3, + "top_coverage": [ + "testReleaseMoreThanMaxIdle", + "testSoftLocalPool", + "testSoftReferenceBehavior", + "concatCharacterAndString", + "unescape", + "unescape_2", + "encodeUrlSupplementary", + "join", + "normaliseWhiteSpace", + "urlControlCharsAreEncoded" + ], + "score": { + "gitnext": { + "recall": 0.46495726495726497, + "precision": 0.9784172661870504, + "predicted": 834, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9840455840455841, + "precision": 0.9982658959537573, + "predicted": 1730, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.9852941176470589, + "precision": 1.0, + "predicted": 67, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9840455840455841, + "precision": 0.9885518030910132, + "predicted": 1747, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.9852941176470589, + "precision": 1.0, + "predicted": 67, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::appendNormalisedWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 446, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "normaliseWhiteSpace", + "normaliseWhitespace", + "testContainsOwnTextToString", + "testContainsTextToString", + "testCloneAfterAttributesHit", + "collectionsCompareIdentityAndOrderNotJustCounts", + "allowsPopulatedSubtreeMove", + "createFromEncoded", + "testClone", + "queryTextIsReadableOnOneLine" + ], + "score": { + "gitnext": { + "recall": 0.289237668161435, + "precision": 0.9148936170212766, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5833333333333334, + "precision": 1.0, + "predicted": 28, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 446, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9737991266375546, + "predicted": 458, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isActuallyWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 132, + "predicted_coverage": 446, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "normaliseWhiteSpace", + "normaliseWhitespace", + "testContainsOwnTextToString", + "testContainsTextToString", + "testCloneAfterAttributesHit", + "collectionsCompareIdentityAndOrderNotJustCounts", + "allowsPopulatedSubtreeMove", + "createFromEncoded", + "testClone", + "queryTextIsReadableOnOneLine" + ], + "score": { + "gitnext": { + "recall": 0.27802690582959644, + "precision": 0.9393939393939394, + "predicted": 132, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5416666666666666, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 446, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9823788546255506, + "predicted": 454, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isAsciiLetter", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1699, + "truth_doctests": 0, + "predicted_gitnext": 39, + "predicted_coverage": 1696, + "missed_by_coverage": [ + "init", + "provideEvaluators", + "rcDataRecoverParser" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 3, + "covered_not_failing_n": 0, + "top_coverage": [ + "isAsciiLetter", + "validatesTextTagNames", + "consumeLetterThenDigitSequence", + "consumeLetterSequenceInAscii", + "ensureTagSearchesAreKnownTags", + "knownTags", + "copyPullThroughDoesNotMutateSource", + "equals", + "copyPullWithCustomizerThroughDoesNotMutateSource", + "valueOfChecksNotNull" + ], + "score": { + "gitnext": { + "recall": 0.020011771630370805, + "precision": 0.8717948717948718, + "predicted": 39, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.18461538461538463, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998234255444379, + "precision": 1.0, + "predicted": 1696, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998234255444379, + "precision": 0.9970605526161082, + "predicted": 1701, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::trimAsciiWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1811, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 1385, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accumulatesWhitespaceReferencesBeforeText", + "afterHeadReAdds", + "allDashCommentsAreNotParseErrors", + "appendThrowsSerializationException", + "beforeAfterRemoveInHeadTailsCurrentSlotAndHeadsFutureSiblings", + "bodyTextKeepsOneRange" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 426, + "covered_not_failing_n": 0, + "top_coverage": [ + "trimsOnlyAsciiWhitespace", + "attributeCaseNormalization", + "caseSupport", + "hasValue", + "orphanNamespace", + "validatesKeysNotEmpty", + "validatesKeysNotEmptyViaSet", + "settersOnOrphanAttribute", + "iteratorThrows", + "testAttributeWithValueContainingToString" + ], + "score": { + "gitnext": { + "recall": 0.07675317504141359, + "precision": 0.9858156028368794, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.42857142857142855, + "precision": 1.0, + "predicted": 30, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.7647708448371066, + "precision": 1.0, + "predicted": 1385, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9285714285714286, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.7680839315295417, + "precision": 0.9985642498205313, + "predicted": 1393, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9285714285714286, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::deduplicate", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 780, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 780, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "canSelectTextNodes", + "selectAncestorsOnContextElement" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 2, + "top_coverage": [ + "parseAttributesAcceptsCompleteInputAtEof", + "parseAttributesAllowsCloserInQuotedValue", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser", + "parseAttributesUsesParserSettings", + "attributeNamedLikeNodePreservesInstructionData", + "parseAttributesParsesNamesAndValues", + "stopsParsingAttributesAtCloser", + "parsesDataWhenAttributesAreAccessed", + "dataSetterUpdatesParsedAttributes" + ], + "score": { + "gitnext": { + "recall": 0.010256410256410256, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.02040816326530612, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9974358974358974, + "precision": 0.9974358974358974, + "predicted": 780, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9974358974358974, + "precision": 0.9974358974358974, + "predicted": 780, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::getIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 610, + "truth_doctests": 0, + "predicted_gitnext": 251, + "predicted_coverage": 610, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "classListIsImmutable", + "classListReturnsRawTokens", + "classNamesHandlesRoughWhitespaceAndDuplicates", + "handlesBaseUri", + "handlesAbsOnUnknownProtocols", + "testHasClassDomMethods", + "whiteSpaceClassElement", + "hasClassUsesHtmlWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.3540983606557377, + "precision": 0.8605577689243028, + "predicted": 251, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.6739130434782609, + "precision": 1.0, + "predicted": 31, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 610, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 46, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9457364341085271, + "predicted": 645, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 46, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::removeIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 28, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 28, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testRemoveAttr", + "leafNodeAttributes", + "attributeSizeDoesNotAutoVivify", + "earlierMatchesAreCachedOnlyWithinOneTraversal", + "testIteratorRemoveConcurrentException", + "dontCloneNullUserData", + "attributes", + "handlesControlCharactersAfterTagName", + "removeAttributeMaintainsFollowingRanges", + "testChainedRemoveAttributes" + ], + "score": { + "gitnext": { + "recall": 0.07142857142857142, + "precision": 0.06451612903225806, + "predicted": 31, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 0.18181818181818182, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 28, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.49122807017543857, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9166666666666666, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::DocumentType::publicId", + "file": "src/main/java/org/jsoup/nodes/DocumentType.java", + "truth": 115, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 115, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "outerHtmlGeneration", + "attributes", + "columnGroupEofProcessesUnclosedTemplate", + "handlesWhitespaceInoDocType", + "testPublicIdentifiersWithWhitespace", + "testPublicAndSystemIdentifiersWithWhitespace", + "testSystemIdentifiersWithWhitespace", + "testDocumentTypeGet", + "whitespaceBeforeHtml", + "whitespaceBeforeSystemIdentifierDoesNotError" + ], + "score": { + "gitnext": { + "recall": 0.02608695652173913, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 115, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 24, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9663865546218487, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 24, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Entities::isNamedEntity", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 51, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 51, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "notMissingMultis", + "notMissingSupplementals", + "noSpuriousDecodes", + "unescapeQueryParametersInTextOnly", + "unescapeEntitiesInAttributes", + "prefixMatch", + "unescape", + "unescapeTracksErrors", + "unescapeMultiChars", + "escape" + ], + "score": { + "gitnext": { + "recall": 0.1568627450980392, + "precision": 0.3333333333333333, + "predicted": 24, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 51, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7611940298507462, + "predicted": 67, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Parser::getMaxDepth", + "file": "src/main/java/org/jsoup/parser/Parser.java", + "truth": 1467, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 1465, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 0, + "top_coverage": [ + "fosterInsertionUsesDocumentHtmlWithoutTableOnStack", + "xmlParserEnablesXmlOutputAndEscapes", + "rootHasXmlSettings", + "NoErrorOnEofIfBodyOpen", + "readerClosedAfterParse", + "htmlParseDefaultsToHtmlOutputSyntax", + "modeSettings", + "commentAtEnd", + "malformedEndTagCommentsExcludeSolidus", + "customTagsAreFlyweights" + ], + "score": { + "gitnext": { + "recall": 0.0027266530334014998, + "precision": 0.4, + "predicted": 10, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 0.09836065573770492, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9986366734832992, + "precision": 1.0, + "predicted": 1465, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9986366734832992, + "precision": 0.9959211420802175, + "predicted": 1471, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Tag::isFormSubmittable", + "file": "src/main/java/org/jsoup/parser/Tag.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 16, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "formSubmittable", + "formDataUsesFirstAttribute", + "usesOnForCheckboxValueIfNoValueSet", + "controlsAddedAfterParseAreLinkedWithForms", + "emptyDropsDescendantLinksAndRetainsOutsideControls", + "formElementsAreLive", + "actionWithNoValue", + "adoptedFormsRetainInputs", + "removeFormElement", + "adoptedFormControlsCanBeMovedOutsideForm" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9411764705882353, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 16, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9411764705882353, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Token::Doctype::getSystemIdentifier", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 130, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 130, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "systemIdentifierUsesSafeQuote", + "unterminatedDoctypeSubsetConsumesToEof", + "doctypeEofReportsOnce", + "attributes", + "preservesEmptyDoctypeInternalSubset", + "doesNotCloseDoctypeSubsetOnQuotedGt", + "columnGroupEofProcessesUnclosedTemplate", + "doctypeNameNormalizationRespectsCaseSettings", + "handlesWhitespaceInoDocType", + "testPublicAndSystemIdentifiersWithWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 130, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 130, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::consumeCssIdentifier", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 383, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 385, + "missed_by_coverage": [], + "covered_not_failing": [ + "failedAlternateQueriesWarnWithoutThrowing", + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 2, + "top_coverage": [ + "consumeCssIdentifierWithEmptyInput", + "consumeCssIdentifier_invalidButSupportedForBackwardsCompatibility", + "consumeCssIdentifier_WebPlatformTests", + "consumeEscapedId", + "hyphenatedCssValuesAcrossBufferBoundary", + "consumeCssValueReplacesLeadingNull", + "unescapeCssIdentifier", + "invalidIdentifierEscapeAcrossBufferBoundary", + "validIdentifierEscapeAcrossBufferBoundary", + "unknownPseudoNodeSelectError" + ], + "score": { + "gitnext": { + "recall": 0.02349869451697128, + "precision": 0.9, + "predicted": 10, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.058823529411764705, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9948051948051948, + "predicted": 385, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9922279792746114, + "predicted": 386, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWhitespace", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1721, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 1023, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "covered_not_failing": [ + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 699, + "covered_not_failing_n": 1, + "top_coverage": [ + "consumeEscapedTag", + "invalidHyphenatedCssValuesAcrossBufferBoundary", + "consumeEscapedId", + "hyphenatedCssValuesAcrossBufferBoundary", + "invalidIdentifierEscapeAcrossBufferBoundary", + "validIdentifierEscapeAcrossBufferBoundary", + "unknownPseudoNodeSelectError", + "compilationModeRestoresEnclosingMode", + "compilationOverrideDoesNotChangeCheckMode", + "parseExceptionOnEmptyAbsKey" + ], + "score": { + "gitnext": { + "recall": 0.06624055781522371, + "precision": 0.957983193277311, + "predicted": 119, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.21212121212121213, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.5938407902382336, + "precision": 0.9990224828934506, + "predicted": 1023, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.8787878787878788, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.5938407902382336, + "precision": 0.9951314508276533, + "predicted": 1027, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.8787878787878788, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWord", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1717, + "truth_doctests": 0, + "predicted_gitnext": 22, + "predicted_coverage": 1019, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "covered_not_failing": [ + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 699, + "covered_not_failing_n": 1, + "top_coverage": [ + "escapeAtEof", + "consumeEscapedTag", + "unknownPseudoNodeSelectError", + "compilationModeRestoresEnclosingMode", + "compilationOverrideDoesNotChangeCheckMode", + "parseExceptionOnEmptyAbsKey", + "exceptOnUnhandledEvaluator", + "consecutiveCombinators", + "parseExceptionOnEmptyKeyVal", + "processingInstructionTargetErrors" + ], + "score": { + "gitnext": { + "recall": 0.0029120559114735, + "precision": 0.22727272727272727, + "predicted": 22, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.5928945835760047, + "precision": 0.9990186457311089, + "predicted": 1019, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.8787878787878788, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.5934769947582994, + "precision": 0.982642237222758, + "predicted": 1037, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.8787878787878788, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addEnforcedAttribute", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 38, + "truth_doctests": 0, + "predicted_gitnext": 38, + "predicted_coverage": 38, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "handlesFramesets", + "malformedRawtext", + "dropsScriptInsideNoscriptFallback", + "handlesControlCharactersAfterTagName", + "testHandlesEmptyAttributes", + "selfClosingVoidIsNotAnError", + "testH20" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addTags", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 72, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 72, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnTags", + "bailsIfRemovingProtocolThatsNotSet", + "testCopyConstructor_noSideEffectOnAttributes", + "namesUseAsciiCaseFolding", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "testCopyConstructor_noSideEffectOnProtocols", + "isSafeAttributeDoesNotModifyLiveAttribute", + "testScriptTagInSafeList", + "testIsValidDocument" + ], + "score": { + "gitnext": { + "recall": 0.9861111111111112, + "precision": 1.0, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 72, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 72, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::select::Evaluator::asPredicate", + "file": "src/main/java/org/jsoup/select/Evaluator.java", + "truth": 957, + "truth_doctests": 0, + "predicted_gitnext": 23, + "predicted_coverage": 957, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "unchangedAndNonCheckingQueriesAreSilent", + "nthLastChildWithNoParent", + "cssSelectorDoesntStackOverflow", + "searchesExcludeSelfButMatchingIncludesSelf", + "testMetaCharsetUpdateDisabled", + "parsesEmptyString", + "testMetaCharsetUpdateEnabledAfterCharsetChange", + "xmlParserCanUseUnlimitedDepth", + "xmlParserHasDefaultDepth", + "deepEndTagClosesElement" + ], + "score": { + "gitnext": { + "recall": 0.022988505747126436, + "precision": 0.9565217391304348, + "predicted": 23, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.12280701754385964, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 957, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9989561586638831, + "predicted": 958, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::ownerDocument", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 891, + "truth_doctests": 0, + "predicted_gitnext": 1012, + "predicted_coverage": 891, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testClone", + "testCloneAfterAttributesHit", + "classListIsImmutable", + "classListReturnsRawTokens", + "classNamesHandlesRoughWhitespaceAndDuplicates", + "hasClassUsesHtmlWhitespace", + "detachedNodeRetainsSiblingIndex", + "onlyChildIndexDoesNotReindexParent", + "lastChildIndexDoesNotReindexOtherSiblings", + "emptyDoesNotModifyAnAllocatedEmptyNodeList" + ], + "score": { + "gitnext": { + "recall": 0.8518518518518519, + "precision": 0.75, + "predicted": 1012, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.9629629629629629, + "precision": 0.896551724137931, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 891, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 54, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7788461538461539, + "predicted": 1144, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::NodeInternals::attributeRange", + "file": "src/main/java/org/jsoup/nodes/NodeInternals.java", + "truth": 39, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 70, + "missed_by_coverage": [], + "covered_not_failing": [ + "adoptedMixedChildrenRetainSourceRanges", + "adoptionRetainsOlderFormattingHistory", + "attributesKeepIdentityAndSourceRanges", + "equalsIgnoresInternalData", + "limitsStaleFormattingHistoryDuringParsing", + "parseAttributesTracksSourceRanges", + "rangesAndPositionsHaveValueEquality", + "reconstructionRetainsOlderHistoryUntilOverflow", + "recoveredRcDataRanges", + "removeAttributeMaintainsFollowingRanges" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 31, + "top_coverage": [ + "parseAttributesTracksSourceRanges", + "blankAbsAttr", + "tracksAdoptedFormattingAttributes", + "tracksFrag", + "mergeHtmlAttributesFromBody", + "tracksXml", + "normalisedBodyAfterContent", + "equalsIgnoresInternalData", + "tracksForeignAttributesWithAdjustedCase", + "tracksFirstAdjustedForeignAttribute" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 8, + "top_hits": 0, + "top_n": 8, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 1, + "top_hits": 0, + "top_n": 1, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5571428571428572, + "predicted": 70, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 10, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.5064935064935064, + "predicted": 77, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 10, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TokeniserState::read", + "file": "src/main/java/org/jsoup/parser/TokeniserState.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 27, + "predicted_coverage": 1497, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1496, + "top_coverage": [ + "resetsNullFlagOnReuse", + "nullReferences", + "dropsProcessingInstructionTargetAtEof", + "parseAttributesAcceptsCompleteInputAtEof", + "readerClosedAfterParse", + "parseAttributesAllowsCloserInQuotedValue", + "xmlParseDefaultsToHtmlOutputSyntax", + "whitespaceReferencePeekCrossesBufferBoundary", + "accumulatesWhitespaceReferencesBeforeText", + "parseAttributesReportsIncompleteValue" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 27, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 10, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.0006680026720106881, + "predicted": 1497, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.01639344262295082, + "predicted": 61, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0006626905235255136, + "predicted": 1509, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.016129032258064516, + "predicted": 62, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::helper::HttpConnection::Response::sameOrigin", + "file": "src/main/java/org/jsoup/helper/HttpConnection.java", + "truth": 19, + "truth_doctests": 0, + "predicted_gitnext": 44, + "predicted_coverage": 19, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "sameOriginUsesSchemeHostAndEffectivePort", + "handlesEmptyRedirect", + "handlesShortBrokenHttpRedirect", + "preservesHeadFor303", + "handlesRedirect", + "secFetchHeadersSurviveRedirect", + "clearsFormDataWhenRedirectingPostToGet", + "credentialsSurviveSameOriginRedirect", + "requestCookiesSurviveRedirect", + "credentialsDoNotCrossRedirectOrigins" + ], + "score": { + "gitnext": { + "recall": 0.10526315789473684, + "precision": 0.045454545454545456, + "predicted": 44, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 19, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.3114754098360656, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::outerHtml", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 116, + "truth_doctests": 0, + "predicted_gitnext": 128, + "predicted_coverage": 179, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreVerifiable", + "appendMustCorrectlyMoveChildrenInsideOneParentElement", + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "caseSensitiveDeclaration", + "cdataPreservesWhiteSpace", + "closesEmptyDoctypeBeforeBodyText", + "cssAndXpathEquivalents", + "doesNotGetAttributesTooEasily", + "emptyElementsIfNoResults" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 63, + "top_coverage": [ + "serializesForHtmlAndXml", + "outerHtmlGeneration", + "rejectsCyclicWholeSourceMove", + "testClone", + "testAddBooleanAttribute", + "rejectsAncestorAsChild", + "rejectsAncestorReplacement", + "rejectsCyclicBatchAtomically", + "foreignElementWithVoidNameRetainsChildren", + "htmlVoidPrintOmitsDomChildren" + ], + "score": { + "gitnext": { + "recall": 0.8879310344827587, + "precision": 0.8046875, + "predicted": 128, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9259259259259259, + "precision": 0.8620689655172413, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.6480446927374302, + "predicted": 179, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.84375, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.6373626373626373, + "predicted": 182, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.84375, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Element::TextAccumulator::hasBlockChild", + "file": "src/main/java/org/jsoup/nodes/Element.java", + "truth": 123, + "truth_doctests": 0, + "predicted_gitnext": 256, + "predicted_coverage": 123, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "allowsPopulatedSubtreeMove", + "normaliseWhiteSpaceHandlesHighSurrogates", + "testFarsi", + "cp1252Entities", + "textHasSpaceAfterBlockTags", + "textHasSpaceBetweenDivAndCenterTags", + "createsStructureFromBodySnippet", + "dropsUnterminatedTag", + "testKeepsPreText", + "testNormalisesText" + ], + "score": { + "gitnext": { + "recall": 0.21138211382113822, + "precision": 0.1015625, + "predicted": 256, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.2692307692307692, + "precision": 0.7, + "predicted": 10, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 123, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.34844192634560905, + "predicted": 353, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.896551724137931, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::setParentNode", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 1554, + "truth_doctests": 0, + "predicted_gitnext": 338, + "predicted_coverage": 1552, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 0, + "top_coverage": [ + "rejectsSelfParenting", + "testHashcodeIsStableWithContentChanges", + "detachedNodeRetainsSiblingIndex", + "onlyChildIndexDoesNotReindexParent", + "lastChildIndexDoesNotReindexOtherSiblings", + "emptyDoesNotModifyAnAllocatedEmptyNodeList", + "movingSiblingBeforeLastChildPreservesNavigation", + "indexedInsertionPreservesValidSiblingIndexes", + "indexedSiblingMoveLeavesIndexesForRepair", + "repeatedInsertionBeforeTablePreservesValidSiblingIndexes" + ], + "score": { + "gitnext": { + "recall": 0.21171171171171171, + "precision": 0.9733727810650887, + "predicted": 338, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.6612903225806451, + "precision": 1.0, + "predicted": 41, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9987129987129987, + "precision": 1.0, + "predicted": 1552, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9987129987129987, + "precision": 0.994234465086483, + "predicted": 1561, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Document::text", + "file": "src/main/java/org/jsoup/nodes/Document.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 140, + "predicted_coverage": 1, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "setTextPreservesDocumentStructure" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.007142857142857143, + "predicted": 140, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.007142857142857143, + "predicted": 140, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::DataNode::outerHtmlHead", + "file": "src/main/java/org/jsoup/nodes/DataNode.java", + "truth": 37, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 37, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "xmlOutputOrphanWithoutCData", + "xmlOutputOtherWithCData", + "xmlOutputScriptWithCData", + "xmlOutputStyleWithCData", + "xmlOutputOtherWithoutCData", + "xmlOutputScriptWithoutCData", + "xmlOutputStyleWithoutCData", + "customDataTagWithHyphenXml", + "handlesQuotesInCommentsInScripts", + "testNoIndentOnScriptAndStyle" + ], + "score": { + "gitnext": { + "recall": 0.1891891891891892, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.07692307692307693, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 37, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 37, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::Token::Character::normalizeNulls", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 1090, + "truth_doctests": 0, + "predicted_gitnext": 3, + "predicted_coverage": 1093, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "notNamespaceAware", + "supportsLocalname", + "supportsPrefixes" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 5, + "top_coverage": [ + "normalizesNulls", + "copiesNullFlag", + "readerClosedAfterParse", + "htmlParseDefaultsToHtmlOutputSyntax", + "testLocationFromString", + "DocumentsWithSameContentAreEqual", + "seesDocRoot", + "canSpecifyOnlyHead", + "optionalPClosersAreNotErrors", + "normaliseWhiteSpaceHandlesHighSurrogates" + ], + "score": { + "gitnext": { + "recall": 0.0027522935779816515, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.01694915254237288, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998165137614679, + "precision": 0.9954254345837146, + "predicted": 1093, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998165137614679, + "precision": 0.9954254345837146, + "predicted": 1093, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TreeBuilder::parseAttributes", + "file": "src/main/java/org/jsoup/parser/TreeBuilder.java", + "truth": 29, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 29, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "parseAttributesAcceptsCompleteInputAtEof", + "parseAttributesAllowsCloserInQuotedValue", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser", + "parseAttributesUsesParserSettings", + "attributeNamedLikeNodePreservesInstructionData", + "parseAttributesParsesNamesAndValues", + "serializesAttributesWithUniqueRepairedNames", + "stopsParsingAttributesAtCloser", + "parsesDataWhenAttributesAreAccessed" + ], + "score": { + "gitnext": { + "recall": 0.5862068965517241, + "precision": 0.7083333333333334, + "predicted": 24, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.5714285714285714, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8055555555555556, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7777777777777778, + "predicted": 9, + "top_hits": 7, + "top_n": 9, + "top_precision": 0.7777777777777778, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Entities::encoderFor", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 573, + "truth_doctests": 0, + "predicted_gitnext": 11, + "predicted_coverage": 586, + "missed_by_coverage": [], + "covered_not_failing": [ + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "cssAndXpathEquivalents", + "emptyElementsIfNoResults", + "notNamespaceAware", + "selectAncestorsOnContextElement", + "selectByAttribute", + "selectOutsideOfElementTree", + "supportsLocalname", + "supportsXpath" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 13, + "top_coverage": [ + "escapeDefaults", + "escapeByClonedOutputSettings", + "escapeSupplementaryCharacter", + "escapeSupplementaryNonUtf", + "escapeSupplementarySequenceNonUtf", + "escapedSupplementary", + "htmlWithLtAndGtInValue", + "html", + "testWithSupplementaryCharacterInAttributeKeyAndValue", + "htmlSkipsInternals" + ], + "score": { + "gitnext": { + "recall": 0.019197207678883072, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9778156996587031, + "predicted": 586, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 44, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9778156996587031, + "predicted": 586, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 44, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.264, + "precision": 0.703, + "top_precision": 0.692, + "caught": 0.905 + }, + "coverage": { + "recall": 0.91, + "precision": 0.999, + "top_precision": 0.995, + "caught": 1.0 + }, + "union": { + "recall": 0.939, + "precision": 0.952, + "top_precision": 0.995, + "caught": 1.0 + }, + "functions": 21 + }, + "random": { + "gitnext": { + "recall": 0.339, + "precision": 0.533, + "top_precision": 0.617, + "caught": 0.833 + }, + "coverage": { + "recall": 1.0, + "precision": 0.848, + "top_precision": 0.858, + "caught": 0.917 + }, + "union": { + "recall": 1.0, + "precision": 0.614, + "top_precision": 0.783, + "caught": 0.917 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.291, + "precision": 0.641, + "top_precision": 0.665, + "caught": 0.879 + }, + "coverage": { + "recall": 0.943, + "precision": 0.944, + "top_precision": 0.945, + "caught": 0.97 + }, + "union": { + "recall": 0.961, + "precision": 0.829, + "top_precision": 0.918, + "caught": 0.97 + }, + "functions": 33 + } + }, + "cost": { + "how": "per-test", + "class_inits_s": 3.6, + "class_inits_booked": true, + "files_with_init_lines": 64, + "mvn_s": 20.2, + "analyze_s": 25.7, + "map_s": 49.6, + "exec_bytes": 8545291, + "dumps": 5372, + "test_methods": 2130, + "classes": 76, + "classes_with_setup_lines": 6, + "run_failed": false, + "names_in_several_classes": 115, + "junit": 5, + "package": "org.jsoup", + "owner": "body", + "repeat": 3, + "plain_s": 14.2, + "tests": 2130, + "map_pairs": 556456, + "map_functions": 1771, + "map_sqlite_bytes": 5718016, + "functions_with_body": 2299, + "functions_never_run": 99, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 3.5 + } + }, + "per-test-fork": { + "repo": "jsoup", + "lang": "java", + "commit": "d90d57521d93", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::jsoup::internal::LegacyMatching::cssAttributeMatches", + "file": "src/main/java/org/jsoup/internal/LegacyMatching.java", + "truth": 7, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 7, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "qualifiedAttributeNamesUseTheSameLiteralAndRegexParsing", + "parsedAttributeValuesRoundTrip", + "aLoneOperandIsNotAnExplicitFlag", + "absoluteAttributeLookupIgnoresKeyCase", + "hyphenatedAttributeValueAtBufferBoundary", + "explicitCompilationCapturesTheWholeQuery", + "explicitFlagsOverrideDefaultsInBothModes" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::Normalizer::lowerCase", + "file": "src/main/java/org/jsoup/internal/Normalizer.java", + "truth": 1689, + "truth_doctests": 0, + "predicted_gitnext": 1183, + "predicted_coverage": 355, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 1334, + "covered_not_failing_n": 0, + "top_coverage": [ + "separatesAsciiAndUnicodeCase", + "testContainsDataToString", + "testAttributeWithValueContainingToString", + "testAttributeWithValueEndingToString", + "testAttributeWithValueNotToString", + "testAttributeWithValueStartingToString", + "testAttributeWithValueToString", + "testContainsOwnTextToString", + "testContainsTextToString", + "caseInsensitiveHeaders" + ], + "score": { + "gitnext": { + "recall": 0.6956779159265838, + "precision": 0.9932375316990701, + "predicted": 1183, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.21018354055654234, + "precision": 1.0, + "predicted": 355, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9558823529411765, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.8040260509177027, + "precision": 0.9941434846266471, + "predicted": 1366, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9852941176470589, + "precision": 1.0, + "predicted": 67, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::SoftPool::release", + "file": "src/main/java/org/jsoup/internal/SoftPool.java", + "truth": 1755, + "truth_doctests": 0, + "predicted_gitnext": 834, + "predicted_coverage": 1731, + "missed_by_coverage": [ + "adoptionDoesNotAddSecondDocumentElement", + "canCreateEmptyConnection", + "caseInsensitiveHeaders", + "downgradesSocksProxyToUrlConnectionExecutor", + "emptyCookieJar", + "fixesHeaderEncodingIfRequired", + "getsHttpClient", + "getsHttpClientByDefault", + "handlesHeaderEncodingOnRequest", + "ignoresEmptySetCookies" + ], + "covered_not_failing": [ + "extendedValuesStillRejectMalformedQuotedSyntax", + "testSoftLocalPool", + "unknownPseudoNodeSelectError" + ], + "missed_by_coverage_n": 27, + "covered_not_failing_n": 3, + "top_coverage": [ + "testReleaseMoreThanMaxIdle", + "testSoftLocalPool", + "testSoftReferenceBehavior", + "concatCharacterAndString", + "unescape", + "unescape_2", + "encodeUrlSupplementary", + "join", + "normaliseWhiteSpace", + "urlControlCharsAreEncoded" + ], + "score": { + "gitnext": { + "recall": 0.46495726495726497, + "precision": 0.9784172661870504, + "predicted": 834, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9846153846153847, + "precision": 0.9982668977469671, + "predicted": 1731, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9846153846153847, + "precision": 0.988558352402746, + "predicted": 1748, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::appendNormalisedWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 446, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "normaliseWhiteSpace", + "normaliseWhitespace", + "testContainsOwnTextToString", + "testContainsTextToString", + "testCloneAfterAttributesHit", + "collectionsCompareIdentityAndOrderNotJustCounts", + "allowsPopulatedSubtreeMove", + "createFromEncoded", + "testClone", + "queryTextIsReadableOnOneLine" + ], + "score": { + "gitnext": { + "recall": 0.289237668161435, + "precision": 0.9148936170212766, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5833333333333334, + "precision": 1.0, + "predicted": 28, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 446, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9737991266375546, + "predicted": 458, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isActuallyWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 132, + "predicted_coverage": 446, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "normaliseWhiteSpace", + "normaliseWhitespace", + "testContainsOwnTextToString", + "testContainsTextToString", + "testCloneAfterAttributesHit", + "collectionsCompareIdentityAndOrderNotJustCounts", + "allowsPopulatedSubtreeMove", + "createFromEncoded", + "testClone", + "queryTextIsReadableOnOneLine" + ], + "score": { + "gitnext": { + "recall": 0.27802690582959644, + "precision": 0.9393939393939394, + "predicted": 132, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5416666666666666, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 446, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9823788546255506, + "predicted": 454, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 48, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isAsciiLetter", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1699, + "truth_doctests": 0, + "predicted_gitnext": 39, + "predicted_coverage": 1696, + "missed_by_coverage": [ + "init", + "provideEvaluators", + "rcDataRecoverParser" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 3, + "covered_not_failing_n": 0, + "top_coverage": [ + "isAsciiLetter", + "validatesTextTagNames", + "consumeLetterThenDigitSequence", + "consumeLetterSequenceInAscii", + "ensureTagSearchesAreKnownTags", + "knownTags", + "copyPullThroughDoesNotMutateSource", + "equals", + "copyPullWithCustomizerThroughDoesNotMutateSource", + "valueOfChecksNotNull" + ], + "score": { + "gitnext": { + "recall": 0.020011771630370805, + "precision": 0.8717948717948718, + "predicted": 39, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.18461538461538463, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998234255444379, + "precision": 1.0, + "predicted": 1696, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998234255444379, + "precision": 0.9970605526161082, + "predicted": 1701, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::trimAsciiWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1811, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 1407, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accumulatesWhitespaceReferencesBeforeText", + "afterHeadReAdds", + "allDashCommentsAreNotParseErrors", + "beforeAfterRemoveInHeadTailsCurrentSlotAndHeadsFutureSiblings", + "bodyTextKeepsOneRange", + "bookmark" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 404, + "covered_not_failing_n": 0, + "top_coverage": [ + "trimsOnlyAsciiWhitespace", + "attributeCaseNormalization", + "caseSupport", + "hasValue", + "orphanNamespace", + "validatesKeysNotEmpty", + "validatesKeysNotEmptyViaSet", + "settersOnOrphanAttribute", + "iteratorThrows", + "testAttributeWithValueContainingToString" + ], + "score": { + "gitnext": { + "recall": 0.07675317504141359, + "precision": 0.9858156028368794, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.42857142857142855, + "precision": 1.0, + "predicted": 30, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.7769188293760353, + "precision": 1.0, + "predicted": 1407, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.7796797349530646, + "precision": 0.9985855728429985, + "predicted": 1414, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::deduplicate", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 780, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 780, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "canSelectTextNodes", + "selectAncestorsOnContextElement" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 2, + "top_coverage": [ + "parseAttributesAcceptsCompleteInputAtEof", + "parseAttributesAllowsCloserInQuotedValue", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser", + "parseAttributesUsesParserSettings", + "attributeNamedLikeNodePreservesInstructionData", + "parseAttributesParsesNamesAndValues", + "stopsParsingAttributesAtCloser", + "parsesDataWhenAttributesAreAccessed", + "dataSetterUpdatesParsedAttributes" + ], + "score": { + "gitnext": { + "recall": 0.010256410256410256, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.02040816326530612, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9974358974358974, + "precision": 0.9974358974358974, + "predicted": 780, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9974358974358974, + "precision": 0.9974358974358974, + "predicted": 780, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::getIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 610, + "truth_doctests": 0, + "predicted_gitnext": 251, + "predicted_coverage": 610, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "classListIsImmutable", + "classListReturnsRawTokens", + "classNamesHandlesRoughWhitespaceAndDuplicates", + "handlesBaseUri", + "handlesAbsOnUnknownProtocols", + "testHasClassDomMethods", + "whiteSpaceClassElement", + "hasClassUsesHtmlWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.3540983606557377, + "precision": 0.8605577689243028, + "predicted": 251, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.6739130434782609, + "precision": 1.0, + "predicted": 31, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 610, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 46, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9457364341085271, + "predicted": 645, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 46, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::removeIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 28, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 28, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testRemoveAttr", + "leafNodeAttributes", + "attributeSizeDoesNotAutoVivify", + "earlierMatchesAreCachedOnlyWithinOneTraversal", + "testIteratorRemoveConcurrentException", + "dontCloneNullUserData", + "attributes", + "handlesControlCharactersAfterTagName", + "removeAttributeMaintainsFollowingRanges", + "testChainedRemoveAttributes" + ], + "score": { + "gitnext": { + "recall": 0.07142857142857142, + "precision": 0.06451612903225806, + "predicted": 31, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 0.18181818181818182, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 28, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.49122807017543857, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9166666666666666, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::DocumentType::publicId", + "file": "src/main/java/org/jsoup/nodes/DocumentType.java", + "truth": 115, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 115, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "outerHtmlGeneration", + "attributes", + "columnGroupEofProcessesUnclosedTemplate", + "handlesWhitespaceInoDocType", + "testPublicIdentifiersWithWhitespace", + "testPublicAndSystemIdentifiersWithWhitespace", + "testSystemIdentifiersWithWhitespace", + "testDocumentTypeGet", + "whitespaceBeforeHtml", + "whitespaceBeforeSystemIdentifierDoesNotError" + ], + "score": { + "gitnext": { + "recall": 0.02608695652173913, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 115, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 24, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9663865546218487, + "predicted": 119, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 24, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Entities::isNamedEntity", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 51, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 51, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "notMissingMultis", + "notMissingSupplementals", + "noSpuriousDecodes", + "unescapeEntitiesInAttributes", + "prefixMatch", + "unescape", + "unescapeTracksErrors", + "unescapeMultiChars", + "escape", + "accumulatesWhitespaceReferencesBeforeText" + ], + "score": { + "gitnext": { + "recall": 0.1568627450980392, + "precision": 0.3333333333333333, + "predicted": 24, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 51, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7611940298507462, + "predicted": 67, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 18, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Parser::getMaxDepth", + "file": "src/main/java/org/jsoup/parser/Parser.java", + "truth": 1467, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 1465, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 0, + "top_coverage": [ + "fosterInsertionUsesDocumentHtmlWithoutTableOnStack", + "xmlParserEnablesXmlOutputAndEscapes", + "rootHasXmlSettings", + "NoErrorOnEofIfBodyOpen", + "readerClosedAfterParse", + "htmlParseDefaultsToHtmlOutputSyntax", + "modeSettings", + "commentAtEnd", + "malformedEndTagCommentsExcludeSolidus", + "customTagsAreFlyweights" + ], + "score": { + "gitnext": { + "recall": 0.0027266530334014998, + "precision": 0.4, + "predicted": 10, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 0.09836065573770492, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9986366734832992, + "precision": 1.0, + "predicted": 1465, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9986366734832992, + "precision": 0.9959211420802175, + "predicted": 1471, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Tag::isFormSubmittable", + "file": "src/main/java/org/jsoup/parser/Tag.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 16, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "formSubmittable", + "formDataUsesFirstAttribute", + "usesOnForCheckboxValueIfNoValueSet", + "controlsAddedAfterParseAreLinkedWithForms", + "formElementsAreLive", + "actionWithNoValue", + "adoptedFormsRetainInputs", + "removeFormElement", + "emptyDropsDescendantLinksAndRetainsOutsideControls", + "adoptedFormControlsCanBeMovedOutsideForm" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9411764705882353, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 16, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9411764705882353, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Token::Doctype::getSystemIdentifier", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 130, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 130, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "systemIdentifierUsesSafeQuote", + "unterminatedDoctypeSubsetConsumesToEof", + "doctypeEofReportsOnce", + "attributes", + "preservesEmptyDoctypeInternalSubset", + "doesNotCloseDoctypeSubsetOnQuotedGt", + "columnGroupEofProcessesUnclosedTemplate", + "doctypeNameNormalizationRespectsCaseSettings", + "handlesWhitespaceInoDocType", + "testPublicAndSystemIdentifiersWithWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 130, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 130, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::consumeCssIdentifier", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 383, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 385, + "missed_by_coverage": [], + "covered_not_failing": [ + "failedAlternateQueriesWarnWithoutThrowing", + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 2, + "top_coverage": [ + "consumeCssIdentifierWithEmptyInput", + "consumeCssIdentifier_invalidButSupportedForBackwardsCompatibility", + "consumeCssIdentifier_WebPlatformTests", + "consumeEscapedId", + "hyphenatedCssValuesAcrossBufferBoundary", + "consumeCssValueReplacesLeadingNull", + "unescapeCssIdentifier", + "invalidIdentifierEscapeAcrossBufferBoundary", + "validIdentifierEscapeAcrossBufferBoundary", + "unknownPseudoNodeSelectError" + ], + "score": { + "gitnext": { + "recall": 0.02349869451697128, + "precision": 0.9, + "predicted": 10, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.058823529411764705, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9948051948051948, + "predicted": 385, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9922279792746114, + "predicted": 386, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 34, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWhitespace", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1721, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 1052, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "covered_not_failing": [ + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 670, + "covered_not_failing_n": 1, + "top_coverage": [ + "consumeEscapedTag", + "invalidHyphenatedCssValuesAcrossBufferBoundary", + "consumeEscapedId", + "hyphenatedCssValuesAcrossBufferBoundary", + "invalidIdentifierEscapeAcrossBufferBoundary", + "validIdentifierEscapeAcrossBufferBoundary", + "unknownPseudoNodeSelectError", + "compilationModeRestoresEnclosingMode", + "compilationOverrideDoesNotChangeCheckMode", + "parseExceptionOnEmptyAbsKey" + ], + "score": { + "gitnext": { + "recall": 0.06624055781522371, + "precision": 0.957983193277311, + "predicted": 119, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.21212121212121213, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.610691458454387, + "precision": 0.9990494296577946, + "predicted": 1052, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9848484848484849, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.610691458454387, + "precision": 0.9952651515151515, + "predicted": 1056, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9848484848484849, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWord", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1717, + "truth_doctests": 0, + "predicted_gitnext": 22, + "predicted_coverage": 1048, + "missed_by_coverage": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "covered_not_failing": [ + "queryTextIsReadableOnOneLine" + ], + "missed_by_coverage_n": 670, + "covered_not_failing_n": 1, + "top_coverage": [ + "escapeAtEof", + "consumeEscapedTag", + "unknownPseudoNodeSelectError", + "compilationModeRestoresEnclosingMode", + "compilationOverrideDoesNotChangeCheckMode", + "parseExceptionOnEmptyAbsKey", + "exceptOnUnhandledEvaluator", + "consecutiveCombinators", + "parseExceptionOnEmptyKeyVal", + "processingInstructionTargetErrors" + ], + "score": { + "gitnext": { + "recall": 0.0029120559114735, + "precision": 0.22727272727272727, + "predicted": 22, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.609784507862551, + "precision": 0.9990458015267175, + "predicted": 1048, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9848484848484849, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.6103669190448456, + "precision": 0.9831144465290806, + "predicted": 1066, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9848484848484849, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addEnforcedAttribute", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 38, + "truth_doctests": 0, + "predicted_gitnext": 38, + "predicted_coverage": 38, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "handlesFramesets", + "malformedRawtext", + "dropsScriptInsideNoscriptFallback", + "handlesControlCharactersAfterTagName", + "testHandlesEmptyAttributes", + "selfClosingVoidIsNotAnError", + "testH20" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addTags", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 72, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 72, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnTags", + "bailsIfRemovingProtocolThatsNotSet", + "testCopyConstructor_noSideEffectOnAttributes", + "namesUseAsciiCaseFolding", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "testCopyConstructor_noSideEffectOnProtocols", + "isSafeAttributeDoesNotModifyLiveAttribute", + "testScriptTagInSafeList", + "testIsValidDocument" + ], + "score": { + "gitnext": { + "recall": 0.9861111111111112, + "precision": 1.0, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 72, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 72, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::select::Evaluator::asPredicate", + "file": "src/main/java/org/jsoup/select/Evaluator.java", + "truth": 957, + "truth_doctests": 0, + "predicted_gitnext": 23, + "predicted_coverage": 957, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "unchangedAndNonCheckingQueriesAreSilent", + "nthLastChildWithNoParent", + "cssSelectorDoesntStackOverflow", + "searchesExcludeSelfButMatchingIncludesSelf", + "testMetaCharsetUpdateDisabled", + "parsesEmptyString", + "testMetaCharsetUpdateEnabledAfterCharsetChange", + "xmlParserCanUseUnlimitedDepth", + "xmlParserHasDefaultDepth", + "deepEndTagClosesElement" + ], + "score": { + "gitnext": { + "recall": 0.022988505747126436, + "precision": 0.9565217391304348, + "predicted": 23, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.12280701754385964, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 957, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9989561586638831, + "predicted": 958, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 57, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::ownerDocument", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 891, + "truth_doctests": 0, + "predicted_gitnext": 1012, + "predicted_coverage": 891, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "testClone", + "testCloneAfterAttributesHit", + "classListIsImmutable", + "classListReturnsRawTokens", + "classNamesHandlesRoughWhitespaceAndDuplicates", + "hasClassUsesHtmlWhitespace", + "detachedNodeRetainsSiblingIndex", + "onlyChildIndexDoesNotReindexParent", + "lastChildIndexDoesNotReindexOtherSiblings", + "emptyDoesNotModifyAnAllocatedEmptyNodeList" + ], + "score": { + "gitnext": { + "recall": 0.8518518518518519, + "precision": 0.75, + "predicted": 1012, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.9629629629629629, + "precision": 0.896551724137931, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 891, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 54, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.7788461538461539, + "predicted": 1144, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::NodeInternals::attributeRange", + "file": "src/main/java/org/jsoup/nodes/NodeInternals.java", + "truth": 39, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 70, + "missed_by_coverage": [], + "covered_not_failing": [ + "adoptedMixedChildrenRetainSourceRanges", + "adoptionRetainsOlderFormattingHistory", + "attributesKeepIdentityAndSourceRanges", + "equalsIgnoresInternalData", + "limitsStaleFormattingHistoryDuringParsing", + "parseAttributesTracksSourceRanges", + "rangesAndPositionsHaveValueEquality", + "reconstructionRetainsOlderHistoryUntilOverflow", + "recoveredRcDataRanges", + "removeAttributeMaintainsFollowingRanges" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 31, + "top_coverage": [ + "parseAttributesTracksSourceRanges", + "blankAbsAttr", + "tracksAdoptedFormattingAttributes", + "tracksFrag", + "mergeHtmlAttributesFromBody", + "tracksXml", + "normalisedBodyAfterContent", + "equalsIgnoresInternalData", + "tracksForeignAttributesWithAdjustedCase", + "tracksFirstAdjustedForeignAttribute" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 8, + "top_hits": 0, + "top_n": 8, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 1, + "top_hits": 0, + "top_n": 1, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5571428571428572, + "predicted": 70, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 10, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.5064935064935064, + "predicted": 77, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 10, + "top_hits": 5, + "top_n": 10, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TokeniserState::read", + "file": "src/main/java/org/jsoup/parser/TokeniserState.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 27, + "predicted_coverage": 1497, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1496, + "top_coverage": [ + "resetsNullFlagOnReuse", + "dropsProcessingInstructionTargetAtEof", + "parseAttributesAcceptsCompleteInputAtEof", + "readerClosedAfterParse", + "parseAttributesAllowsCloserInQuotedValue", + "xmlParseDefaultsToHtmlOutputSyntax", + "whitespaceReferencePeekCrossesBufferBoundary", + "accumulatesWhitespaceReferencesBeforeText", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 27, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 10, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.0006680026720106881, + "predicted": 1497, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.01639344262295082, + "predicted": 61, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0006626905235255136, + "predicted": 1509, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.016129032258064516, + "predicted": 62, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::helper::HttpConnection::Response::sameOrigin", + "file": "src/main/java/org/jsoup/helper/HttpConnection.java", + "truth": 19, + "truth_doctests": 0, + "predicted_gitnext": 44, + "predicted_coverage": 19, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "sameOriginUsesSchemeHostAndEffectivePort", + "handlesEmptyRedirect", + "handlesShortBrokenHttpRedirect", + "preservesHeadFor303", + "handlesRedirect", + "secFetchHeadersSurviveRedirect", + "clearsFormDataWhenRedirectingPostToGet", + "credentialsSurviveSameOriginRedirect", + "requestCookiesSurviveRedirect", + "credentialsDoNotCrossRedirectOrigins" + ], + "score": { + "gitnext": { + "recall": 0.10526315789473684, + "precision": 0.045454545454545456, + "predicted": 44, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 19, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.3114754098360656, + "predicted": 61, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::outerHtml", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 116, + "truth_doctests": 0, + "predicted_gitnext": 128, + "predicted_coverage": 179, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreVerifiable", + "appendMustCorrectlyMoveChildrenInsideOneParentElement", + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "caseSensitiveDeclaration", + "cdataPreservesWhiteSpace", + "closesEmptyDoctypeBeforeBodyText", + "cssAndXpathEquivalents", + "doesNotGetAttributesTooEasily", + "emptyElementsIfNoResults" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 63, + "top_coverage": [ + "serializesForHtmlAndXml", + "rejectsCyclicWholeSourceMove", + "testClone", + "testAddBooleanAttribute", + "rejectsAncestorAsChild", + "rejectsAncestorReplacement", + "rejectsCyclicBatchAtomically", + "foreignElementWithVoidNameRetainsChildren", + "htmlVoidPrintOmitsDomChildren", + "htmlAppendableWritesInnerAndOuterHtml" + ], + "score": { + "gitnext": { + "recall": 0.8879310344827587, + "precision": 0.8046875, + "predicted": 128, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9259259259259259, + "precision": 0.8620689655172413, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.6480446927374302, + "predicted": 179, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.84375, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.6373626373626373, + "predicted": 182, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.84375, + "predicted": 32, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Element::TextAccumulator::hasBlockChild", + "file": "src/main/java/org/jsoup/nodes/Element.java", + "truth": 123, + "truth_doctests": 0, + "predicted_gitnext": 256, + "predicted_coverage": 123, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "allowsPopulatedSubtreeMove", + "testFarsi", + "cp1252Entities", + "textHasSpaceAfterBlockTags", + "textHasSpaceBetweenDivAndCenterTags", + "createsStructureFromBodySnippet", + "dropsUnterminatedTag", + "testKeepsPreText", + "testNormalisesText", + "cp1252EntitiesProduceError" + ], + "score": { + "gitnext": { + "recall": 0.21138211382113822, + "precision": 0.1015625, + "predicted": 256, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.2692307692307692, + "precision": 0.7, + "predicted": 10, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 123, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.34844192634560905, + "predicted": 353, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.896551724137931, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::setParentNode", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 1554, + "truth_doctests": 0, + "predicted_gitnext": 338, + "predicted_coverage": 1552, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 0, + "top_coverage": [ + "rejectsSelfParenting", + "testHashcodeIsStableWithContentChanges", + "detachedNodeRetainsSiblingIndex", + "onlyChildIndexDoesNotReindexParent", + "lastChildIndexDoesNotReindexOtherSiblings", + "emptyDoesNotModifyAnAllocatedEmptyNodeList", + "movingSiblingBeforeLastChildPreservesNavigation", + "indexedInsertionPreservesValidSiblingIndexes", + "indexedSiblingMoveLeavesIndexesForRepair", + "repeatedInsertionBeforeTablePreservesValidSiblingIndexes" + ], + "score": { + "gitnext": { + "recall": 0.21171171171171171, + "precision": 0.9733727810650887, + "predicted": 338, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.6612903225806451, + "precision": 1.0, + "predicted": 41, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9987129987129987, + "precision": 1.0, + "predicted": 1552, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9987129987129987, + "precision": 0.994234465086483, + "predicted": 1561, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Document::text", + "file": "src/main/java/org/jsoup/nodes/Document.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 140, + "predicted_coverage": 1, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "setTextPreservesDocumentStructure" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.007142857142857143, + "predicted": 140, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.007142857142857143, + "predicted": 140, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::DataNode::outerHtmlHead", + "file": "src/main/java/org/jsoup/nodes/DataNode.java", + "truth": 37, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 37, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "xmlOutputOrphanWithoutCData", + "xmlOutputOtherWithCData", + "xmlOutputScriptWithCData", + "xmlOutputStyleWithCData", + "xmlOutputScriptWithoutCData", + "xmlOutputStyleWithoutCData", + "xmlOutputOtherWithoutCData", + "customDataTagWithHyphenXml", + "handlesQuotesInCommentsInScripts", + "testNoIndentOnScriptAndStyle" + ], + "score": { + "gitnext": { + "recall": 0.1891891891891892, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.07692307692307693, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 37, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 1.0, + "predicted": 37, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 13, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::Token::Character::normalizeNulls", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 1090, + "truth_doctests": 0, + "predicted_gitnext": 3, + "predicted_coverage": 1093, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "notNamespaceAware", + "supportsLocalname", + "supportsPrefixes" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 5, + "top_coverage": [ + "normalizesNulls", + "copiesNullFlag", + "readerClosedAfterParse", + "htmlParseDefaultsToHtmlOutputSyntax", + "testLocationFromString", + "DocumentsWithSameContentAreEqual", + "seesDocRoot", + "canSpecifyOnlyHead", + "optionalPClosersAreNotErrors", + "handlesManyChildren" + ], + "score": { + "gitnext": { + "recall": 0.0027522935779816515, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.01694915254237288, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998165137614679, + "precision": 0.9954254345837146, + "predicted": 1093, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998165137614679, + "precision": 0.9954254345837146, + "predicted": 1093, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TreeBuilder::parseAttributes", + "file": "src/main/java/org/jsoup/parser/TreeBuilder.java", + "truth": 29, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 29, + "missed_by_coverage": [], + "covered_not_failing": [], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 0, + "top_coverage": [ + "parseAttributesAcceptsCompleteInputAtEof", + "parseAttributesAllowsCloserInQuotedValue", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser", + "parseAttributesUsesParserSettings", + "attributeNamedLikeNodePreservesInstructionData", + "parseAttributesParsesNamesAndValues", + "serializesAttributesWithUniqueRepairedNames", + "stopsParsingAttributesAtCloser", + "parsesDataWhenAttributesAreAccessed" + ], + "score": { + "gitnext": { + "recall": 0.5862068965517241, + "precision": 0.7083333333333334, + "predicted": 24, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.5714285714285714, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 1.0, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.8055555555555556, + "predicted": 36, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7777777777777778, + "predicted": 9, + "top_hits": 7, + "top_n": 9, + "top_precision": 0.7777777777777778, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Entities::encoderFor", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 573, + "truth_doctests": 0, + "predicted_gitnext": 11, + "predicted_coverage": 586, + "missed_by_coverage": [], + "covered_not_failing": [ + "canDitchNamespaces", + "canSupplyAlternateFactoryImpl", + "cssAndXpathEquivalents", + "emptyElementsIfNoResults", + "notNamespaceAware", + "selectAncestorsOnContextElement", + "selectByAttribute", + "selectOutsideOfElementTree", + "supportsLocalname", + "supportsXpath" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 13, + "top_coverage": [ + "escapeDefaults", + "escapeByClonedOutputSettings", + "escapeSupplementaryCharacter", + "escapeSupplementaryNonUtf", + "escapeSupplementarySequenceNonUtf", + "escapedSupplementary", + "htmlWithLtAndGtInValue", + "html", + "testWithSupplementaryCharacterInAttributeKeyAndValue", + "htmlSkipsInternals" + ], + "score": { + "gitnext": { + "recall": 0.019197207678883072, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.9778156996587031, + "predicted": 586, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 44, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.9778156996587031, + "predicted": 586, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 44, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.264, + "precision": 0.703, + "top_precision": 0.692, + "caught": 0.905 + }, + "coverage": { + "recall": 0.914, + "precision": 0.999, + "top_precision": 0.995, + "caught": 1.0 + }, + "union": { + "recall": 0.942, + "precision": 0.953, + "top_precision": 0.995, + "caught": 1.0 + }, + "functions": 21 + }, + "random": { + "gitnext": { + "recall": 0.339, + "precision": 0.533, + "top_precision": 0.617, + "caught": 0.833 + }, + "coverage": { + "recall": 1.0, + "precision": 0.848, + "top_precision": 0.858, + "caught": 0.917 + }, + "union": { + "recall": 1.0, + "precision": 0.614, + "top_precision": 0.783, + "caught": 0.917 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.291, + "precision": 0.641, + "top_precision": 0.665, + "caught": 0.879 + }, + "coverage": { + "recall": 0.945, + "precision": 0.944, + "top_precision": 0.945, + "caught": 0.97 + }, + "union": { + "recall": 0.963, + "precision": 0.829, + "top_precision": 0.918, + "caught": 0.97 + }, + "functions": 33 + } + }, + "cost": { + "how": "per-test-fork", + "class_inits_s": 3.3, + "class_inits_booked": true, + "files_with_init_lines": 64, + "mvn_s": 110.0, + "analyze_s": 25.8, + "map_s": 139.0, + "exec_bytes": 8878815, + "dumps": 5372, + "test_methods": 2130, + "classes": 76, + "classes_with_setup_lines": 6, + "run_failed": false, + "names_in_several_classes": 115, + "junit": 5, + "package": "org.jsoup", + "owner": "body", + "repeat": 3, + "plain_s": 16.5, + "tests": 2130, + "map_pairs": 559173, + "map_functions": 1771, + "map_sqlite_bytes": 5746688, + "functions_with_body": 2299, + "functions_never_run": 99, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 8.4 + } + }, + "per-class": { + "repo": "jsoup", + "lang": "java", + "commit": "d90d57521d93", + "seed": 7, + "gitnext_commit": "14bcb17", + "functions": [ + { + "group": "pick", + "symbol": "org::jsoup::internal::LegacyMatching::cssAttributeMatches", + "file": "src/main/java/org/jsoup/internal/LegacyMatching.java", + "truth": 7, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 193, + "missed_by_coverage": [], + "covered_not_failing": [ + "adjacentSiblings", + "adjacentSiblingsWithId", + "and", + "anyAttribute", + "attributeSelectorHandlesInternalData", + "attributeSelectorQuotedWhitespace", + "attributeTokenOperatorUsesRegex", + "attributeWithBrackets", + "blankElements", + "blankNodes" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 186, + "top_coverage": [ + "childrenRetainTheirCapturedMatchingModes", + "compilationModeRestoresEnclosingMode", + "compilationOverrideDoesNotChangeCheckMode", + "compiledContextsAreReusableAcrossDocumentsAndThreads", + "compiledModeSurvivesPropertyChangesAndLazyTraversal", + "compiledModesAndFlagsSurvivePropertyChangesAndReuse", + "compiledQueriesAreReusableAcrossScopesAndThreads", + "compiledQueriesKeepTheirMode", + "compiledRegexKeepsItsModeWhenComposed", + "concurrentCompilationsKeepTheirOwnModes" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.03626943005181347, + "predicted": 193, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.3333333333333333, + "predicted": 6, + "top_hits": 2, + "top_n": 6, + "top_precision": 0.3333333333333333, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.03626943005181347, + "predicted": 193, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.3333333333333333, + "predicted": 6, + "top_hits": 2, + "top_n": 6, + "top_precision": 0.3333333333333333, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::Normalizer::lowerCase", + "file": "src/main/java/org/jsoup/internal/Normalizer.java", + "truth": 1689, + "truth_doctests": 0, + "predicted_gitnext": 1183, + "predicted_coverage": 1941, + "missed_by_coverage": [ + "emptyCookieJar", + "init", + "provideEvaluators", + "testOutputSettings" + ], + "covered_not_failing": [ + "allowsPopulatedSubtreeMove", + "allowsSpaceInUrl", + "attributeRegexRoundTripsLeadingWhitespace", + "attributeValueIdentifiersReplaceLeadingNull", + "bailsIfRemovingProtocolThatsNotSet", + "brSemantics", + "bulkChildTransferPreservesDestinationIndexValidity", + "buttonSemantics", + "canBeInsensitive", + "canCustomizeAll" + ], + "missed_by_coverage_n": 4, + "covered_not_failing_n": 256, + "top_coverage": [ + "separatesAsciiAndUnicodeCase", + "trimsOnlyAsciiWhitespace", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "namesUseAsciiCaseFolding", + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnAttributes", + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "testCopyConstructor_noSideEffectOnProtocols", + "testCopyConstructor_noSideEffectOnTags" + ], + "score": { + "gitnext": { + "recall": 0.6956779159265838, + "precision": 0.9932375316990701, + "predicted": 1183, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9976317347542925, + "precision": 0.8681092220504895, + "predicted": 1941, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9976317347542925, + "precision": 0.8658787255909558, + "predicted": 1946, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::SoftPool::release", + "file": "src/main/java/org/jsoup/internal/SoftPool.java", + "truth": 1755, + "truth_doctests": 0, + "predicted_gitnext": 834, + "predicted_coverage": 1973, + "missed_by_coverage": [ + "emptyCookieJar", + "init", + "provideEvaluators", + "testOutputSettings" + ], + "covered_not_failing": [ + "advance", + "allowsSpaceInUrl", + "bailsIfRemovingProtocolThatsNotSet", + "brSemantics", + "bufferUp", + "bulkChildTransferPreservesDestinationIndexValidity", + "buttonSemantics", + "cachesStrings", + "canBeInsensitive", + "canCustomizeAll" + ], + "missed_by_coverage_n": 4, + "covered_not_failing_n": 222, + "top_coverage": [ + "testReleaseMoreThanMaxIdle", + "testBorrowFromEmptyPool", + "testSoftLocalPool", + "testSoftReferenceBehavior", + "compactsBufferWithActiveMark", + "distinguishesExactLimitFromTruncation", + "largeSkipSaturatesProgress", + "readAndSkipRespectLimit", + "respectsMaxCapDuringFill", + "retainsLookaheadWhenLimitIsRaised" + ], + "score": { + "gitnext": { + "recall": 0.46495726495726497, + "precision": 0.9784172661870504, + "predicted": 834, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.8529411764705882, + "precision": 1.0, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9977207977207977, + "precision": 0.8874809934110491, + "predicted": 1973, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9977207977207977, + "precision": 0.8830055471507816, + "predicted": 1983, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 68, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::appendNormalisedWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 1705, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1259, + "top_coverage": [ + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit", + "isHexDigit", + "isNumeric", + "isWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.289237668161435, + "precision": 0.9148936170212766, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5833333333333334, + "precision": 1.0, + "predicted": 28, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.26158357771261, + "predicted": 1705, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.8727272727272727, + "predicted": 55, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2612770943175161, + "predicted": 1707, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8727272727272727, + "predicted": 55, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isActuallyWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 446, + "truth_doctests": 0, + "predicted_gitnext": 132, + "predicted_coverage": 1705, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1259, + "top_coverage": [ + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit", + "isHexDigit", + "isNumeric", + "isWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.27802690582959644, + "precision": 0.9393939393939394, + "predicted": 132, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.5416666666666666, + "precision": 1.0, + "predicted": 26, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.26158357771261, + "predicted": 1705, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.8727272727272727, + "predicted": 55, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.2612770943175161, + "predicted": 1707, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8727272727272727, + "predicted": 55, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::isAsciiLetter", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1699, + "truth_doctests": 0, + "predicted_gitnext": 39, + "predicted_coverage": 1926, + "missed_by_coverage": [ + "init", + "provideEvaluators", + "rcDataRecoverParser" + ], + "covered_not_failing": [ + "advance", + "allowsSpaceInUrl", + "bailsIfRemovingProtocolThatsNotSet", + "booleanCaseInsensitive", + "booleanEmptyString", + "booleanNullAttributesConsistent", + "bufferUp", + "cachesStrings", + "canEnableAndDisableLineNumberTracking", + "canFetchBinaryAsBytes" + ], + "missed_by_coverage_n": 3, + "covered_not_failing_n": 230, + "top_coverage": [ + "advance", + "bufferUp", + "cachesStrings", + "canEnableAndDisableLineNumberTracking", + "canTrackNewlines", + "consume", + "consumeDoubleQuotedAttributeConsumesThruSingleQuote", + "consumeDoubleQuotedAttributeConsumesThruSingleQuoteToAmp", + "consumeLetterSequenceInAscii", + "consumeLetterThenDigitSequence" + ], + "score": { + "gitnext": { + "recall": 0.020011771630370805, + "precision": 0.8717948717948718, + "predicted": 39, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.18461538461538463, + "precision": 1.0, + "predicted": 12, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998234255444379, + "precision": 0.8805815160955348, + "predicted": 1926, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998234255444379, + "precision": 0.8805815160955348, + "predicted": 1926, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 65, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::internal::StringUtil::trimAsciiWhitespace", + "file": "src/main/java/org/jsoup/internal/StringUtil.java", + "truth": 1811, + "truth_doctests": 0, + "predicted_gitnext": 141, + "predicted_coverage": 1947, + "missed_by_coverage": [ + "emptyCookieJar", + "init", + "provideEvaluators", + "rcDataRecoverParser", + "testOutputSettings" + ], + "covered_not_failing": [ + "allowsSpaceInUrl", + "bailsIfRemovingProtocolThatsNotSet", + "canSetOptions", + "chompBalanced", + "chompBalancedMatchesAsMuchAsPossible", + "chompBalancedRemainder", + "chompBalancedThrowIllegalArgumentException", + "chompEscapedBalanced", + "cloneAttributes", + "cloneGetsUniqueUserDataMap" + ], + "missed_by_coverage_n": 5, + "covered_not_failing_n": 141, + "top_coverage": [ + "separatesAsciiAndUnicodeCase", + "trimsOnlyAsciiWhitespace", + "attributeCaseNormalization", + "attributesCaseNormalization", + "caseSupport", + "recognizesNamespaceDeclarations", + "restoresBindingsAfterNestedScopes", + "stateGrowsLinearlyWithScopeDepth", + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute" + ], + "score": { + "gitnext": { + "recall": 0.07675317504141359, + "precision": 0.9858156028368794, + "predicted": 141, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.42857142857142855, + "precision": 1.0, + "predicted": 30, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9972390944229708, + "precision": 0.9275808936825886, + "predicted": 1947, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9972390944229708, + "precision": 0.9275808936825886, + "predicted": 1947, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 70, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::deduplicate", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 780, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 1722, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "adjacentSiblings", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 944, + "top_coverage": [ + "canFilter", + "canFindFirst", + "canRemove", + "canStream", + "canStreamParallel", + "elementStream", + "canFilterForElements", + "canFilterForTextNodes", + "canIterateFirstEmptySibling", + "canIterateJustOneSibling" + ], + "score": { + "gitnext": { + "recall": 0.010256410256410256, + "precision": 1.0, + "predicted": 8, + "top_hits": 8, + "top_n": 8, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.02040816326530612, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9974358974358974, + "precision": 0.4518002322880372, + "predicted": 1722, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8909090909090909, + "predicted": 55, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9974358974358974, + "precision": 0.4518002322880372, + "predicted": 1722, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8909090909090909, + "predicted": 55, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::getIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 610, + "truth_doctests": 0, + "predicted_gitnext": 251, + "predicted_coverage": 1707, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addedAttributeUsesInheritedNamespace", + "addsTagOnAttributesIfNotSet", + "adjacentSiblings" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1097, + "top_coverage": [ + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "namesUseAsciiCaseFolding", + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnAttributes", + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "testCopyConstructor_noSideEffectOnProtocols", + "testCopyConstructor_noSideEffectOnTags", + "canFilter", + "canFindFirst" + ], + "score": { + "gitnext": { + "recall": 0.3540983606557377, + "precision": 0.8605577689243028, + "predicted": 251, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.6739130434782609, + "precision": 1.0, + "predicted": 31, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.35735207967193905, + "predicted": 1707, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8679245283018868, + "predicted": 53, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.35714285714285715, + "predicted": 1708, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8679245283018868, + "predicted": 53, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Attributes::removeIgnoreCase", + "file": "src/main/java/org/jsoup/nodes/Attributes.java", + "truth": 28, + "truth_doctests": 0, + "predicted_gitnext": 31, + "predicted_coverage": 542, + "missed_by_coverage": [], + "covered_not_failing": [ + "absAttr", + "accessorsDoNotVivifyAttributes", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "after", + "afterNode", + "allAttributes", + "allPseudoTagProtocolsNormalizeRelativeLinks", + "appendChildren", + "appendMustCorrectlyMoveChildrenInsideOneParentElement" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 514, + "top_coverage": [ + "booleanCaseInsensitive", + "booleanEmptyString", + "booleanNullAttributesConsistent", + "cloneAttributes", + "cloneGetsUniqueUserDataMap", + "dontCloneNullUserData", + "equalsCountsDuplicateAttributes", + "equalsIgnoresInternalData", + "equalsIsOrderInsensitive", + "hashCodeDistinguishesSwappedNameAndValue" + ], + "score": { + "gitnext": { + "recall": 0.07142857142857142, + "precision": 0.06451612903225806, + "predicted": 31, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 0.18181818181818182, + "precision": 0.6666666666666666, + "predicted": 3, + "top_hits": 2, + "top_n": 3, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.05166051660516605, + "predicted": 542, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4782608695652174, + "predicted": 23, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.05166051660516605, + "predicted": 542, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.4782608695652174, + "predicted": 23, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::DocumentType::publicId", + "file": "src/main/java/org/jsoup/nodes/DocumentType.java", + "truth": 115, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 1237, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "absoluteAttributeLookupIgnoresKeyCase", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addedAttributeUsesInheritedNamespace", + "adjacentSiblings", + "adjacentSiblingsWithId" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1122, + "top_coverage": [ + "attributes", + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "constructorValidationThrowsExceptionOnNulls", + "outerHtmlGeneration", + "systemIdentifierUsesSafeQuote", + "testRoundTrip", + "accumulatesWhitespaceReferencesBeforeText", + "bufferUpInAttributeVal", + "canParseCdataEndingAtEdgeOfBuffer" + ], + "score": { + "gitnext": { + "recall": 0.02608695652173913, + "precision": 0.42857142857142855, + "predicted": 7, + "top_hits": 3, + "top_n": 7, + "top_precision": 0.42857142857142855, + "caught": 1.0, + "by_class": { + "recall": 0.08333333333333333, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.09296685529506872, + "predicted": 1237, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6857142857142857, + "predicted": 35, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.09296685529506872, + "predicted": 1237, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6857142857142857, + "predicted": 35, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::nodes::Entities::isNamedEntity", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 51, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 983, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure", + "addedAttributeUsesInheritedNamespace", + "addsTagOnAttributesIfNotSet", + "adjustedForeignAttributesDedupe", + "adjustsForeignNameCase" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 932, + "top_coverage": [ + "testClone", + "testCloneCopyTagSet", + "testUtf8", + "unescapeEntities", + "unescapeEntitiesHandlesLargeInput", + "unescapeEntitiesInAttributes", + "unescapeQueryParametersInTextOnly", + "unescapeTracksErrors", + "alwaysEscapeLtAndGtInAttributeValues", + "caseSensitive" + ], + "score": { + "gitnext": { + "recall": 0.1568627450980392, + "precision": 0.3333333333333333, + "predicted": 24, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.16666666666666666, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.05188199389623601, + "predicted": 983, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6428571428571429, + "predicted": 28, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.05188199389623601, + "predicted": 983, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6428571428571429, + "predicted": 28, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Parser::getMaxDepth", + "file": "src/main/java/org/jsoup/parser/Parser.java", + "truth": 1467, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 1864, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle", + "adoptionDropsNodeWhenItsReferenceIsRemoved", + "adoptionInsertionAtCapacityPreservesBookmark", + "allowsPopulatedSubtreeMove", + "allowsSpaceInUrl", + "appendMustCorrectlyMoveChildrenInsideOneParentElement" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 399, + "top_coverage": [ + "appendThrowsSerializationException", + "supplemental", + "wrap", + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit" + ], + "score": { + "gitnext": { + "recall": 0.0027266530334014998, + "precision": 0.4, + "predicted": 10, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 0.09836065573770492, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9986366734832992, + "precision": 0.7859442060085837, + "predicted": 1864, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9838709677419355, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9986366734832992, + "precision": 0.7859442060085837, + "predicted": 1864, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9838709677419355, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Tag::isFormSubmittable", + "file": "src/main/java/org/jsoup/parser/Tag.java", + "truth": 16, + "truth_doctests": 0, + "predicted_gitnext": 17, + "predicted_coverage": 422, + "missed_by_coverage": [], + "covered_not_failing": [ + "NoErrorOnEofIfBodyOpen", + "actionWithNoBaseUri", + "adjustedForeignAttributesDedupe", + "adjustsForeignNameCase", + "adoption", + "adoptionKeepsBlockInsideNestedTemplate", + "adoptionKeepsNestedTableInsideTemplate", + "adoptionPreservesChildOrder", + "adoptionRetainsOlderFormattingHistory", + "afterHeadReAdds" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 406, + "top_coverage": [ + "brSemantics", + "buttonSemantics", + "canBeInsensitive", + "canSetOptions", + "defaultSemantics", + "divSemantics", + "equality", + "equals", + "formSubmittable", + "imgSemantics" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.9411764705882353, + "predicted": 17, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 5, + "top_hits": 5, + "top_n": 5, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.037914691943127965, + "predicted": 422, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5555555555555556, + "predicted": 9, + "top_hits": 5, + "top_n": 9, + "top_precision": 0.5555555555555556, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.037914691943127965, + "predicted": 422, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5555555555555556, + "predicted": 9, + "top_hits": 5, + "top_n": 9, + "top_precision": 0.5555555555555556, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::Token::Doctype::getSystemIdentifier", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 130, + "truth_doctests": 0, + "predicted_gitnext": 0, + "predicted_coverage": 1302, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "absoluteAttributeLookupIgnoresKeyCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addedAttributeUsesInheritedNamespace", + "adjacentSiblings" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1172, + "top_coverage": [ + "attributes", + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "constructorValidationThrowsExceptionOnNulls", + "outerHtmlGeneration", + "systemIdentifierUsesSafeQuote", + "testRoundTrip", + "accumulatesWhitespaceReferencesBeforeText", + "bufferUpInAttributeVal", + "canParseCdataEndingAtEdgeOfBuffer" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 0, + "top_hits": 0, + "top_n": 0, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.09984639016897082, + "predicted": 1302, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7222222222222222, + "predicted": 36, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.09984639016897082, + "predicted": 1302, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7222222222222222, + "predicted": 36, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::consumeCssIdentifier", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 383, + "truth_doctests": 0, + "predicted_gitnext": 10, + "predicted_coverage": 1393, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure", + "actionWithNoBaseUri" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1010, + "top_coverage": [ + "canFilterForElements", + "canFilterForTextNodes", + "canIterateFirstEmptySibling", + "canIterateJustOneSibling", + "canIterateNodes", + "canModifyFilteredElements", + "canRemoveViaIterator", + "canRemoveViaNode", + "canReplace", + "canRestart" + ], + "score": { + "gitnext": { + "recall": 0.02349869451697128, + "precision": 0.9, + "predicted": 10, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.058823529411764705, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.27494615936826994, + "predicted": 1393, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7555555555555555, + "predicted": 45, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.27494615936826994, + "predicted": 1393, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7555555555555555, + "predicted": 45, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWhitespace", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1721, + "truth_doctests": 0, + "predicted_gitnext": 119, + "predicted_coverage": 1931, + "missed_by_coverage": [ + "emptyCookieJar", + "init", + "provideEvaluators", + "testOutputSettings" + ], + "covered_not_failing": [ + "allowsPopulatedSubtreeMove", + "allowsSpaceInUrl", + "bailsIfRemovingProtocolThatsNotSet", + "brSemantics", + "bulkChildTransferPreservesDestinationIndexValidity", + "buttonSemantics", + "canBeInsensitive", + "canCustomizeAll", + "canCustomizeSome", + "canSetOptions" + ], + "missed_by_coverage_n": 4, + "covered_not_failing_n": 214, + "top_coverage": [ + "parseCookie", + "xmlOutputOrphanWithoutCData", + "xmlOutputOtherWithCData", + "xmlOutputOtherWithoutCData", + "xmlOutputScriptWithCData", + "xmlOutputScriptWithoutCData", + "xmlOutputStyleWithCData", + "xmlOutputStyleWithoutCData", + "connectFailedDelegatesToSystemDefault", + "createsOneClientForConcurrentSessionRequests" + ], + "score": { + "gitnext": { + "recall": 0.06624055781522371, + "precision": 0.957983193277311, + "predicted": 119, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.21212121212121213, + "precision": 1.0, + "predicted": 14, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9976757699012202, + "precision": 0.8891765924391507, + "predicted": 1931, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 66, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9976757699012202, + "precision": 0.8887163561076604, + "predicted": 1932, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 66, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::parser::TokenQueue::matchesWord", + "file": "src/main/java/org/jsoup/parser/TokenQueue.java", + "truth": 1717, + "truth_doctests": 0, + "predicted_gitnext": 22, + "predicted_coverage": 1931, + "missed_by_coverage": [ + "emptyCookieJar", + "init", + "provideEvaluators", + "testOutputSettings" + ], + "covered_not_failing": [ + "allowsPopulatedSubtreeMove", + "allowsSpaceInUrl", + "bailsIfRemovingProtocolThatsNotSet", + "brSemantics", + "bulkChildTransferPreservesDestinationIndexValidity", + "buttonSemantics", + "canBeInsensitive", + "canCustomizeAll", + "canCustomizeSome", + "canSetOptions" + ], + "missed_by_coverage_n": 4, + "covered_not_failing_n": 218, + "top_coverage": [ + "parseCookie", + "xmlOutputOrphanWithoutCData", + "xmlOutputOtherWithCData", + "xmlOutputOtherWithoutCData", + "xmlOutputScriptWithCData", + "xmlOutputScriptWithoutCData", + "xmlOutputStyleWithCData", + "xmlOutputStyleWithoutCData", + "connectFailedDelegatesToSystemDefault", + "createsOneClientForConcurrentSessionRequests" + ], + "score": { + "gitnext": { + "recall": 0.0029120559114735, + "precision": 0.22727272727272727, + "predicted": 22, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9976703552708212, + "precision": 0.8871051268772656, + "predicted": 1931, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 66, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9976703552708212, + "precision": 0.8871051268772656, + "predicted": 1931, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 66, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addEnforcedAttribute", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 38, + "truth_doctests": 0, + "predicted_gitnext": 38, + "predicted_coverage": 372, + "missed_by_coverage": [], + "covered_not_failing": [ + "NoErrorOnEofIfBodyOpen", + "addsTagOnAttributesIfNotSet", + "adjustedForeignAttributesDedupe", + "adjustsForeignNameCase", + "adoptedFormControlsCanBeMovedOutsideForm", + "adoption", + "adoptionKeepsBlockInsideNestedTemplate", + "adoptionKeepsNestedTableInsideTemplate", + "adoptionPreservesChildOrder", + "adoptionRetainsOlderFormattingHistory" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 334, + "top_coverage": [ + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "namesUseAsciiCaseFolding", + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnAttributes", + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "testCopyConstructor_noSideEffectOnProtocols", + "testCopyConstructor_noSideEffectOnTags", + "addsTagOnAttributesIfNotSet", + "allAttributes" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 1.0, + "predicted": 38, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.10215053763440861, + "predicted": 372, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 6, + "top_hits": 3, + "top_n": 6, + "top_precision": 0.5, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.10215053763440861, + "predicted": 372, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5, + "predicted": 6, + "top_hits": 3, + "top_n": 6, + "top_precision": 0.5, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::safety::Safelist::addTags", + "file": "src/main/java/org/jsoup/safety/Safelist.java", + "truth": 72, + "truth_doctests": 0, + "predicted_gitnext": 71, + "predicted_coverage": 373, + "missed_by_coverage": [], + "covered_not_failing": [ + "NoErrorOnEofIfBodyOpen", + "adjustedForeignAttributesDedupe", + "adjustsForeignNameCase", + "adoptedFormControlsCanBeMovedOutsideForm", + "adoption", + "adoptionKeepsBlockInsideNestedTemplate", + "adoptionKeepsNestedTableInsideTemplate", + "adoptionPreservesChildOrder", + "adoptionRetainsOlderFormattingHistory", + "afterHeadReAdds" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 301, + "top_coverage": [ + "enforcedAttributeMatchesInputKeyCaseInsensitively", + "isSafeAttributeDoesNotModifyLiveAttribute", + "namesUseAsciiCaseFolding", + "noscriptIsBlocked", + "testCopyConstructor_noSideEffectOnAttributes", + "testCopyConstructor_noSideEffectOnEnforcedAttributes", + "testCopyConstructor_noSideEffectOnProtocols", + "testCopyConstructor_noSideEffectOnTags", + "canCustomizeSafeTests", + "addsTagOnAttributesIfNotSet" + ], + "score": { + "gitnext": { + "recall": 0.9861111111111112, + "precision": 1.0, + "predicted": 71, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 4, + "top_hits": 4, + "top_n": 4, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.19302949061662197, + "predicted": 373, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.19302949061662197, + "predicted": 373, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5714285714285714, + "predicted": 7, + "top_hits": 4, + "top_n": 7, + "top_precision": 0.5714285714285714, + "caught": 1.0 + } + } + } + }, + { + "group": "pick", + "symbol": "org::jsoup::select::Evaluator::asPredicate", + "file": "src/main/java/org/jsoup/select/Evaluator.java", + "truth": 957, + "truth_doctests": 0, + "predicted_gitnext": 23, + "predicted_coverage": 1820, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "addsTagOnAttributesIfNotSet", + "adoptedMixedChildrenRetainSourceRanges", + "adoptionDoesNotAddSecondDocumentElement", + "adoptionDropsNodeThatWouldCreateCycle" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 863, + "top_coverage": [ + "cdataNulls", + "copiesNullFlag", + "normalizesNulls", + "nullReferences", + "resetsNullFlagOnReuse", + "brSemantics", + "buttonSemantics", + "canBeInsensitive", + "canSetOptions", + "defaultSemantics" + ], + "score": { + "gitnext": { + "recall": 0.022988505747126436, + "precision": 0.9565217391304348, + "predicted": 23, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.12280701754385964, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5258241758241758, + "predicted": 1820, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9661016949152542, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.5255354200988468, + "predicted": 1821, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9661016949152542, + "predicted": 59, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::ownerDocument", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 891, + "truth_doctests": 0, + "predicted_gitnext": 1012, + "predicted_coverage": 1761, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 870, + "top_coverage": [ + "appendThrowsSerializationException", + "supplemental", + "wrap", + "attributes", + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "constructorValidationThrowsExceptionOnNulls", + "outerHtmlGeneration", + "systemIdentifierUsesSafeQuote", + "testRoundTrip" + ], + "score": { + "gitnext": { + "recall": 0.8518518518518519, + "precision": 0.75, + "predicted": 1012, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.9629629629629629, + "precision": 0.896551724137931, + "predicted": 58, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.5059625212947189, + "predicted": 1761, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9818181818181818, + "predicted": 55, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.4988801791713326, + "predicted": 1786, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::NodeInternals::attributeRange", + "file": "src/main/java/org/jsoup/nodes/NodeInternals.java", + "truth": 39, + "truth_doctests": 0, + "predicted_gitnext": 8, + "predicted_coverage": 535, + "missed_by_coverage": [], + "covered_not_failing": [ + "NoErrorOnEofIfBodyOpen", + "adjustedForeignAttributesDedupe", + "adjustsForeignNameCase", + "adoptedFormControlsCanBeMovedOutsideForm", + "adoptedMixedChildrenRetainSourceRanges", + "adoption", + "adoptionKeepsBlockInsideNestedTemplate", + "adoptionKeepsNestedTableInsideTemplate", + "adoptionPreservesChildOrder", + "adoptionRetainsOlderFormattingHistory" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 496, + "top_coverage": [ + "canCustomizeSafeTests", + "booleanCaseInsensitive", + "booleanEmptyString", + "booleanNullAttributesConsistent", + "cloneAttributes", + "cloneGetsUniqueUserDataMap", + "dontCloneNullUserData", + "equalsCountsDuplicateAttributes", + "equalsIgnoresInternalData", + "equalsIsOrderInsensitive" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 8, + "top_hits": 0, + "top_n": 8, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 1, + "top_hits": 0, + "top_n": 1, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.07289719626168224, + "predicted": 535, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.29411764705882354, + "predicted": 17, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.07289719626168224, + "predicted": 535, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.29411764705882354, + "predicted": 17, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TokeniserState::read", + "file": "src/main/java/org/jsoup/parser/TokeniserState.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 27, + "predicted_coverage": 1864, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1863, + "top_coverage": [ + "appendThrowsSerializationException", + "supplemental", + "wrap", + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit" + ], + "score": { + "gitnext": { + "recall": 0.0, + "precision": 0.0, + "predicted": 27, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 0.0, + "precision": 0.0, + "predicted": 10, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.000536480686695279, + "predicted": 1864, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.016129032258064516, + "predicted": 62, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0005341880341880342, + "predicted": 1872, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.015873015873015872, + "predicted": 63, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::helper::HttpConnection::Response::sameOrigin", + "file": "src/main/java/org/jsoup/helper/HttpConnection.java", + "truth": 19, + "truth_doctests": 0, + "predicted_gitnext": 44, + "predicted_coverage": 134, + "missed_by_coverage": [], + "covered_not_failing": [ + "baseHrefCorrectAfterHttpEquiv", + "bodyAfterParseThrowsValidationError", + "bodyAndBytesAvailableBeforeParse", + "bufferedParseWorksWhenCharsetDetectionFullyReadsResponse", + "bufferedStreamParserWorksWhenCharsetDetectionFullyReadsResponse", + "canAuthToProxyAndServer", + "canAuthenticate", + "canAuthenticateToProxy", + "canConnectToLocalServer", + "canConnectToLocalTlsServer" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 115, + "top_coverage": [ + "sameOriginUsesSchemeHostAndEffectivePort", + "canCreateEmptyConnection", + "caseInsensitiveHeaders", + "connectToEncodedUrl", + "connectWithUrl", + "cookie", + "data", + "encodeUrl", + "encodeUrlSupplementary", + "encodedUrlDoesntDoubleEncode" + ], + "score": { + "gitnext": { + "recall": 0.10526315789473684, + "precision": 0.045454545454545456, + "predicted": 44, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 0.5, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.1417910447761194, + "predicted": 134, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.1417910447761194, + "predicted": 134, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 6, + "top_hits": 6, + "top_n": 6, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::outerHtml", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 116, + "truth_doctests": 0, + "predicted_gitnext": 128, + "predicted_coverage": 1406, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1290, + "top_coverage": [ + "outerHtmlGeneration", + "attributes", + "constructorValidationOkWithBlankName", + "constructorValidationOkWithBlankPublicAndSystemIds", + "constructorValidationThrowsExceptionOnNulls", + "systemIdentifierUsesSafeQuote", + "testRoundTrip", + "booleanAttributesAreEmptyStringValues", + "booleanAttributesAreNotCaseSensitive", + "canSetValueToNull" + ], + "score": { + "gitnext": { + "recall": 0.8879310344827587, + "precision": 0.8046875, + "predicted": 128, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.9259259259259259, + "precision": 0.8620689655172413, + "predicted": 29, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.08250355618776671, + "predicted": 1406, + "top_hits": 4, + "top_n": 10, + "top_precision": 0.4, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6428571428571429, + "predicted": 42, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.08238636363636363, + "predicted": 1408, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.6428571428571429, + "predicted": 42, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Element::TextAccumulator::hasBlockChild", + "file": "src/main/java/org/jsoup/nodes/Element.java", + "truth": 123, + "truth_doctests": 0, + "predicted_gitnext": 256, + "predicted_coverage": 1368, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1245, + "top_coverage": [ + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit", + "isHexDigit", + "isNumeric", + "isWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.21138211382113822, + "precision": 0.1015625, + "predicted": 256, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 0.2692307692307692, + "precision": 0.7, + "predicted": 10, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.08991228070175439, + "predicted": 1368, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.65, + "predicted": 40, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.08984660336011688, + "predicted": 1369, + "top_hits": 6, + "top_n": 10, + "top_precision": 0.6, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.65, + "predicted": 40, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Node::setParentNode", + "file": "src/main/java/org/jsoup/nodes/Node.java", + "truth": 1554, + "truth_doctests": 0, + "predicted_gitnext": 338, + "predicted_coverage": 1864, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "accumulatesWhitespaceReferencesBeforeText", + "allowsSpaceInUrl", + "attributeNamedLikeNodePreservesInstructionData", + "attributeRegexRoundTripsLeadingWhitespace", + "attributeValueIdentifiersReplaceLeadingNull", + "bailsIfRemovingProtocolThatsNotSet", + "booleanCaseInsensitive", + "booleanEmptyString", + "booleanNullAttributesConsistent", + "brSemantics" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 312, + "top_coverage": [ + "appendThrowsSerializationException", + "supplemental", + "wrap", + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit" + ], + "score": { + "gitnext": { + "recall": 0.21171171171171171, + "precision": 0.9733727810650887, + "predicted": 338, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 0.6612903225806451, + "precision": 1.0, + "predicted": 41, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.9987129987129987, + "precision": 0.8326180257510729, + "predicted": 1864, + "top_hits": 2, + "top_n": 10, + "top_precision": 0.2, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.9987129987129987, + "precision": 0.8286171916711158, + "predicted": 1873, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 1.0, + "predicted": 62, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Document::text", + "file": "src/main/java/org/jsoup/nodes/Document.java", + "truth": 1, + "truth_doctests": 0, + "predicted_gitnext": 140, + "predicted_coverage": 39, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "accessorsAreCaseInsensitive", + "accessorsWillNormalizeStructure", + "charsetOnEmptyDoc", + "cloneHasIndependentChildElements", + "expectForm", + "forms", + "framesetSupportsBodyMethod", + "htmlParseDefaultsToHtmlOutputSyntax" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 38, + "top_coverage": [ + "setTextPreservesDocumentStructure", + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "accessorsAreCaseInsensitive", + "accessorsWillNormalizeStructure", + "charsetOnEmptyDoc", + "cloneHasIndependentChildElements", + "expectForm", + "forms", + "framesetSupportsBodyMethod" + ], + "score": { + "gitnext": { + "recall": 1.0, + "precision": 0.007142857142857143, + "predicted": 140, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.02564102564102564, + "predicted": 39, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.16666666666666666, + "predicted": 6, + "top_hits": 1, + "top_n": 6, + "top_precision": 0.16666666666666666, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.0056179775280898875, + "predicted": 178, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.03333333333333333, + "predicted": 30, + "top_hits": 1, + "top_n": 10, + "top_precision": 0.1, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::DataNode::outerHtmlHead", + "file": "src/main/java/org/jsoup/nodes/DataNode.java", + "truth": 37, + "truth_doctests": 0, + "predicted_gitnext": 7, + "predicted_coverage": 1049, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "DocumentsWithSameContentAreVerifiable", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absoluteAttributeLookupIgnoresKeyCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes", + "accessorsWillNormalizeStructure" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1012, + "top_coverage": [ + "xmlOutputOrphanWithoutCData", + "xmlOutputOtherWithCData", + "xmlOutputOtherWithoutCData", + "xmlOutputScriptWithCData", + "xmlOutputScriptWithoutCData", + "xmlOutputStyleWithCData", + "xmlOutputStyleWithoutCData", + "canCustomizeAll", + "canCustomizeSome", + "canParseWithCustomization" + ], + "score": { + "gitnext": { + "recall": 0.1891891891891892, + "precision": 1.0, + "predicted": 7, + "top_hits": 7, + "top_n": 7, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.07692307692307693, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.03527168732125834, + "predicted": 1049, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.43333333333333335, + "predicted": 30, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.03527168732125834, + "predicted": 1049, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.43333333333333335, + "predicted": 30, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::Token::Character::normalizeNulls", + "file": "src/main/java/org/jsoup/parser/Token.java", + "truth": 1090, + "truth_doctests": 0, + "predicted_gitnext": 3, + "predicted_coverage": 1810, + "missed_by_coverage": [ + "init", + "provideEvaluators" + ], + "covered_not_failing": [ + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absoluteAttributesRespectAttributeNameCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsWillNormalizeStructure", + "accumulatesWhitespaceReferencesBeforeText", + "actionWithNoBaseUri", + "actionWithNoValue", + "addedAttributeUsesInheritedNamespace", + "adjustedForeignAttributesDedupe" + ], + "missed_by_coverage_n": 2, + "covered_not_failing_n": 722, + "top_coverage": [ + "allowsSpaceInUrl", + "concatCharacterAndString", + "identifiesHttpSchemes", + "isAscii", + "isAsciiLetter", + "isBlank", + "isDigit", + "isHexDigit", + "isNumeric", + "isWhitespace" + ], + "score": { + "gitnext": { + "recall": 0.0027522935779816515, + "precision": 1.0, + "predicted": 3, + "top_hits": 3, + "top_n": 3, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.01694915254237288, + "precision": 1.0, + "predicted": 1, + "top_hits": 1, + "top_n": 1, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 0.998165137614679, + "precision": 0.6011049723756906, + "predicted": 1810, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.9833333333333333, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 0.998165137614679, + "precision": 0.6011049723756906, + "predicted": 1810, + "top_hits": 3, + "top_n": 10, + "top_precision": 0.3, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.9833333333333333, + "predicted": 60, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::parser::TreeBuilder::parseAttributes", + "file": "src/main/java/org/jsoup/parser/TreeBuilder.java", + "truth": 29, + "truth_doctests": 0, + "predicted_gitnext": 24, + "predicted_coverage": 326, + "missed_by_coverage": [], + "covered_not_failing": [ + "aLoneOperandIsNotAnExplicitFlag", + "absoluteAttributeLookupIgnoresKeyCase", + "absoluteAttributesRespectAttributeNameCase", + "adjacentSiblings", + "adjacentSiblingsWithId", + "adoptedMixedChildrenRetainSourceRanges", + "and", + "anyAttribute", + "attributeSelectorHandlesInternalData", + "attributeSelectorQuotedWhitespace" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 297, + "top_coverage": [ + "parseAttributesAcceptsCompleteInputAtEof", + "parseAttributesAllowsCloserInQuotedValue", + "parseAttributesLeavesParserReusable", + "parseAttributesParsesNamesAndValues", + "parseAttributesReportsIncompleteValue", + "parseAttributesStopsAtTagCloser", + "parseAttributesTracksSourceRanges", + "parseAttributesUsesParserSettings", + "canStartWithEq", + "dropsSlashFromAttributeName" + ], + "score": { + "gitnext": { + "recall": 0.5862068965517241, + "precision": 0.7083333333333334, + "predicted": 24, + "top_hits": 9, + "top_n": 10, + "top_precision": 0.9, + "caught": 1.0, + "by_class": { + "recall": 0.5714285714285714, + "precision": 0.6666666666666666, + "predicted": 6, + "top_hits": 4, + "top_n": 6, + "top_precision": 0.6666666666666666, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.08895705521472393, + "predicted": 326, + "top_hits": 8, + "top_n": 10, + "top_precision": 0.8, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.7, + "predicted": 10, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.08761329305135952, + "predicted": 331, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.5833333333333334, + "predicted": 12, + "top_hits": 7, + "top_n": 10, + "top_precision": 0.7, + "caught": 1.0 + } + } + } + }, + { + "group": "random", + "symbol": "org::jsoup::nodes::Entities::encoderFor", + "file": "src/main/java/org/jsoup/nodes/Entities.java", + "truth": 573, + "truth_doctests": 0, + "predicted_gitnext": 11, + "predicted_coverage": 1591, + "missed_by_coverage": [], + "covered_not_failing": [ + "DocumentsWithSameContentAreEqual", + "NoErrorOnEofIfBodyOpen", + "aLoneOperandIsNotAnExplicitFlag", + "absAttr", + "absHandlesDotFromIndex", + "absHandlesRelativeQuery", + "absoluteAttributeLookupIgnoresKeyCase", + "acceptsLenientXmlProcessingInstructionTarget", + "accessorsAreCaseInsensitive", + "accessorsDoNotVivifyAttributes" + ], + "missed_by_coverage_n": 0, + "covered_not_failing_n": 1018, + "top_coverage": [ + "brSemantics", + "buttonSemantics", + "canBeInsensitive", + "canSetOptions", + "defaultSemantics", + "divSemantics", + "equality", + "equals", + "formSubmittable", + "imgSemantics" + ], + "score": { + "gitnext": { + "recall": 0.019197207678883072, + "precision": 1.0, + "predicted": 11, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 0.045454545454545456, + "precision": 1.0, + "predicted": 2, + "top_hits": 2, + "top_n": 2, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "coverage": { + "recall": 1.0, + "precision": 0.36015084852294155, + "predicted": 1591, + "top_hits": 0, + "top_n": 10, + "top_precision": 0.0, + "caught": 0.0, + "by_class": { + "recall": 1.0, + "precision": 0.8979591836734694, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + }, + "union": { + "recall": 1.0, + "precision": 0.36015084852294155, + "predicted": 1591, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0, + "by_class": { + "recall": 1.0, + "precision": 0.8979591836734694, + "predicted": 49, + "top_hits": 10, + "top_n": 10, + "top_precision": 1.0, + "caught": 1.0 + } + } + } + } + ], + "summary": { + "pick": { + "gitnext": { + "recall": 0.264, + "precision": 0.703, + "top_precision": 0.692, + "caught": 0.905 + }, + "coverage": { + "recall": 0.999, + "precision": 0.425, + "top_precision": 0.371, + "caught": 0.857 + }, + "union": { + "recall": 0.999, + "precision": 0.425, + "top_precision": 0.714, + "caught": 0.952 + }, + "functions": 21 + }, + "random": { + "gitnext": { + "recall": 0.339, + "precision": 0.533, + "top_precision": 0.617, + "caught": 0.833 + }, + "coverage": { + "recall": 1.0, + "precision": 0.236, + "top_precision": 0.25, + "caught": 0.667 + }, + "union": { + "recall": 1.0, + "precision": 0.234, + "top_precision": 0.583, + "caught": 0.917 + }, + "functions": 12 + }, + "all": { + "gitnext": { + "recall": 0.291, + "precision": 0.641, + "top_precision": 0.665, + "caught": 0.879 + }, + "coverage": { + "recall": 0.999, + "precision": 0.356, + "top_precision": 0.327, + "caught": 0.788 + }, + "union": { + "recall": 0.999, + "precision": 0.355, + "top_precision": 0.667, + "caught": 0.939 + }, + "functions": 33 + } + }, + "cost": { + "how": "per-class", + "mvn_s": 413.9, + "analyze_s": 2.1, + "map_s": 416.0, + "exec_bytes": 657374, + "dumps": 76, + "test_methods": 2130, + "classes": 76, + "names_in_several_classes": 115, + "junit": 5, + "package": "org.jsoup", + "owner": "body", + "repeat": 3, + "plain_s": 15.3, + "tests": 2130, + "map_pairs": 1521462, + "map_functions": 1771, + "map_sqlite_bytes": 15364096, + "functions_with_body": 2299, + "functions_never_run": 99, + "fail_alone_not_together": [], + "fail_together_not_alone": [], + "slowdown_runs": 27.2 + } + } +} \ No newline at end of file diff --git a/eval/results/learn-tests-jsoup.md b/eval/results/learn-tests-jsoup.md new file mode 100644 index 0000000..ce1ab47 --- /dev/null +++ b/eval/results/learn-tests-jsoup.md @@ -0,0 +1,315 @@ +# learn-tests на jsoup (java): сводка + +Все функции (pick + случайные). Полнота / точность / из 10 падают / поймано первыми 10. По методу — как истина eval (имя метода теста); по классу — класс, где есть падающий метод. + +| способ | единица | функций | граф (`tests-for`) | покрытие | объединение | карта, с | × обычного | +|---|---|---:|---|---|---|---:|---:| +| per-test | метод | 33 | 29% / 64% / 66% / 88% | 94% / 94% / 94% / 97% | 96% / 83% / 92% / 97% | 49.6 | 3.5 | +| per-test | класс | 33 | 40% / 81% / 82% / 88% | 98% / 95% / 96% / 97% | 99% / 91% / 92% / 97% | 49.6 | 3.5 | +| per-test-fork | метод | 33 | 29% / 64% / 66% / 88% | 94% / 94% / 94% / 97% | 96% / 83% / 92% / 97% | 139.0 | 8.4 | +| per-test-fork | класс | 33 | 40% / 81% / 82% / 88% | 100% / 95% / 96% / 97% | 100% / 91% / 92% / 97% | 139.0 | 8.4 | +| per-class | метод | 33 | 29% / 64% / 66% / 88% | 100% / 36% / 33% / 79% | 100% / 36% / 67% / 94% | 416.0 | 27.2 | +| per-class | класс | 33 | 40% / 81% / 82% / 88% | 100% / 74% / 80% / 97% | 100% / 73% / 80% / 97% | 416.0 | 27.2 | + +# learn-tests (per-test) на jsoup (java): покрытие по тестам против `tests-for` +jsoup d90d57521d93, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 21 | gitnext | 26% | 70% | 69% | 90% | +| pick (кэш eval) | 21 | coverage | 91% | 100% | 100% | 100% | +| pick (кэш eval) | 21 | union | 94% | 95% | 100% | 100% | +| случайные | 12 | gitnext | 34% | 53% | 62% | 83% | +| случайные | 12 | coverage | 100% | 85% | 86% | 92% | +| случайные | 12 | union | 100% | 61% | 78% | 92% | +| все | 33 | gitnext | 29% | 64% | 66% | 88% | +| все | 33 | coverage | 94% | 94% | 94% | 97% | +| все | 33 | union | 96% | 83% | 92% | 97% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::jsoup::internal::LegacyMatching::cssAttributeMatches` | 7 | 0% / 0% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::Normalizer::lowerCase` | 1689 | 70% / 80% | 18% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, absAttr | +| pick | `org::jsoup::internal::SoftPool::release` | 1755 | 46% / 60% | 98% / 100% / 90% | adoptionDoesNotAddSecondDocumentElement, canCreateEmptyConnection, caseInsensitiveHeaders, downgradesSocksProxyToUrlConnectionExecutor | +| pick | `org::jsoup::internal::StringUtil::appendNormalisedWhitespace` | 446 | 29% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::StringUtil::isActuallyWhitespace` | 446 | 28% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::StringUtil::isAsciiLetter` | 1699 | 2% / 90% | 100% / 100% / 100% | init, provideEvaluators, rcDataRecoverParser | +| pick | `org::jsoup::internal::StringUtil::trimAsciiWhitespace` | 1811 | 8% / 100% | 76% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::nodes::Attributes::deduplicate` | 780 | 1% / 100% | 100% / 100% / 100% | init, provideEvaluators | +| pick | `org::jsoup::nodes::Attributes::getIgnoreCase` | 610 | 35% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::Attributes::removeIgnoreCase` | 28 | 7% / 10% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::DocumentType::publicId` | 115 | 3% / 43% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::Entities::isNamedEntity` | 51 | 16% / 30% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::Parser::getMaxDepth` | 1467 | 0% / 40% | 100% / 100% / 100% | init, provideEvaluators | +| pick | `org::jsoup::parser::Tag::isFormSubmittable` | 16 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::Token::Doctype::getSystemIdentifier` | 130 | 0% / 0% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::TokenQueue::consumeCssIdentifier` | 383 | 2% / 90% | 100% / 99% / 100% | | +| pick | `org::jsoup::parser::TokenQueue::matchesWhitespace` | 1721 | 7% / 90% | 59% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::parser::TokenQueue::matchesWord` | 1717 | 0% / 30% | 59% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::safety::Safelist::addEnforcedAttribute` | 38 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::safety::Safelist::addTags` | 72 | 99% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::select::Evaluator::asPredicate` | 957 | 2% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::ownerDocument` | 891 | 85% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::NodeInternals::attributeRange` | 39 | 0% / 0% | 100% / 56% / 30% | | +| random | `org::jsoup::parser::TokeniserState::read` | 1 | 0% / 0% | 100% / 0% / 0% | | +| random | `org::jsoup::helper::HttpConnection::Response::sameOrigin` | 19 | 11% / 20% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::outerHtml` | 116 | 89% / 100% | 100% / 65% / 100% | | +| random | `org::jsoup::nodes::Element::TextAccumulator::hasBlockChild` | 123 | 21% / 60% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::setParentNode` | 1554 | 21% / 70% | 100% / 100% / 100% | init, provideEvaluators | +| random | `org::jsoup::nodes::Document::text` | 1 | 100% / 10% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::DataNode::outerHtmlHead` | 37 | 19% / 100% | 100% / 100% / 100% | | +| random | `org::jsoup::parser::Token::Character::normalizeNulls` | 1090 | 0% / 100% | 100% / 100% / 100% | init, provideEvaluators | +| random | `org::jsoup::parser::TreeBuilder::parseAttributes` | 29 | 59% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Entities::encoderFor` | 573 | 2% / 100% | 100% / 98% / 100% | | + +## Цена + +- `how`: per-test +- `class_inits_s`: 3.6 +- `class_inits_booked`: True +- `files_with_init_lines`: 64 +- `mvn_s`: 20.2 +- `analyze_s`: 25.7 +- `map_s`: 49.6 +- `exec_bytes`: 8545291 +- `dumps`: 5372 +- `test_methods`: 2130 +- `classes`: 76 +- `classes_with_setup_lines`: 6 +- `run_failed`: False +- `names_in_several_classes`: 115 +- `junit`: 5 +- `package`: org.jsoup +- `owner`: body +- `repeat`: 3 +- `plain_s`: 14.2 +- `tests`: 2130 +- `map_pairs`: 556456 +- `map_functions`: 1771 +- `map_sqlite_bytes`: 5718016 +- `functions_with_body`: 2299 +- `functions_never_run`: 99 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 3.5 + +# learn-tests (per-test-fork) на jsoup (java): покрытие по тестам против `tests-for` +jsoup d90d57521d93, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 21 | gitnext | 26% | 70% | 69% | 90% | +| pick (кэш eval) | 21 | coverage | 91% | 100% | 100% | 100% | +| pick (кэш eval) | 21 | union | 94% | 95% | 100% | 100% | +| случайные | 12 | gitnext | 34% | 53% | 62% | 83% | +| случайные | 12 | coverage | 100% | 85% | 86% | 92% | +| случайные | 12 | union | 100% | 61% | 78% | 92% | +| все | 33 | gitnext | 29% | 64% | 66% | 88% | +| все | 33 | coverage | 94% | 94% | 94% | 97% | +| все | 33 | union | 96% | 83% | 92% | 97% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::jsoup::internal::LegacyMatching::cssAttributeMatches` | 7 | 0% / 0% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::Normalizer::lowerCase` | 1689 | 70% / 80% | 21% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, absAttr | +| pick | `org::jsoup::internal::SoftPool::release` | 1755 | 46% / 60% | 98% / 100% / 90% | adoptionDoesNotAddSecondDocumentElement, canCreateEmptyConnection, caseInsensitiveHeaders, downgradesSocksProxyToUrlConnectionExecutor | +| pick | `org::jsoup::internal::StringUtil::appendNormalisedWhitespace` | 446 | 29% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::StringUtil::isActuallyWhitespace` | 446 | 28% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::internal::StringUtil::isAsciiLetter` | 1699 | 2% / 90% | 100% / 100% / 100% | init, provideEvaluators, rcDataRecoverParser | +| pick | `org::jsoup::internal::StringUtil::trimAsciiWhitespace` | 1811 | 8% / 100% | 78% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::nodes::Attributes::deduplicate` | 780 | 1% / 100% | 100% / 100% / 100% | init, provideEvaluators | +| pick | `org::jsoup::nodes::Attributes::getIgnoreCase` | 610 | 35% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::Attributes::removeIgnoreCase` | 28 | 7% / 10% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::DocumentType::publicId` | 115 | 3% / 43% | 100% / 100% / 100% | | +| pick | `org::jsoup::nodes::Entities::isNamedEntity` | 51 | 16% / 30% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::Parser::getMaxDepth` | 1467 | 0% / 40% | 100% / 100% / 100% | init, provideEvaluators | +| pick | `org::jsoup::parser::Tag::isFormSubmittable` | 16 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::Token::Doctype::getSystemIdentifier` | 130 | 0% / 0% | 100% / 100% / 100% | | +| pick | `org::jsoup::parser::TokenQueue::consumeCssIdentifier` | 383 | 2% / 90% | 100% / 99% / 100% | | +| pick | `org::jsoup::parser::TokenQueue::matchesWhitespace` | 1721 | 7% / 90% | 61% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::parser::TokenQueue::matchesWord` | 1717 | 0% / 30% | 61% / 100% / 100% | DocumentsWithSameContentAreEqual, DocumentsWithSameContentAreVerifiable, NoErrorOnEofIfBodyOpen, acceptsLenientXmlProcessingInstructionTarget | +| pick | `org::jsoup::safety::Safelist::addEnforcedAttribute` | 38 | 100% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::safety::Safelist::addTags` | 72 | 99% / 100% | 100% / 100% / 100% | | +| pick | `org::jsoup::select::Evaluator::asPredicate` | 957 | 2% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::ownerDocument` | 891 | 85% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::NodeInternals::attributeRange` | 39 | 0% / 0% | 100% / 56% / 30% | | +| random | `org::jsoup::parser::TokeniserState::read` | 1 | 0% / 0% | 100% / 0% / 0% | | +| random | `org::jsoup::helper::HttpConnection::Response::sameOrigin` | 19 | 11% / 20% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::outerHtml` | 116 | 89% / 100% | 100% / 65% / 100% | | +| random | `org::jsoup::nodes::Element::TextAccumulator::hasBlockChild` | 123 | 21% / 60% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Node::setParentNode` | 1554 | 21% / 70% | 100% / 100% / 100% | init, provideEvaluators | +| random | `org::jsoup::nodes::Document::text` | 1 | 100% / 10% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::DataNode::outerHtmlHead` | 37 | 19% / 100% | 100% / 100% / 100% | | +| random | `org::jsoup::parser::Token::Character::normalizeNulls` | 1090 | 0% / 100% | 100% / 100% / 100% | init, provideEvaluators | +| random | `org::jsoup::parser::TreeBuilder::parseAttributes` | 29 | 59% / 90% | 100% / 100% / 100% | | +| random | `org::jsoup::nodes::Entities::encoderFor` | 573 | 2% / 100% | 100% / 98% / 100% | | + +## Цена + +- `how`: per-test-fork +- `class_inits_s`: 3.3 +- `class_inits_booked`: True +- `files_with_init_lines`: 64 +- `mvn_s`: 110.0 +- `analyze_s`: 25.8 +- `map_s`: 139.0 +- `exec_bytes`: 8878815 +- `dumps`: 5372 +- `test_methods`: 2130 +- `classes`: 76 +- `classes_with_setup_lines`: 6 +- `run_failed`: False +- `names_in_several_classes`: 115 +- `junit`: 5 +- `package`: org.jsoup +- `owner`: body +- `repeat`: 3 +- `plain_s`: 16.5 +- `tests`: 2130 +- `map_pairs`: 559173 +- `map_functions`: 1771 +- `map_sqlite_bytes`: 5746688 +- `functions_with_body`: 2299 +- `functions_never_run`: 99 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 8.4 + +# learn-tests (per-class) на jsoup (java): покрытие по тестам против `tests-for` +jsoup d90d57521d93, gitnext 14bcb17, seed 7. Истина — eval (тесты, падающие при throw в начале функции). pick — функции с ответом в кэше eval, random — случайные. + +| выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---:|---|---:|---:|---:|---:| +| pick (кэш eval) | 21 | gitnext | 26% | 70% | 69% | 90% | +| pick (кэш eval) | 21 | coverage | 100% | 42% | 37% | 86% | +| pick (кэш eval) | 21 | union | 100% | 42% | 71% | 95% | +| случайные | 12 | gitnext | 34% | 53% | 62% | 83% | +| случайные | 12 | coverage | 100% | 24% | 25% | 67% | +| случайные | 12 | union | 100% | 23% | 58% | 92% | +| все | 33 | gitnext | 29% | 64% | 66% | 88% | +| все | 33 | coverage | 100% | 36% | 33% | 79% | +| все | 33 | union | 100% | 36% | 67% | 94% | + +## По функциям + +| выборка | функция | падают | граф: полн / из 10 | покрытие: полн / точн / из 10 | мимо покрытия | +|---|---|---:|---|---|---| +| pick | `org::jsoup::internal::LegacyMatching::cssAttributeMatches` | 7 | 0% / 0% | 100% / 4% / 0% | | +| pick | `org::jsoup::internal::Normalizer::lowerCase` | 1689 | 70% / 80% | 100% / 87% / 30% | emptyCookieJar, init, provideEvaluators, testOutputSettings | +| pick | `org::jsoup::internal::SoftPool::release` | 1755 | 46% / 60% | 100% / 89% / 70% | emptyCookieJar, init, provideEvaluators, testOutputSettings | +| pick | `org::jsoup::internal::StringUtil::appendNormalisedWhitespace` | 446 | 29% / 100% | 100% / 26% / 0% | | +| pick | `org::jsoup::internal::StringUtil::isActuallyWhitespace` | 446 | 28% / 100% | 100% / 26% / 0% | | +| pick | `org::jsoup::internal::StringUtil::isAsciiLetter` | 1699 | 2% / 90% | 100% / 88% / 20% | init, provideEvaluators, rcDataRecoverParser | +| pick | `org::jsoup::internal::StringUtil::trimAsciiWhitespace` | 1811 | 8% / 100% | 100% / 93% / 70% | emptyCookieJar, init, provideEvaluators, rcDataRecoverParser | +| pick | `org::jsoup::nodes::Attributes::deduplicate` | 780 | 1% / 100% | 100% / 45% / 100% | init, provideEvaluators | +| pick | `org::jsoup::nodes::Attributes::getIgnoreCase` | 610 | 35% / 100% | 100% / 36% / 20% | | +| pick | `org::jsoup::nodes::Attributes::removeIgnoreCase` | 28 | 7% / 10% | 100% / 5% / 10% | | +| pick | `org::jsoup::nodes::DocumentType::publicId` | 115 | 3% / 43% | 100% / 9% / 30% | | +| pick | `org::jsoup::nodes::Entities::isNamedEntity` | 51 | 16% / 30% | 100% / 5% / 30% | | +| pick | `org::jsoup::parser::Parser::getMaxDepth` | 1467 | 0% / 40% | 100% / 79% / 20% | init, provideEvaluators | +| pick | `org::jsoup::parser::Tag::isFormSubmittable` | 16 | 100% / 100% | 100% / 4% / 10% | | +| pick | `org::jsoup::parser::Token::Doctype::getSystemIdentifier` | 130 | 0% / 0% | 100% / 10% / 30% | | +| pick | `org::jsoup::parser::TokenQueue::consumeCssIdentifier` | 383 | 2% / 90% | 100% / 27% / 30% | | +| pick | `org::jsoup::parser::TokenQueue::matchesWhitespace` | 1721 | 7% / 90% | 100% / 89% / 90% | emptyCookieJar, init, provideEvaluators, testOutputSettings | +| pick | `org::jsoup::parser::TokenQueue::matchesWord` | 1717 | 0% / 30% | 100% / 89% / 90% | emptyCookieJar, init, provideEvaluators, testOutputSettings | +| pick | `org::jsoup::safety::Safelist::addEnforcedAttribute` | 38 | 100% / 100% | 100% / 10% / 30% | | +| pick | `org::jsoup::safety::Safelist::addTags` | 72 | 99% / 100% | 100% / 19% / 90% | | +| pick | `org::jsoup::select::Evaluator::asPredicate` | 957 | 2% / 90% | 100% / 53% / 10% | | +| random | `org::jsoup::nodes::Node::ownerDocument` | 891 | 85% / 90% | 100% / 51% / 60% | | +| random | `org::jsoup::nodes::NodeInternals::attributeRange` | 39 | 0% / 0% | 100% / 7% / 10% | | +| random | `org::jsoup::parser::TokeniserState::read` | 1 | 0% / 0% | 100% / 0% / 0% | | +| random | `org::jsoup::helper::HttpConnection::Response::sameOrigin` | 19 | 11% / 20% | 100% / 14% / 10% | | +| random | `org::jsoup::nodes::Node::outerHtml` | 116 | 89% / 100% | 100% / 8% / 40% | | +| random | `org::jsoup::nodes::Element::TextAccumulator::hasBlockChild` | 123 | 21% / 60% | 100% / 9% / 0% | | +| random | `org::jsoup::nodes::Node::setParentNode` | 1554 | 21% / 70% | 100% / 83% / 20% | init, provideEvaluators | +| random | `org::jsoup::nodes::Document::text` | 1 | 100% / 10% | 100% / 3% / 10% | | +| random | `org::jsoup::nodes::DataNode::outerHtmlHead` | 37 | 19% / 100% | 100% / 4% / 70% | | +| random | `org::jsoup::parser::Token::Character::normalizeNulls` | 1090 | 0% / 100% | 100% / 60% / 0% | init, provideEvaluators | +| random | `org::jsoup::parser::TreeBuilder::parseAttributes` | 29 | 59% / 90% | 100% / 9% / 80% | | +| random | `org::jsoup::nodes::Entities::encoderFor` | 573 | 2% / 100% | 100% / 36% / 0% | | + +## Цена + +- `how`: per-class +- `mvn_s`: 413.9 +- `analyze_s`: 2.1 +- `map_s`: 416.0 +- `exec_bytes`: 657374 +- `dumps`: 76 +- `test_methods`: 2130 +- `classes`: 76 +- `names_in_several_classes`: 115 +- `junit`: 5 +- `package`: org.jsoup +- `owner`: body +- `repeat`: 3 +- `plain_s`: 15.3 +- `tests`: 2130 +- `map_pairs`: 1521462 +- `map_functions`: 1771 +- `map_sqlite_bytes`: 15364096 +- `functions_with_body`: 2299 +- `functions_never_run`: 99 +- `fail_alone_not_together`: [] +- `fail_together_not_alone`: [] +- `slowdown_runs`: 27.2 + +# По классу теста + +Класс засчитан, если в нём есть метод с падающим именем (имя в нескольких классах — все они). + +| способ | выборка | функций | источник | полнота | точность | из 10 падают | поймано | +|---|---|---:|---|---:|---:|---:|---:| +| per-test | pick | 21 | gitnext | 39% | 89% | 89% | 90% | +| per-test | pick | 21 | coverage | 96% | 100% | 100% | 100% | +| per-test | pick | 21 | union | 98% | 100% | 100% | 100% | +| per-test | random | 12 | gitnext | 42% | 68% | 71% | 83% | +| per-test | random | 12 | coverage | 100% | 86% | 88% | 92% | +| per-test | random | 12 | union | 100% | 75% | 78% | 92% | +| per-test | all | 33 | gitnext | 40% | 81% | 82% | 88% | +| per-test | all | 33 | coverage | 98% | 95% | 96% | 97% | +| per-test | all | 33 | union | 99% | 91% | 92% | 97% | +| per-test-fork | pick | 21 | gitnext | 39% | 89% | 89% | 90% | +| per-test-fork | pick | 21 | coverage | 100% | 100% | 100% | 100% | +| per-test-fork | pick | 21 | union | 100% | 100% | 100% | 100% | +| per-test-fork | random | 12 | gitnext | 42% | 68% | 71% | 83% | +| per-test-fork | random | 12 | coverage | 100% | 86% | 88% | 92% | +| per-test-fork | random | 12 | union | 100% | 75% | 78% | 92% | +| per-test-fork | all | 33 | gitnext | 40% | 81% | 82% | 88% | +| per-test-fork | all | 33 | coverage | 100% | 95% | 96% | 97% | +| per-test-fork | all | 33 | union | 100% | 91% | 92% | 97% | +| per-class | pick | 21 | gitnext | 39% | 89% | 89% | 90% | +| per-class | pick | 21 | coverage | 100% | 80% | 85% | 100% | +| per-class | pick | 21 | union | 100% | 80% | 86% | 100% | +| per-class | random | 12 | gitnext | 42% | 68% | 71% | 83% | +| per-class | random | 12 | coverage | 100% | 65% | 71% | 92% | +| per-class | random | 12 | union | 100% | 62% | 72% | 92% | +| per-class | all | 33 | gitnext | 40% | 81% | 82% | 88% | +| per-class | all | 33 | coverage | 100% | 74% | 80% | 97% | +| per-class | all | 33 | union | 100% | 73% | 80% | 97% | + +# Цена + +| | per-test | per-test-fork | per-class | +|---|---:|---:|---:| +| plain_s | 14.2 | 16.5 | 15.3 | +| mvn_s | 20.2 | 110.0 | 413.9 | +| analyze_s | 25.7 | 25.8 | 2.1 | +| map_s | 49.6 | 139.0 | 416.0 | +| slowdown_runs | 3.5 | 8.4 | 27.2 | +| exec_bytes | 8545291 | 8878815 | 657374 | +| map_sqlite_bytes | 5718016 | 5746688 | 15364096 | +| dumps | 5372 | 5372 | 76 | +| test_methods | 2130 | 2130 | 2130 | +| classes | 76 | 76 | 76 | +| names_in_several_classes | 115 | 115 | 115 | +| map_pairs | 556456 | 559173 | 1521462 | +| map_functions | 1771 | 1771 | 1771 | +| functions_with_body | 2299 | 2299 | 2299 | +| functions_never_run | 99 | 99 | 99 | + +jsoup d90d57521d93 From 6233e3637dc03967c4926cf31b505326206c252a Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 05:18:16 +0300 Subject: [PATCH 15/18] changelog: the Java coverage map prototype Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de27da2..73cc9e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ `tests-for`, `look` and `check` name the test files that really run the code — 100% of the failing ones on yaml and zustand (the call graph: 16% and 68%), for 2.6–6.6 ordinary test runs. Says which package to install when `@vitest/coverage-v8` is missing; jest is not supported yet. +- eval: a coverage map for Java measured as a prototype (`eval/learn_tests_java.py`, JaCoCo with a dump per test + in one Maven run): 94% and 99% of the failing tests on jsoup and json-java (the call graph: 29% and 71%) for + 2.1–3.5 ordinary test runs. Not in `learn-tests` yet. - **Rust: functions declared inside a macro call are symbols** (`quickcheck! { fn prop(..) -> bool { .. } }`, `proptest!`, a local `macro_rules!` that writes `#[test]`). In indexmap 43 of 192 tests were outside the graph and the coverage map, now 4; `why` put the function at fault first in 14 of 20 seeded Rust bugs, now in 16–18, and From 9c1a8e670536b4ac45a5c4daa2df8b39d1dae6ed Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 09:46:44 +0300 Subject: [PATCH 16/18] rename --text reaches the documentation; check says a failing test comes first; look takes a list From the agent runs on three models (648 runs): - rename done with the tool failed the judge on the same tasks in every model. Two of the five: the old name stayed in docs/*.rst. rename now reports the name in documentation the index lists but does not parse (.md .rst .txt .adoc) where it is written as code, and --text rewrites it. - with eleven tools the 9B model never began a fix with check (0 of 18), the 4B model 5 times; with five tools 16 of 18. The check description (MCP, eval) now begins with the failing test. - look: 'a.go:497-506, a_test.go:123-128' is two places; 'TestX/subtest' is TestX. Co-Authored-By: Claude Fable 5.2 --- crates/gitnext-cli/src/mcp.rs | 2 +- crates/gitnext-ops/src/edit.rs | 21 ++++++++++++++++++ crates/gitnext-ops/src/look.rs | 13 +++++++++++ crates/gitnext-ops/tests/look.rs | 4 ++++ crates/gitnext-ops/tests/rename_impact.rs | 27 +++++++++++++++++++++++ eval/agent.py | 4 ++-- 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/gitnext-cli/src/mcp.rs b/crates/gitnext-cli/src/mcp.rs index d6a9789..dc7eeb3 100644 --- a/crates/gitnext-cli/src/mcp.rs +++ b/crates/gitnext-cli/src/mcp.rs @@ -104,7 +104,7 @@ fn tools() -> Value { json!({ "items": { "type": "array", "items": { "type": "object", "properties": { "symbol": { "type": "string" }, "summary": { "type": "string" } }, "required": ["symbol", "summary"] } } }), &["items"]), tool("locate", "Paste a compiler error, panic, test failure or traceback: returns the repo symbols at those file:line locations (innermost function/method) and the code of the top one.", json!({ "text": { "type": "string" } }), &["text"]), - tool("check", "After a change: builds the project and runs the tests of what changed, answers with a verdict (ok / what failed, where, the message). With `test`: the same as `why`.", + tool("check", "A test fails: call this first with test = its name — the same as `why`. After a change: call it with no arguments — builds the project and runs the tests of what changed, answers with a verdict (ok / what failed, where, the message).", json!({ "test": { "type": "string", "description": "a failing test to explain instead of checking the change" } }), &[]), tool("why", "A test fails: runs it and returns the failure in a few lines (what, where, the message) and the code the test reaches, most likely culprit first, with the code of the top ones. Use this first when told that a test fails.", json!({ "test": { "type": "string", "description": "test name; `greater_than` finds `test_greater_than`" } }), &["test"]), diff --git a/crates/gitnext-ops/src/edit.rs b/crates/gitnext-ops/src/edit.rs index 75034cb..69a1a45 100644 --- a/crates/gitnext-ops/src/edit.rs +++ b/crates/gitnext-ops/src/edit.rs @@ -257,6 +257,11 @@ pub fn rename_leftover_text(root: &Path, r: &mut RenameReport) -> Result<()> { Ok(()) } +/// Prose about the code: a rename looks for the old name there too. +fn is_doc(path: &str) -> bool { + matches!(path.rsplit('.').next().unwrap_or(""), "md" | "mdx" | "rst" | "txt" | "adoc") +} + fn is_ident(c: u8) -> bool { c.is_ascii_alphanumeric() || c == b'_' } @@ -414,6 +419,22 @@ pub fn rename(root: &Path, g: &Graph, i: usize, new_name: &str, dry_run: bool) - } } } + // documentation the index lists but does not parse (.md, .rst, .txt, .adoc): the name there is text. + // Only where it is written as code — `name()`, `` `name` ``, `Type.name` — or cannot be an English word. + let distinctive = old.contains('_') || old.chars().skip(1).any(|c| c.is_uppercase() || c.is_ascii_digit()); + for f in g.files.iter().filter(|f| !f.parsed && is_doc(&f.path)) { + for (n, text) in lines_of(&f.path).iter().enumerate() { + let b = text.as_bytes(); + let code_shaped = |c: usize| { + distinctive + || matches!(b.get(c + old.len()), Some(b'(' | b'`')) + || (c > 0 && matches!(b[c - 1], b'`' | b'.' | b':' | b'#')) + }; + if word_hits(text, &old).into_iter().any(code_shaped) { + report.leftover.push(format!("{}:{}: {}", f.path, n + 1, text.trim().chars().take(100).collect::())); + } + } + } report.leftover.sort(); report.leftover.dedup(); diff --git a/crates/gitnext-ops/src/look.rs b/crates/gitnext-ops/src/look.rs index 2784421..0475292 100644 --- a/crates/gitnext-ops/src/look.rs +++ b/crates/gitnext-ops/src/look.rs @@ -36,10 +36,23 @@ pub fn look(ws: &Workspace, g: &Graph, what: &str, lite: bool) -> String { Some((a, _)) if file_of(g, a.trim()).is_some() => a.trim(), _ => what, }; + // several places in one target: `a.go:497-506, a_test.go:123-128` + if what.contains(", ") { + let parts: Vec<&str> = what.split(", ").map(str::trim).filter(|p| !p.is_empty()).collect(); + if parts.len() > 1 && parts.len() <= 6 && parts.iter().all(|p| place(ws, g, p, lite).is_some() || file_of(g, p).is_some()) { + return parts.iter().map(|p| look(ws, g, p, lite)).collect::>().join("\n"); + } + } // a place in a file, the way tracebacks, test runners and models write it if let Some(s) = place(ws, g, what, lite) { return s; } + // a subtest as Go names it: `TestClientIP/earlier_persists` is in `TestClientIP` + if let Some((test, _)) = what.split_once('/') { + if !test.contains(['.', ' ']) && file_of(g, what).is_none() && g.find(test).iter().any(|&i| g.syms[i].sym.test) { + return look(ws, g, test, lite); + } + } // a file or folder (exact, or a unique file name like `utils.py`) let p = what.trim_start_matches("./").trim_end_matches('/'); if let Some(f) = file_of(g, what) { diff --git a/crates/gitnext-ops/tests/look.rs b/crates/gitnext-ops/tests/look.rs index 2a7117d..303396e 100644 --- a/crates/gitnext-ops/tests/look.rs +++ b/crates/gitnext-ops/tests/look.rs @@ -55,6 +55,10 @@ fn a_name_in_a_file_is_found_however_it_is_written() { // a file followed by a shell pipe is the file let piped = look(&ws, &g, "src/display.rs | grep -n pad", false); assert_eq!(piped, look(&ws, &g, "src/display.rs", false)); + // several places at once, and a Go-style subtest + let two = look(&ws, &g, "src/eval.rs:5-7, src/display.rs:1-3", false); + assert!(two.contains("src/eval.rs:5-7 (of 12 lines)") && two.contains("src/display.rs:1-3 (of 7 lines)"), "{two}"); + assert_eq!(look(&ws, &g, "test_greater_than/case_one", false), look(&ws, &g, "test_greater_than", false)); // names with `::` that are not files are names as before let q = look(&ws, &g, "eval::matches_greater", false); assert_eq!(q, by_name); diff --git a/crates/gitnext-ops/tests/rename_impact.rs b/crates/gitnext-ops/tests/rename_impact.rs index 679fe6f..f17585d 100644 --- a/crates/gitnext-ops/tests/rename_impact.rs +++ b/crates/gitnext-ops/tests/rename_impact.rs @@ -111,3 +111,30 @@ fn rename_text_leaves_other_symbols_in_code_alone() { assert!(text.contains("# show() is ours") && text.contains("def show(x)") && text.contains("return show(1), 'show'"), "{text}"); std::fs::remove_dir_all(&dir).ok(); } + +#[test] +fn rename_text_reaches_the_documentation() { + let dir = project( + "rename-docs", + &[ + ("app/__init__.py", ""), + ("app/lexer.py", "def default_initialization():\n return 1\n\n\ndef close():\n return default_initialization()\n"), + ("docs/extending.rst", "Call ``default_initialization()`` first.\nThen close the file; `close()` does it.\n"), + ("README.md", "Use `Lexer.default_initialization` to start. Keep it close at hand.\n"), + ], + ); + let g = graph(&dir); + let mut r = edit::rename(&dir, &g, g.find("default_initialization")[0], "default_init", false).unwrap(); + assert!(r.leftover.iter().any(|l| l.starts_with("docs/extending.rst:1:")) && r.leftover.iter().any(|l| l.starts_with("README.md:1:")), "{:?}", r.leftover); + edit::rename_leftover_text(&dir, &mut r).unwrap(); + assert_eq!(std::fs::read_to_string(dir.join("docs/extending.rst")).unwrap(), "Call ``default_init()`` first.\nThen close the file; `close()` does it.\n"); + assert_eq!(std::fs::read_to_string(dir.join("README.md")).unwrap(), "Use `Lexer.default_init` to start. Keep it close at hand.\n"); + // a name that is also a word: in the documentation only where it is written as code + let g = graph(&dir); + let mut r = edit::rename(&dir, &g, g.find("close")[0], "shut", false).unwrap(); + assert_eq!(r.leftover.iter().filter(|l| l.starts_with("docs/") || l.starts_with("README")).count(), 1, "{:?}", r.leftover); + edit::rename_leftover_text(&dir, &mut r).unwrap(); + assert_eq!(std::fs::read_to_string(dir.join("docs/extending.rst")).unwrap(), "Call ``default_init()`` first.\nThen close the file; `shut()` does it.\n"); + assert_eq!(std::fs::read_to_string(dir.join("README.md")).unwrap(), "Use `Lexer.default_init` to start. Keep it close at hand.\n"); + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/eval/agent.py b/eval/agent.py index 32cc71d..0d2395f 100644 --- a/eval/agent.py +++ b/eval/agent.py @@ -87,8 +87,8 @@ def tool(name, desc, props, req): tool("look", "gitnext: everything about a function, class, file or error in one call — its code, who calls it, what it calls, the tests that reach it and how to run them. Use this first.", {"what": S}, ["what"]), tool("rename", "gitnext: rename a function/class everywhere (definition, calls, imports, tests, comments).", {"old": S, "new": S}, ["old", "new"]), tool("replace_symbol", "gitnext: replace the whole code of one function/class by name, without reading the file.", {"symbol": S, "code": S}, ["symbol", "code"]), - tool("check", "gitnext: run the tests. No arguments: after a change — builds the project, runs the tests of what you changed, answers with a verdict. " - "With `test`: why that test fails — the failure in a few lines and the code to fix, the most likely culprit first.", {"test": S}, []), + tool("check", "gitnext: a test fails — call this first with test = its name: it runs the test and shows the failure in a few lines and the code to fix, " + "the most likely culprit first. After a change: call it with no arguments — builds the project, runs the tests of what you changed, answers with a verdict.", {"test": S}, []), ] # mode `lite`: these and nothing else — what gitnext's lite MCP profile gives a small model (plus `finish`) LITE_TOOLS = [ From f23100956f6cafacebdd541edabf3afb3429a1f7 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 09:46:45 +0300 Subject: [PATCH 17/18] the agent runs on three models: 216 tasks each way; README and charts follow them Qwen 3.5 4B, 9B, 27B on six projects, the same 72 tasks plain / gitnext / lite: 144 -> 156 -> 163 of 216 solved; a median of 9-10 steps -> 4-5 -> 3-4; the 27B model fixes 14 of 18 seeded bugs against 7. For one model the difference in solved tasks is inside the noise. The 24-task comparison of the earlier runs (4 against 15) did not repeat - plain solves 19 of those 24 with today's harness - and is no longer quoted. Every row: eval/results/agent-kaggle-20261113.json; tables: eval/agent_stats.py. why on Rust with the map: 17 of 20 after macros became symbols (chart: 86% of 57). Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 6 ++++ README.md | 20 ++++++------- README.ru.md | 20 ++++++------- docs/images/agent.ru.svg | 58 ++++++++++++++++++++------------------ docs/images/agent.svg | 58 ++++++++++++++++++++------------------ docs/images/make_charts.py | 36 +++++++++++------------ docs/images/why.ru.svg | 12 ++++---- docs/images/why.svg | 12 ++++---- eval/agent_stats.py | 48 +++++++++++++++++++++++++++++++ eval/results/COVERAGE.md | 2 +- eval/results/HISTORY.md | 49 ++++++++++++++++++++++++++++++++ 11 files changed, 214 insertions(+), 107 deletions(-) create mode 100644 eval/agent_stats.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 73cc9e8..c54b240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ `src/eval.rs::matches_greater`, `tests/test_x.py::TestA::test_b[case]`, `eval.rs matches_greater`. Words of which only some are names list those names instead of "nothing found". In a trial with Qwen 3.5 4B and 27B, 37 of 89 `look` calls were written one of these ways and found nothing; all are answered now. +- **`gitnext rename --text` reaches the documentation** (`.md`, `.rst`, `.txt`, `.adoc`): the old name written as + code there (`name()`, `` `name` ``, `Type.name`, or a name that cannot be a word) is renamed with the rest. +- `look` also takes a list of places (`a.go:497-506, a_test.go:123-128`) and a Go subtest (`TestX/case`); the MCP + `check` tool says first that a failing test is what it is for. +- eval: the agent runs repeated on three models, 216 tasks each way (`eval/results/HISTORY.md`): +9 points of solved + tasks with the five tools, a third of the steps; the earlier 24-task comparison did not repeat. - **`gitnext dead` names only what nothing refers to.** It knew calls only: a function used as a value, a class used as a type, a base or a metaclass, an imported name, a Go `init` were "likely dead" — 160 of 225 on ten projects, and deleting them broke the build of six. Now the name must be written nowhere outside the symbol. diff --git a/README.md b/README.md index 71b2baa..63c8f05 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,15 @@ The repository stays plain git — GitHub, CI and your other tools keep working. Everything here is measured on other people's code — ten open-source projects in five languages — and judged by their own compilers and test suites. How each number is obtained and how much of it is noise: [eval/results/HISTORY.md](eval/results/HISTORY.md). -### A small local model solves more +### Agents get there in a third of the steps -

Tasks solved by a 4B local model: with plain tools 4 of 24, with gitnext tools 17 of 24

+

Tasks solved by three local models: 144 of 216 with plain tools, 163 of 216 with the five gitnext tools

-The same 4B model, the same tasks, the same judge. With plain tools it guesses file paths and answers from memory; with gitnext one call gives it the code, the callers and the tests. For small models there is a short list of five tools (`gitnext mcp --profile lite`): in a separate run of 18 tasks the model solved 17 with the five and 14 with eleven, and spent 15% fewer tokens. +Three local models (Qwen 3.5 4B, 9B and 27B), the same 72 tasks each way — find a function, list its callers, rename it, fix a seeded bug — on six projects in Python, Rust and Go, judged by the projects' own compilers and tests. With the five gitnext tools (`gitnext mcp --profile lite`) they solve 163 of 216 tasks against 144 with plain read / grep / edit / run (+9 points; the 95% interval is +1 to +17), in a median of 3–4 steps instead of 9–10, and the 27B model sends 2.4 times fewer tokens. The clear gain is a bug fixed by the larger model: 14 of 18 against 7, never the other way round. For one model alone the difference in solved tasks is inside the noise, and an earlier 24-task run that showed 4 against 15 for the 4B model did not repeat: with today's harness the plain model solves 19 of those 24. ### A failing test leads to the function at fault -

Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 81% with the learned coverage map

+

Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 86% with the learned coverage map

`gitnext why ` runs the test and answers in a dozen lines: the failure itself, the code the test ran with the most likely culprit first, and the lines that changed since the tests last passed. `gitnext learn-tests` is one test run with coverage (Python, Go, Rust); a long build or test log goes through `gitnext digest` (a vitest run with 35 failures: 843 lines → 12). @@ -45,7 +45,7 @@ The same 4B model, the same tasks, the same judge. With plain tools it guesses f | who calls a function — correct | 98% | — | 100% | 100% | 100% | **99%** ±1 | | rename — the project still builds and its tests pass | 93% | 90% | 85% | 96% | 87% | **91%** ±3 | | tests for a function — one of the first 10 named does fail | 84% | 81% | 41% | 95% | 73% | **79%** ±6 | -| … with a learned coverage map — failing tests found | 100% | 100% | — | — | 100% | | +| … with a learned coverage map — failing tests found | 100% | 100% | 100% | — | 100% | | 232 functions for callers, 338 renames, 167 functions for tests; ± is the 90% interval, a difference inside it is noise. Python has no compiler to say who calls what, so callers are not scored there. `python3 eval/run.py` repeats the run; the full table is [eval/results/latest.md](eval/results/latest.md). @@ -57,11 +57,11 @@ Most of an agent session is the system prompt re-sent on every step, so the win ### What is checked, and what is not good yet -- 138 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). -- TypeScript: `tests-for` finds 19% of the failing test files (it was 3%); the coverage map exists for Python, Go and Rust only. +- 139 unit tests; parser output is pinned by fixtures in eight languages; every command and every MCP tool — 86 in all — is run on each of the ten projects (860 calls, none fails). What checks each function: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- Java has no coverage map in the product yet (a prototype with JaCoCo finds 94–99% of the failing tests); for JS / TS it needs vitest. - A rename breaks the build or the tests in 9% of cases — mostly dynamic access and names shared by unrelated types. -- A 4B model rarely fixes a seeded bug yet: 3 of 12 on click. -- Rust tests written inside a macro are not seen (22% of the tests of indexmap). +- Small models rarely fix a seeded bug: 6 of 18 at best for the 4B and the 9B model. With the full tool list they do not start from `check ` (the 9B model never did in 18 tasks) — give them the short list. +- A rename done with the tool passed the judge no more often than one done by hand (12–13 of 18 against 14): names that are also common words, and mentions in documentation — the latter are renamed now. ## Install @@ -91,7 +91,7 @@ gitnext # open the TUI | edit one function without reading the file | `gitnext edit replace -f new.py` / `replace_symbol` | | rename everywhere (imports, tests, `__all__`, `mod.func`) | `gitnext rename ` / `rename_symbol` | | what breaks, which tests to run | `gitnext impact ` · `gitnext tests-for ` (the 10 most likely to fail + one command for the rest) | -| which tests really run a function | `gitnext learn-tests` — the tests run once with coverage (Python: pytest-cov; Go; Rust: `llvm-tools-preview`); `tests-for`, `look`, `check` and `why` then use the real list | +| which tests really run a function | `gitnext learn-tests` — the tests run once with coverage (Python: pytest-cov; Go; Rust: `llvm-tools-preview`; JS / TS: vitest with `@vitest/coverage-v8`); `tests-for`, `look`, `check` and `why` then use the real list | | did my change break anything | `gitnext check` — builds and runs the tests of what changed, prints a verdict (`gitnext rename A B --check` does it right after the rename) | | a test fails — why | `gitnext why ` / `check` with `test` — runs it: the failure in a few lines, the code the test ran with the most likely culprit first, the lines changed since the tests passed | | a long build or test log | `cargo test 2>&1 \| gitnext digest` — what failed, where, the message (a vitest run with 35 failures: 843 lines → 12) | diff --git a/README.ru.md b/README.ru.md index f7ecde4..3dce4be 100644 --- a/README.ru.md +++ b/README.ru.md @@ -26,15 +26,15 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он Всё здесь измерено на чужом коде — десять открытых проектов на пяти языках — и судят компиляторы и тесты самих проектов. Как получена каждая цифра и какая её часть — шум: [eval/results/HISTORY.md](eval/results/HISTORY.md). -### Маленькая локальная модель решает больше +### Агент доходит до ответа за треть шагов -

Задания, решённые локальной моделью на 4B: с обычными инструментами 4 из 24, с gitnext 17 из 24

+

Задания, решённые тремя локальными моделями: 144 из 216 с обычными инструментами, 163 из 216 с пятью инструментами gitnext

-Та же модель на 4B, те же задания, тот же судья. С обычными инструментами она угадывает пути к файлам и отвечает по памяти; с gitnext один вызов даёт ей код, вызывающих и тесты. Для маленьких моделей есть короткий список из пяти инструментов (`gitnext mcp --profile lite`): в отдельном замере из 18 заданий модель решила 17 с пятью инструментами и 14 с одиннадцатью, потратив на 15% меньше токенов. +Три локальные модели (Qwen 3.5 4B, 9B и 27B), одни и те же 72 задания в каждом режиме — найти функцию, назвать вызывающих, переименовать, починить подсаженную ошибку — на шести проектах на Python, Rust и Go; судят компилятор и тесты самих проектов. С пятью инструментами gitnext (`gitnext mcp --profile lite`) модели решают 163 задания из 216 против 144 с обычными «читать / искать / править / запускать» (+9 пунктов; 95%-й интервал от +1 до +17), за 3–4 шага вместо 9–10 (медиана), а модель 27B отправляет в 2,4 раза меньше токенов. Заметный выигрыш — починка ошибки большой моделью: 14 из 18 против 7, и ни одного случая наоборот. Для одной модели разница в числе решённых заданий не выходит за погрешность, а прежний замер на 24 заданиях, где у модели 4B было 4 против 15, не повторился: с нынешним агентом модель без gitnext решает 19 из тех же 24. ### От упавшего теста — к виновной функции -

Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 81% с картой покрытия

+

Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 86% с картой покрытия

`gitnext why <тест>` запускает тест и отвечает в десяток строк: сам сбой, код, который тест запускал, — самый вероятный виновник первым — и строки, изменённые с тех пор, как тесты проходили. `gitnext learn-tests` — один прогон тестов с покрытием (Python, Go, Rust). Длинный лог сборки или тестов сжимает `gitnext digest` (прогон vitest с 35 падениями: 843 строки → 12). @@ -46,7 +46,7 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он | кто вызывает функцию — верно | 98% | — | 100% | 100% | 100% | **99%** ±1 | | переименование — проект собирается, тесты проходят | 93% | 90% | 85% | 96% | 87% | **91%** ±3 | | тесты функции — хоть один из первых 10 названных падает | 84% | 81% | 41% | 95% | 73% | **79%** ±6 | -| … с картой покрытия — найдено падающих тестов | 100% | 100% | — | — | 100% | | +| … с картой покрытия — найдено падающих тестов | 100% | 100% | 100% | — | 100% | | 232 функции для вызывающих, 338 переименований, 167 функций для тестов; ± — интервал 90%, разница внутри него — шум. У Python нет компилятора, который скажет, кто кого вызывает, поэтому вызывающие там не оцениваются. Повторить: `python3 eval/run.py`; полная таблица — [eval/results/latest.md](eval/results/latest.md). @@ -58,11 +58,11 @@ gitnext — спутник git (`gitnext` = `git` + индекс кода). Он ### Что проверено и что пока плохо -- 138 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). -- TypeScript: `tests-for` находит 19% падающих файлов тестов (было 3%); карта покрытия есть только для Python, Go и Rust. +- 139 модульных тестов; вывод парсеров закреплён эталонами на восьми языках; каждая команда и каждый MCP-инструмент — всего 86 — запускаются на каждом из десяти проектов (860 вызовов, ни один не падает). Чем проверена каждая функция: [eval/results/COVERAGE.md](eval/results/COVERAGE.md). +- Для Java карты покрытия в продукте пока нет (прототип на JaCoCo находит 94–99% падающих тестов); для JS / TS нужен vitest. - Переименование ломает сборку или тесты в 9% случаев — в основном динамический доступ и одинаковые имена у несвязанных типов. -- Модель на 4B пока редко чинит подсаженную ошибку: 3 из 12 на click. -- Тесты Rust, написанные внутри макроса, не видны (22% тестов indexmap). +- Малые модели редко чинят подсаженную ошибку: в лучшем случае 6 из 18 у 4B и у 9B. С полным набором инструментов они не начинают с `check <тест>` (модель 9B — ни разу за 18 заданий): им нужен короткий набор. +- Переименование инструментом проходило проверку не чаще ручного (12–13 из 18 против 14): имена, совпадающие с обычными словами, и упоминания в документации — последние теперь тоже переименовываются. ## Установка @@ -92,7 +92,7 @@ gitnext # открыть TUI | поправить функцию, не читая файл | `gitnext edit replace -f new.py` / `replace_symbol` | | переименовать везде (импорты, тесты, `__all__`, `mod.func`) | `gitnext rename <старое> <новое>` / `rename_symbol` | | что сломается и какие тесты гонять | `gitnext impact ` · `gitnext tests-for ` (10 самых вероятных + одна команда на остальные) | -| какие тесты на самом деле исполняют функцию | `gitnext learn-tests` — один прогон тестов с покрытием (Python: pytest-cov; Go; Rust: `llvm-tools-preview`); `tests-for`, `look`, `check` и `why` дальше берут настоящий список | +| какие тесты на самом деле исполняют функцию | `gitnext learn-tests` — один прогон тестов с покрытием (Python: pytest-cov; Go; Rust: `llvm-tools-preview`; JS / TS: vitest с `@vitest/coverage-v8`); `tests-for`, `look`, `check` и `why` дальше берут настоящий список | | не сломал ли я что-то правкой | `gitnext check` — собирает проект, гонит тесты изменённого и выдаёт вердикт (`gitnext rename A B --check` — сразу после переименования) | | тест падает — почему | `gitnext why <тест>` / `check` с `test` — запускает его: сбой в нескольких строках, код, который тест запускал (самый вероятный виновник первым), строки, изменённые с тех пор, как тесты проходили | | длинный лог сборки или тестов | `cargo test 2>&1 \| gitnext digest` — что упало, где, сообщение | diff --git a/docs/images/agent.ru.svg b/docs/images/agent.ru.svg index da74b8b..4ac7602 100644 --- a/docs/images/agent.ru.svg +++ b/docs/images/agent.ru.svg @@ -1,33 +1,35 @@ - -Задания, решённые локальной моделью на 4B: с обычными инструментами 4 из 24, с gitnext 17 из 24 + +Задания, решённые тремя локальными моделями: 144 из 216 с обычными инструментами, 163 из 216 с пятью инструментами gitnext -Локальная модель на 4B в настоящих проектах: решено заданий -Qwen 3.5 4B на llama.cpp · click (Python) и semver (Rust) · по 6 заданий каждого вида · судят компилятор и тесты самих проектов +Три локальные модели в настоящих проектах: решено заданий +Qwen 3.5 на llama.cpp · шесть проектов на Python, Rust и Go · одни и те же 72 задания · судят компилятор и тесты проектов обычные инструменты: читать, искать, править, запускать -та же модель с инструментами gitnext +пять инструментов gitnext: look, replace, add_after, rename, check -Найти, где функция - -4 из 6 - -6 из 6 -Кто вызывает функцию - -0 из 6 - -6 из 6 -Переименовать везде -проект должен собираться - -0 из 6 +Qwen 3.5 4B +шагов (медиана) 10 → 3 + +64% · 46 из 72 + +72% · 52 из 72 +Qwen 3.5 9B +шагов (медиана) 9 → 4 + +61% · 44 из 72 + +71% · 51 из 72 +Qwen 3.5 27B +шагов 10 → 3 · токенов ÷ 2,4 + +75% · 54 из 72 -5 из 6 -Починить упавший тест -подсаженная ошибка - -0 из 6 - -0 из 6 -24 задания: с обычными инструментами решено 4, с gitnext — 17 (15, если от модели не требовать сначала посмотреть код). -Починка ошибки пока не решена: здесь 0 из 6; в более позднем замере на click 3 из 12 — когда gitnext why узнал, что на деле запускает упавший тест. +83% · 60 из 72 +Все три модели ++9 пунктов, 95%-й интервал от +1 до +17 + +67% · 144 из 216 + +75% · 163 из 216 +Для одной модели разница в числе решённых заданий не выходит за погрешность; шаги и токены различаются в каждом виде заданий. +Починка подсаженной ошибки моделью 27B: 7 из 18 с обычными инструментами, 14 из 18 с gitnext. Малые модели чинят 3–6 из 18 в любом режиме. diff --git a/docs/images/agent.svg b/docs/images/agent.svg index ca61fbb..e38ff17 100644 --- a/docs/images/agent.svg +++ b/docs/images/agent.svg @@ -1,33 +1,35 @@ - -Tasks solved by a 4B local model: with plain tools 4 of 24, with gitnext tools 17 of 24 + +Tasks solved by three local models: 144 of 216 with plain tools, 163 of 216 with the five gitnext tools -A 4B local model on real projects: tasks solved -Qwen 3.5 4B on llama.cpp · click (Python) and semver (Rust) · 6 tasks of each kind · judged by the projects' own compilers and tests +Three local models on real projects: tasks solved +Qwen 3.5 on llama.cpp · six projects in Python, Rust and Go · the same 72 tasks each way · judged by the projects' compilers and tests plain tools: read, grep, edit, run -the same model with gitnext tools +the five gitnext tools: look, replace, add_after, rename, check -Find where a function is - -4 of 6 - -6 of 6 -Who calls this function - -0 of 6 - -6 of 6 -Rename it everywhere -the project must still build - -0 of 6 +Qwen 3.5 4B +median steps 10 → 3 + +64% · 46 of 72 + +72% · 52 of 72 +Qwen 3.5 9B +median steps 9 → 4 + +61% · 44 of 72 + +71% · 51 of 72 +Qwen 3.5 27B +median steps 10 → 3 · tokens ÷ 2.4 + +75% · 54 of 72 -5 of 6 -Fix a failing test -a seeded bug - -0 of 6 - -0 of 6 -24 tasks: 4 solved with plain tools, 17 with gitnext (15 when the model is not asked to look before it answers). -Fixing a bug is the open one: 0 of 6 here; 3 of 12 on click in a later run, once gitnext why knew what the failing test really runs. +83% · 60 of 72 +All three models ++9 points, 95% interval +1 to +17 + +67% · 144 of 216 + +75% · 163 of 216 +For one model alone the difference in solved tasks is inside the noise; the steps and tokens differ in every kind of task. +Fixing a seeded bug with the 27B model: 7 of 18 with plain tools, 14 of 18 with gitnext. The small models fix 3–6 of 18 either way. diff --git a/docs/images/make_charts.py b/docs/images/make_charts.py index 093c76a..44e6b43 100644 --- a/docs/images/make_charts.py +++ b/docs/images/make_charts.py @@ -67,35 +67,35 @@ def pct(n: int, d: int, word: str = "of") -> tuple[float, str]: return n / d, f"{round(100 * n / d)}% · {n} {word} {d}" -AGENT = [(4, 6), (0, 6), (0, 5), (0, 0)] # find, who calls, rename, fix: plain tools, with gitnext -WHY = [(1, 18, 20), (6, 14, 17), (3, 14, 20), (10, 46, 57)] # python, go, rust, all: graph, map, bugs +AGENT = [(46, 52, 72), (44, 51, 72), (54, 60, 72), (144, 163, 216)] # 4B, 9B, 27B, all: plain tools, the five gitnext tools, tasks +WHY = [(1, 18, 20), (6, 14, 17), (3, 17, 20), (10, 49, 57)] # python, go, rust, all: graph, map, bugs -chart(HERE / "agent.svg", "A 4B local model on real projects: tasks solved", - "Qwen 3.5 4B on llama.cpp · click (Python) and semver (Rust) · 6 tasks of each kind · judged by the projects' own compilers and tests", - ("plain tools: read, grep, edit, run", "the same model with gitnext tools"), - [(n, s, of6(a), of6(b)) for (n, s), (a, b) in zip([("Find where a function is", ""), ("Who calls this function", ""), ("Rename it everywhere", "the project must still build"), ("Fix a failing test", "a seeded bug")], AGENT)], - ["24 tasks: 4 solved with plain tools, 17 with gitnext (15 when the model is not asked to look before it answers).", - "Fixing a bug is the open one: 0 of 6 here; 3 of 12 on click in a later run, once gitnext why knew what the failing test really runs."], - "Tasks solved by a 4B local model: with plain tools 4 of 24, with gitnext tools 17 of 24") -chart(HERE / "agent.ru.svg", "Локальная модель на 4B в настоящих проектах: решено заданий", - "Qwen 3.5 4B на llama.cpp · click (Python) и semver (Rust) · по 6 заданий каждого вида · судят компилятор и тесты самих проектов", - ("обычные инструменты: читать, искать, править, запускать", "та же модель с инструментами gitnext"), - [(n, s, of6(a, "из"), of6(b, "из")) for (n, s), (a, b) in zip([("Найти, где функция", ""), ("Кто вызывает функцию", ""), ("Переименовать везде", "проект должен собираться"), ("Починить упавший тест", "подсаженная ошибка")], AGENT)], - ["24 задания: с обычными инструментами решено 4, с gitnext — 17 (15, если от модели не требовать сначала посмотреть код).", - "Починка ошибки пока не решена: здесь 0 из 6; в более позднем замере на click 3 из 12 — когда gitnext why узнал, что на деле запускает упавший тест."], - "Задания, решённые локальной моделью на 4B: с обычными инструментами 4 из 24, с gitnext 17 из 24") +chart(HERE / "agent.svg", "Three local models on real projects: tasks solved", + "Qwen 3.5 on llama.cpp · six projects in Python, Rust and Go · the same 72 tasks each way · judged by the projects' compilers and tests", + ("plain tools: read, grep, edit, run", "the five gitnext tools: look, replace, add_after, rename, check"), + [(n, s, pct(a, d), pct(b, d)) for (n, s), (a, b, d) in zip([("Qwen 3.5 4B", "median steps 10 → 3"), ("Qwen 3.5 9B", "median steps 9 → 4"), ("Qwen 3.5 27B", "median steps 10 → 3 · tokens ÷ 2.4"), ("All three models", "+9 points, 95% interval +1 to +17")], AGENT)], + ["For one model alone the difference in solved tasks is inside the noise; the steps and tokens differ in every kind of task.", + "Fixing a seeded bug with the 27B model: 7 of 18 with plain tools, 14 of 18 with gitnext. The small models fix 3–6 of 18 either way."], + "Tasks solved by three local models: 144 of 216 with plain tools, 163 of 216 with the five gitnext tools") +chart(HERE / "agent.ru.svg", "Три локальные модели в настоящих проектах: решено заданий", + "Qwen 3.5 на llama.cpp · шесть проектов на Python, Rust и Go · одни и те же 72 задания · судят компилятор и тесты проектов", + ("обычные инструменты: читать, искать, править, запускать", "пять инструментов gitnext: look, replace, add_after, rename, check"), + [(n, s, pct(a, d, "из"), pct(b, d, "из")) for (n, s), (a, b, d) in zip([("Qwen 3.5 4B", "шагов (медиана) 10 → 3"), ("Qwen 3.5 9B", "шагов (медиана) 9 → 4"), ("Qwen 3.5 27B", "шагов 10 → 3 · токенов ÷ 2,4"), ("Все три модели", "+9 пунктов, 95%-й интервал от +1 до +17")], AGENT)], + ["Для одной модели разница в числе решённых заданий не выходит за погрешность; шаги и токены различаются в каждом виде заданий.", + "Починка подсаженной ошибки моделью 27B: 7 из 18 с обычными инструментами, 14 из 18 с gitnext. Малые модели чинят 3–6 из 18 в любом режиме."], + "Задания, решённые тремя локальными моделями: 144 из 216 с обычными инструментами, 163 из 216 с пятью инструментами gitnext") chart(HERE / "why.svg", "A test fails: is the function at fault first in the list?", "57 seeded bugs in 6 projects: one operator changed inside a function, then gitnext why is asked", ("from the call graph alone", "after gitnext learn-tests (one test run with coverage)"), [(n, s, pct(a, d), pct(b, d)) for (n, s), (a, b, d) in zip([("Python", "click, sqlparse"), ("Go", "cobra, chi"), ("Rust", "semver, indexmap"), ("All six projects", "")], WHY)], ["With the map, why lists what the test really ran, puts the code changed since the tests last passed on top, and shows the changed lines.", "Reproduce: python3 eval/why_rank.py --learn --variants static,default — tables in eval/results/why-rank*.md"], - "Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 81% with the learned coverage map") + "Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 86% with the learned coverage map") chart(HERE / "why.ru.svg", "Тест упал: стоит ли виновная функция первой в списке?", "57 подсаженных ошибок в 6 проектах: в функции меняется один оператор, затем спрашиваем gitnext why <упавший тест>", ("только по графу вызовов", "после gitnext learn-tests (один прогон тестов с покрытием)"), [(n, s, pct(a, d, "из"), pct(b, d, "из")) for (n, s), (a, b, d) in zip([("Python", "click, sqlparse"), ("Go", "cobra, chi"), ("Rust", "semver, indexmap"), ("Все шесть проектов", "")], WHY)], ["С картой why показывает то, что тест на деле запускал, ставит первым код, изменённый с тех пор, как тесты проходили, и сами изменённые строки.", "Воспроизвести: python3 eval/why_rank.py --learn --variants static,default — таблицы в eval/results/why-rank*.md"], - "Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 81% с картой покрытия") + "Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 86% с картой покрытия") print("written:", ", ".join(p.name for p in sorted(HERE.glob("*.svg")) if p.name.startswith(("agent", "why")))) diff --git a/docs/images/why.ru.svg b/docs/images/why.ru.svg index 365ca76..3042a9d 100644 --- a/docs/images/why.ru.svg +++ b/docs/images/why.ru.svg @@ -1,5 +1,5 @@ - -Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 81% с картой покрытия + +Доля подсаженных ошибок, где gitnext why называет виновную функцию первой: 18% по графу вызовов, 86% с картой покрытия Тест упал: стоит ли виновная функция первой в списке? 57 подсаженных ошибок в 6 проектах: в функции меняется один оператор, затем спрашиваем gitnext why <упавший тест> @@ -22,13 +22,13 @@ semver, indexmap 15% · 3 из 20 - -70% · 14 из 20 + +85% · 17 из 20 Все шесть проектов 18% · 10 из 57 - -81% · 46 из 57 + +86% · 49 из 57 С картой why показывает то, что тест на деле запускал, ставит первым код, изменённый с тех пор, как тесты проходили, и сами изменённые строки. Воспроизвести: python3 eval/why_rank.py --learn --variants static,default — таблицы в eval/results/why-rank*.md diff --git a/docs/images/why.svg b/docs/images/why.svg index 180155f..39cf7bb 100644 --- a/docs/images/why.svg +++ b/docs/images/why.svg @@ -1,5 +1,5 @@ - -Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 81% with the learned coverage map + +Share of seeded bugs where gitnext why names the function at fault first: 18% from the call graph, 86% with the learned coverage map A test fails: is the function at fault first in the list? 57 seeded bugs in 6 projects: one operator changed inside a function, then gitnext why <failing test> is asked @@ -22,13 +22,13 @@ semver, indexmap 15% · 3 of 20 - -70% · 14 of 20 + +85% · 17 of 20 All six projects 18% · 10 of 57 - -81% · 46 of 57 + +86% · 49 of 57 With the map, why lists what the test really ran, puts the code changed since the tests last passed on top, and shows the changed lines. Reproduce: python3 eval/why_rank.py --learn --variants static,default — tables in eval/results/why-rank*.md diff --git a/eval/agent_stats.py b/eval/agent_stats.py new file mode 100644 index 0000000..69ecbd2 --- /dev/null +++ b/eval/agent_stats.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""The tables of an agent run over several models: solved, steps and tokens by model, kind and mode; +the same task in two modes as a pair (sign test, bootstrap over tasks). + + python3 eval/agent_stats.py eval/results/agent-kaggle-20261113.json + +The file is a list of the rows `eval/agent.py` writes, each with `model` and `set` (the task set, 0-based) added. +""" +import json, sys, statistics, collections, random, math +rows = json.load(open(sys.argv[1])) +MODELS = ["qwen3.5-4b", "qwen3.5-9b", "qwen3.5-27b"]; MODES = ["base", "gitnext", "lite"]; TASKS = ["find", "callers", "fix", "rename"] +by = collections.defaultdict(list) +for r in rows: + by[(r["model"], r["task"], r["mode"])].append(r); by[(r["model"], "all", r["mode"])].append(r) +def cell(rs): + ok = sum(bool(r.get("passed")) for r in rs); st = [r["steps"] for r in rs if r.get("steps")] + tok = [r["tokens_in"] for r in rs if r.get("tokens_in")]; sec = [r["seconds"] for r in rs if r.get("seconds")] + return ok, len(rs), statistics.median(st) if st else 0, sum(tok) / max(1, len(tok)) / 1000, sum(sec) / max(1, len(sec)) +print("errors (no steps):", collections.Counter((r["model"], r["mode"]) for r in rows if not r.get("steps"))) +for m in MODELS: + print(f"\n== {m}") + for t in TASKS + ["all"]: + print(f" {t:8}", " | ".join(f"{md}: {c[0]:2}/{c[1]} st {c[2]:>4.0f} tok {c[3]:>5.1f}k {c[4]:>4.0f}s" for md in MODES for c in [cell(by[(m, t, md)])])) +# paired: the same task (repo, set, task) in two modes +def paired(m, a, b, task=None): + key = lambda r: (r["repo"], r["set"], r["task"]) + A = {key(r): r for r in rows if r["model"] == m and r["mode"] == a and (task is None or r["task"] == task)} + B = {key(r): r for r in rows if r["model"] == m and r["mode"] == b and (task is None or r["task"] == task)} + ks = sorted(set(A) & set(B)) + win = sum(bool(B[k]["passed"]) and not A[k]["passed"] for k in ks); lose = sum(bool(A[k]["passed"]) and not B[k]["passed"] for k in ks) + n = win + lose + p = sum(math.comb(n, i) for i in range(0, min(win, lose) + 1)) * 2 / 2 ** n if n else 1.0 + rng = random.Random(1); d = [bool(B[k]["passed"]) - bool(A[k]["passed"]) for k in ks] + bs = sorted(sum(rng.choice(d) for _ in d) / len(d) for _ in range(4000)) + both = [k for k in ks if A[k].get("tokens_in") and B[k].get("tokens_in")] + ratio = statistics.median(A[k]["tokens_in"] / B[k]["tokens_in"] for k in both) if both else 0 + solved = [k for k in ks if A[k]["passed"] and B[k]["passed"] and A[k].get("tokens_in") and B[k].get("tokens_in")] + r2 = statistics.median(A[k]["tokens_in"] / B[k]["tokens_in"] for k in solved) if solved else 0 + return len(ks), win, lose, min(1.0, p), sum(d) / len(d), bs[100], bs[3899], ratio, r2, len(solved) +print("\npaired, the same 72 tasks: n, +wins, -losses, sign-test p, diff [95% CI], median token ratio (all), (both solved, n)") +for m in MODELS: + for a, b in (("base", "gitnext"), ("base", "lite"), ("gitnext", "lite")): + n, w, l, p, d, lo, hi, ra, r2, ns = paired(m, a, b) + print(f" {m:12} {a:>7} -> {b:7} n={n} +{w} -{l} p={p:.4f} diff {d*100:+.0f} pp [{lo*100:+.0f}, {hi*100:+.0f}] tokens x{ra:.1f} (both solved x{r2:.1f}, n={ns})") +print("\nby repo, solved of 12 per mode:") +for m in MODELS: + print(" ", m, " ".join(f"{repo}:" + "/".join(str(sum(bool(r['passed']) for r in rows if r['model'] == m and r['repo'] == repo and r['mode'] == md)) for md in MODES) for repo in ["click", "sqlparse", "semver", "indexmap", "cobra", "chi"])) +print("\nguard stops:", {m: collections.Counter((r["mode"], (r.get("guard") or {}).get("stopped")) for r in rows if r["model"] == m and (r.get("guard") or {}).get("stopped")) for m in MODELS}) diff --git a/eval/results/COVERAGE.md b/eval/results/COVERAGE.md index 86bdbce..b88364a 100644 --- a/eval/results/COVERAGE.md +++ b/eval/results/COVERAGE.md @@ -17,7 +17,7 @@ that the answer is right. | `why`, `check ` | place of the function at fault for a seeded bug (`why_rank.py`, `why-rank.md`) | with the learned map the culprit is first in 81% of 57 seeded bugs (Python 90%, Go 82%, Rust 70%); from the call graph alone in 18% | | the tools as a small model uses them | `agent.py`, Qwen 3.5 4B: find / callers / rename / fix | see the agent runs in `HISTORY.md` | -## 2. Unit tests (`cargo test --workspace`, 138 tests) +## 2. Unit tests (`cargo test --workspace`, 139 tests) | function | tests | |---|---| diff --git a/eval/results/HISTORY.md b/eval/results/HISTORY.md index 69f17e5..d194c28 100644 --- a/eval/results/HISTORY.md +++ b/eval/results/HISTORY.md @@ -56,6 +56,55 @@ solved tasks are too few to compare; the steps and tokens differ in every single limit on such calls. `look` takes these now. In the three lost `gitnext` runs out of four the model named the bug and stopped without an edit. One run ended on a `replace` cut by the token limit — the harness asks again now. +### The full run on a rented GPU: three models, 216 tasks each way + +Kaggle, two T4 cards, llama.cpp with CUDA, the settings of the Mac runs; gitnext 0.3.0 plus the `look` change above. +Qwen 3.5 4B, 9B and 27B (Q4_K_M); six projects (click, sqlparse, semver, indexmap, cobra, chi); find, who calls, fix, +rename; three sets of tasks per project (seeds 11, 12, 13) — 72 tasks, the same for every model and mode, 648 runs. +Every row: `eval/results/agent-kaggle-20261113.json`; the tables below: `python3 eval/agent_stats.py` on it. + +Solved, of 72 (median steps · tokens sent per task · seconds per task): + +| model | plain | gitnext (11 tools) | lite (5 tools) | +|---|---|---|---| +| Qwen 3.5 4B | 46 (10 · 54.7 k · 37 s) | 47 (5 · 44.7 k · 30 s) | 52 (3 · 41.1 k · 28 s) | +| Qwen 3.5 9B | 44 (9 · 53.9 k · 51 s) | 50 (5 · 38.0 k · 44 s) | 51 (4 · 36.6 k · 50 s) | +| Qwen 3.5 27B | 54 (10 · 49.6 k · 123 s) | 59 (4 · 18.8 k · 55 s) | 60 (3 · 20.9 k · 65 s) | +| all three, of 216 | 144 | 156 | 163 | + +By kind, of 18 (plain → gitnext → lite): + +| model | find | who calls | fix | rename | +|---|---|---|---|---| +| 4B | 17 → 18 → 18 | 12 → 15 → 16 | 3 → 2 → 6 | 14 → 12 → 12 | +| 9B | 18 → 18 → 18 | 10 → 14 → 15 | 4 → 5 → 6 | 12 → 13 → 12 | +| 27B | 18 → 18 → 18 | 15 → 14 → 16 | 7 → 14 → 14 | 14 → 13 → 12 | + +What is and is not shown (the same task in two modes is a pair; sign test on the pairs that differ; the interval is a +bootstrap over the 72 tasks): +- **One model, all kinds: inside the noise.** Plain → lite: 4B +12 −6 (p = 0.24), 9B +10 −3 (p = 0.09), 27B +10 −4 + (p = 0.18). Plain → gitnext: +7 −6, +10 −4, +8 −3. +- **Three models together:** plain → lite 144 → 163 of 216, +32 −13, +8.8 points, 95% interval +0.9 to +16.7. + Plain → gitnext 144 → 156, +25 −13, +5.6 points, interval −1.4 to +13.0. +- **Fixing a bug with the 27B model:** 7 → 14 of 18 in both gitnext modes, +7 −0 (p = 0.016). `check ` was its + first call in all 36 runs. The small models fix 2–6 of 18 whatever the tools. +- **Steps:** a median of 9–10 plain, 4–5 with gitnext, 3–4 with the five tools — in every model. +- **The earlier Mac result did not repeat.** Runs 1–3 above gave the 4B model 3–4 of 24 plain and 11–15 with gitnext + on click and semver. Here, on the same two projects, plain is 19 of 24 and gitnext 18. Since run 3 the harness + itself changed (the guard's hints act in every mode: 29 of 72 plain runs got one) and llama.cpp is another build; + which of the two moved the plain numbers was not measured. The numbers of runs 1–3 should not be quoted any more. + +What the conversations showed: +- With eleven tools the small models do not start a fix from `check `: the 9B model never in 18 tasks, the 4B + in 5 (they call `look`). With five tools: 16 of 18 each. The tool description now starts with the failing test. +- `rename` with the tool lost to a rename by hand on the judge (old name nowhere outside `*.md`): the same five tasks + in every model. Two are a name in the `.rst` documentation (sqlparse) — `rename --text` covers documentation now; + the others are names that are also common words (`Name`, `remaining`) or a standard-library function (`FileServer`), + where the judge's `git grep` finds other things. +- `look` after the change, the 553 calls of the lite fix runs: 18 named something that is not there (a path outside + the repo, a shell command, a library function), 5 a name that is not in the file given, 62 repeated an earlier call. +- Rust `why` with the map after macros became symbols: the function at fault first in 17 of 20 (it was 14). + ## Run 8 · in TypeScript the test is the file; coverage maps for Go and Rust **tests-for, TypeScript.** `it('…', () => …)` is not a named function; its calls are booked to the module of the From aa2f7a313bb3900950a9048a4fd59d27a59e417f Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 13 Nov 2026 09:53:58 +0300 Subject: [PATCH 18/18] 0.3.1 Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 2 +- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- crates/gitnext-cli/Cargo.toml | 12 ++++++------ crates/gitnext-docs/Cargo.toml | 4 ++-- crates/gitnext-index/Cargo.toml | 2 +- crates/gitnext-ops/Cargo.toml | 8 ++++---- crates/gitnext-tui/Cargo.toml | 8 ++++---- packaging/README.md | 6 +++--- packaging/npm/package.json | 2 +- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c54b240..938cf25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.3.1 — 2026-11-13 - **`gitnext look` takes a place in a file**, written the way tracebacks, test runners and models write it: `src/eval.rs:62` (the function around the line), `tests/test_x.py:120-160` (those lines), diff --git a/Cargo.lock b/Cargo.lock index 10dee48..2445aea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,7 +617,7 @@ dependencies = [ [[package]] name = "gitnext-cli" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "clap", @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "gitnext-core" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "git2", @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "gitnext-docs" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "gitnext-index", @@ -652,7 +652,7 @@ dependencies = [ [[package]] name = "gitnext-index" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "gitnext-parse", @@ -665,7 +665,7 @@ dependencies = [ [[package]] name = "gitnext-ops" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "git2", @@ -679,7 +679,7 @@ dependencies = [ [[package]] name = "gitnext-parse" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "serde", @@ -695,7 +695,7 @@ dependencies = [ [[package]] name = "gitnext-tui" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "gitnext-core", diff --git a/Cargo.toml b/Cargo.toml index 63f0877..23f0b15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = ["crates/*"] exclude = ["eval/.work"] [workspace.package] -version = "0.3.0" +version = "0.3.1" edition = "2021" license = "MIT" diff --git a/crates/gitnext-cli/Cargo.toml b/crates/gitnext-cli/Cargo.toml index cbc62a7..e98b5bf 100644 --- a/crates/gitnext-cli/Cargo.toml +++ b/crates/gitnext-cli/Cargo.toml @@ -7,12 +7,12 @@ license.workspace = true [dependencies] anyhow = "1.0.104" clap = { version = "4.6.7", features = ["derive"] } -gitnext-core = { version = "0.3.0", path = "../gitnext-core" } -gitnext-docs = { version = "0.3.0", path = "../gitnext-docs" } -gitnext-index = { version = "0.3.0", path = "../gitnext-index" } -gitnext-ops = { version = "0.3.0", path = "../gitnext-ops" } -gitnext-parse = { version = "0.3.0", path = "../gitnext-parse" } -gitnext-tui = { version = "0.3.0", path = "../gitnext-tui" } +gitnext-core = { version = "0.3.1", path = "../gitnext-core" } +gitnext-docs = { version = "0.3.1", path = "../gitnext-docs" } +gitnext-index = { version = "0.3.1", path = "../gitnext-index" } +gitnext-ops = { version = "0.3.1", path = "../gitnext-ops" } +gitnext-parse = { version = "0.3.1", path = "../gitnext-parse" } +gitnext-tui = { version = "0.3.1", path = "../gitnext-tui" } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.154" diff --git a/crates/gitnext-docs/Cargo.toml b/crates/gitnext-docs/Cargo.toml index 845c607..c8c9d15 100644 --- a/crates/gitnext-docs/Cargo.toml +++ b/crates/gitnext-docs/Cargo.toml @@ -6,5 +6,5 @@ license.workspace = true [dependencies] anyhow = "1.0.104" -gitnext-index = { version = "0.3.0", path = "../gitnext-index" } -gitnext-parse = { version = "0.3.0", path = "../gitnext-parse" } +gitnext-index = { version = "0.3.1", path = "../gitnext-index" } +gitnext-parse = { version = "0.3.1", path = "../gitnext-parse" } diff --git a/crates/gitnext-index/Cargo.toml b/crates/gitnext-index/Cargo.toml index 29b1d9e..8b22b0c 100644 --- a/crates/gitnext-index/Cargo.toml +++ b/crates/gitnext-index/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true [dependencies] anyhow = "1.0.104" -gitnext-parse = { version = "0.3.0", path = "../gitnext-parse" } +gitnext-parse = { version = "0.3.1", path = "../gitnext-parse" } ignore = "0.4.34" rayon = "1.12.0" rusqlite = { version = "0.41.1", features = ["bundled"] } diff --git a/crates/gitnext-ops/Cargo.toml b/crates/gitnext-ops/Cargo.toml index c8ee6d9..f0ce958 100644 --- a/crates/gitnext-ops/Cargo.toml +++ b/crates/gitnext-ops/Cargo.toml @@ -7,9 +7,9 @@ license.workspace = true [dependencies] anyhow = "1.0.104" git2 = { version = "0.21.0", default-features = false } -gitnext-core = { version = "0.3.0", path = "../gitnext-core" } -gitnext-docs = { version = "0.3.0", path = "../gitnext-docs" } -gitnext-index = { version = "0.3.0", path = "../gitnext-index" } -gitnext-parse = { version = "0.3.0", path = "../gitnext-parse" } +gitnext-core = { version = "0.3.1", path = "../gitnext-core" } +gitnext-docs = { version = "0.3.1", path = "../gitnext-docs" } +gitnext-index = { version = "0.3.1", path = "../gitnext-index" } +gitnext-parse = { version = "0.3.1", path = "../gitnext-parse" } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.154" diff --git a/crates/gitnext-tui/Cargo.toml b/crates/gitnext-tui/Cargo.toml index 775fa69..8f706ff 100644 --- a/crates/gitnext-tui/Cargo.toml +++ b/crates/gitnext-tui/Cargo.toml @@ -6,9 +6,9 @@ license.workspace = true [dependencies] anyhow = "1.0.104" -gitnext-core = { version = "0.3.0", path = "../gitnext-core" } -gitnext-index = { version = "0.3.0", path = "../gitnext-index" } -gitnext-ops = { version = "0.3.0", path = "../gitnext-ops" } -gitnext-parse = { version = "0.3.0", path = "../gitnext-parse" } +gitnext-core = { version = "0.3.1", path = "../gitnext-core" } +gitnext-index = { version = "0.3.1", path = "../gitnext-index" } +gitnext-ops = { version = "0.3.1", path = "../gitnext-ops" } +gitnext-parse = { version = "0.3.1", path = "../gitnext-parse" } ratatui = "0.30.2" unicode-width = "0.2.3" diff --git a/packaging/README.md b/packaging/README.md index 07462a1..f5f0445 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -19,9 +19,9 @@ For each release (after the release workflow has finished and the assets are up) ```sh cd ~/Developer/gitnext -sh packaging/homebrew/update.sh v0.3.0 > ~/Developer/homebrew-tap/Formula/gitnext.rb +sh packaging/homebrew/update.sh v0.3.1 > ~/Developer/homebrew-tap/Formula/gitnext.rb cd ~/Developer/homebrew-tap -git add Formula/gitnext.rb && git commit -m "gitnext 0.3.0" && git push +git add Formula/gitnext.rb && git commit -m "gitnext 0.3.1" && git push brew install pnnnx/tap/gitnext && gitnext --version # check ``` @@ -38,7 +38,7 @@ For each release: ```sh cd ~/Developer/gitnext/packaging/npm -npm version 0.3.0 --no-git-tag-version # must match the release tag (v0.3.0) +npm version 0.3.0 --no-git-tag-version # must match the release tag (v0.3.1) npm login # once npm publish npx -y gitnext --version # check diff --git a/packaging/npm/package.json b/packaging/npm/package.json index 7dd5e99..236b8db 100644 --- a/packaging/npm/package.json +++ b/packaging/npm/package.json @@ -1,6 +1,6 @@ { "name": "gitnext", - "version": "0.3.0", + "version": "0.3.1", "description": "git that understands code: code map, symbol graph and plain-language git for AI agents and people", "license": "MIT", "homepage": "https://github.com/pnnnx/gitnext",