diff --git a/interactive/Cargo.toml b/interactive/Cargo.toml index adb8ea20a..c598919ff 100644 --- a/interactive/Cargo.toml +++ b/interactive/Cargo.toml @@ -16,7 +16,6 @@ columnar = { workspace = true } # The columnar kernels for the interpreted backend, pinned by git rev. corgi = { git = "https://github.com/frankmcsherry/WIP", rev = "de0f2ac91d31ac63641035e5a5bb1b5640fe9424", features = ["serde"] } differential-dataflow = { workspace = true } -mimalloc = "0.1.48" serde = { version = "1.0", features = ["derive"] } smallvec = "1.15.1" timely = { workspace = true } diff --git a/interactive/README.md b/interactive/README.md index c72c07436..b29d57025 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -73,9 +73,18 @@ The flow moves through four steps: 4. The `examples/` directory contains back-ends that execute programs. The `examples/programs/` directory contains example programs, intentionally simple at the moment. -You can run any of them with one of the example harnesses, for example +The one executable is the server (the `ddir-server` crate in `server/`, documented in +`server/README.md`; the session scripts in `examples/server/` show it at work); a program runs by +loading it, feeding its inputs, and closing epochs. For example, on a random graph of 100 nodes and +200 edges, 10 of which change each epoch, for 100 epochs, on four workers of the Corgi backend: ``` -cargo run --release --example ddir_vec -- ./examples/programs/reach.ddp 2 100 200 1 100 +cd interactive +printf 'load reach from examples/programs/reach.ddp +feed reach 0 from random:nodes=100,edges=200,churn=10 +feed reach 1 0 +tick 100 +exit +' | DDIR_BACKEND=corgi DDIR_WORKERS=4 cargo run --release -p ddir-server ``` More generally, you can run diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md index 652f26c91..26d9d32ab 100644 --- a/interactive/examples/aoc2023/README.md +++ b/interactive/examples/aoc2023/README.md @@ -17,17 +17,17 @@ mixed-arity inputs. ## Run - cargo build --release --example ddir - ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch + cargo build --release -p ddir-server + ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch + ./run.sh corgi # the same through the Corgi columnar backend `run.sh` first runs `transcribe.py` (python3) to regenerate `gen/`, then -runs each part as `EDGES_FILE=gen/ ddir --backend=vec -dayNN/partN.ddp 10 0 1 0`; the answer is the `Int` on the -`[partN]` inspect line. +runs each part as one server session piped into `ddir_server` — `load` the +program from its file, `feed … from` the fact file into input 0, `tick` — and +reads the answer off the `[partN]` inspect line. -`run.sh` asserts the **vec** backend only; corgi currently disagrees or -crashes on 4 of these parts (known open issues, minimized separately) and -is not asserted yet. +Both backends pass all 33 parts. Corgi needs day05's arity-padded inputs +(`run.sh corgi` transcribes with `--pad`). ## Verdicts diff --git a/interactive/examples/aoc2023/run.sh b/interactive/examples/aoc2023/run.sh index ade1193b1..bf253f55c 100755 --- a/interactive/examples/aoc2023/run.sh +++ b/interactive/examples/aoc2023/run.sh @@ -1,17 +1,23 @@ #!/bin/sh -# Run every AoC 2023 program on the vec backend and check the answers. -# Usage: ./run.sh [path/to/ddir] (build with: cargo build --release --example ddir) +# Run every AoC 2023 program through the DDIR server and check the answers. +# Usage: ./run.sh [vec|corgi] [path/to/ddir_server] +# (build with: cargo build --release -p ddir-server) cd "$(dirname "$0")" || exit 1 -DDIR=${1:-../../../target/release/examples/ddir} -python3 transcribe.py || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files +BACKEND=${1:-vec} +SERVER=${2:-../../../target/release/ddir_server} +PAD=; [ "$BACKEND" = corgi ] && PAD=--pad # corgi needs day05's uniform-arity copies +python3 transcribe.py $PAD || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files fail=0 while read -r day part expected; do case "$day" in ''|'#'*) continue;; esac dir=day$day inp=gen/$dir/input.txt [ -f "gen/$dir/input$part.txt" ] && inp=gen/$dir/input$part.txt # day05/day15: per-part inputs - arity=$(head -1 "$inp" | awk '{print NF}') - got=$(EDGES_FILE=$inp "$DDIR" --backend=vec "$dir/part$part.ddp" "$arity" 10 0 1 0 2>&1 \ + [ -n "$PAD" ] && [ -f "gen/$dir/input${part}p.txt" ] && inp=gen/$dir/input${part}p.txt + # One session per part: load the program, feed its input from the fact + # file, close the epoch. The answer is the `Int` on the `[partN]` inspect line. + got=$(printf 'load p from %s\nfeed p 0 from %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ + | DDIR_BACKEND="$BACKEND" "$SERVER" 2>&1 \ | sed -n "s/.*\\[part$part\\].*Int(\\(-\\{0,1\\}[0-9]*\\)).*/\\1/p") if [ "$got" = "$expected" ]; then echo "day$day part$part: ok ($got)" diff --git a/interactive/examples/ddir.rs b/interactive/examples/ddir.rs deleted file mode 100644 index d262f1d00..000000000 --- a/interactive/examples/ddir.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! The DDIR driver: parse, lower, render on the chosen backend, execute. -//! -//! Usage: `ddir [flags] [batch] [rounds] [timely args]` -//! -//! Inputs: with `EDGES_FILE` set, rows come from that file — one row per line, -//! whitespace-separated `i64` fields, assigned round-robin to the program's -//! inputs (`line % n_inputs`). Otherwise rows are synthesized by `gen_row`. -//! -//! Flags: -//! - `--explain`: apply the explanation rewrite after lowering; the last input -//! becomes the query input (the demand envelope `(key ; val ++ q)`). -//! - `--query=K:V[,q]`: seed the query input with one row (requires --explain). -//! - `--debug-demand`: tap every demand collection with an Inspect. -//! - `--diag`: serve timely/DD diagnostics on port 51371. -//! - `--backend=vec|corgi`: rendering substrate (default `vec`). The corgi -//! backend exchanges by key hash, so both take `-w`. -//! - `--sync=K`: await completion only every K rounds (default 1), letting K -//! timestamps retire with whatever inter-timestamp concurrency the system -//! finds — the open(er)-loop regime DD adapts into under load. - -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -use differential_dataflow::dynamic::pointstamp::PointStamp; -use differential_dataflow::input::Input; - -use interactive::parse; -use interactive::lower; -use interactive::scope_ir as st; -use interactive::ir::{Diff, Value}; -use interactive::backend::vec::{render_tree, Row}; -use interactive::backend::corgi::render_tree_rows; - -#[derive(Clone, Default)] -struct Flags { - explain: bool, - query: Option, - debug_demand: bool, - diag: bool, - corgi: bool, - sync: u64, -} - -fn run( - name: &str, - stmts: Vec, - n_inputs: usize, - nodes: u64, - edges: u64, - arity: usize, - batch: u64, - rounds: Option, - flags: Flags, - timely_args: Vec, -) { - let explain = flags.explain; - let mut tree = lower::lower_tree(stmts); - // --explain: rewrite for self-explanation before optimization (the rules - // assume single-op Linears). Sources are the root's positional inputs; the - // query arrives as one extra input appended after them. - if explain { - let source_shapes: Vec<(usize, usize)> = tree.root.imports.iter().map(|imp| match &imp.from { - st::Source::Input(_) => (arity, 0usize), - other => panic!("ddir --explain: unsupported source {:?}", other), - }).collect(); - let shape = interactive::explain::export_shape(&tree, &source_shapes); - eprintln!("explain: first export shape (k={}, v={}); query is (key[{}] ; val[{}] ++ q)", shape.0, shape.1, shape.0, shape.1); - let options = interactive::explain::Options { debug_inspects: flags.debug_demand }; - tree = interactive::explain::explain_with(&tree, &source_shapes, options); - } - let ops_before = tree.op_count(); - tree.optimize(); - let tree_export_idx = tree.root.exports.iter().position(|e| e.name == "result").unwrap_or(0); - println!("{}: {} ops before optimize, {} after; driving export {:?}", - name, ops_before, tree.op_count(), tree.root.exports[tree_export_idx].name); - let name = name.to_string(); - let edges_file = std::env::var("EDGES_FILE").ok(); - let total_inputs = if explain { n_inputs + 1 } else { n_inputs }; - let query_input_idx = if explain { Some(n_inputs) } else { None }; - - timely::execute_from_args(timely_args.into_iter(), move |worker| { - - // --diag registers timely/DD logging and serves the diagnostics - // WebSocket on worker 0 (port 51371) — see diagnostics/README.md. - let _diag = if flags.diag { - let state = diagnostics::logging::register(worker, false); - if worker.index() == 0 { - Some(diagnostics::server::Server::start(51371, state.sink)) - } else { - drop(state.sink); - None - } - } else { None }; - - let (mut inputs, probe) = worker.dataflow::(|scope| { - let mut handles = Vec::new(); - let mut collections = Vec::new(); - for _ in 0..total_inputs { - let (h, c) = scope.new_collection::<(Row, Row), Diff>(); - handles.push(h); collections.push(c); - } - let mut probe = timely::dataflow::ProbeHandle::new(); - let output = scope.iterative::, _, _>(|inner| { - let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect(); - let root_imports: Vec<_> = tree.root.imports.iter().map(|imp| match &imp.from { - st::Source::Input(n) => entered[*n].clone(), - st::Source::Trace(name) => panic!("ddir: Import {:?} not supported in this harness (no trace registry).", name), - st::Source::Parent(_) => unreachable!("root scope cannot import from a parent"), - }).collect(); - let exports = if flags.corgi { - render_tree_rows(&tree.root, inner, 0, root_imports) - } else { - render_tree(&tree.root, inner, 0, root_imports) - }; - exports[tree_export_idx].clone().leave(scope) - }); - output.probe_with(&mut probe); - (handles, probe) - }); - - let index = worker.index(); - let peers = worker.peers(); - - let timer = std::time::Instant::now(); - let timer_load = std::time::Instant::now(); - // With `EDGES_FILE` set, rows come from the file (one row per line, - // whitespace-separated i64 fields), assigned round-robin to inputs. - // Otherwise `gen_row` synthesizes `edges` rows. - if let Some(path) = &edges_file { - let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("Cannot read {}: {}", path, e)); - for (e, line) in text.lines().filter(|l| !l.trim().is_empty()).enumerate() { - if e % peers == index { - let input_idx = e % n_inputs; - let fields: Vec = line.split_whitespace().map(|t| Value::Int(t.parse::().unwrap())).collect(); - inputs[input_idx].update((Value::Tuple(fields), Value::unit()), 1); - } - } - } else { - for e in 0..edges { - if (e as usize) % peers == index { - let input_idx = (e as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(e, nodes, arity), 1); - } - } - } - // Seed the query input (worker 0 only) from --query="k:v[,q]". The row - // is the flat demand envelope `(key[k] ; val[v] ++ [q])` — at depth 0 - // there is no chain, so the value is just V's fields then the query id. - if let Some(q_idx) = query_input_idx { - if index == 0 { - if let Some(qstr) = flags.query.clone() { - let parse_fields = |s: &str| -> Vec { - if s.is_empty() { vec![] } else { s.split(',').map(|t| Value::Int(t.trim().parse::().unwrap())).collect() } - }; - let (k_str, vq_str) = qstr.split_once(':').unwrap_or((qstr.as_str(), "")); - let q_key = Value::Tuple(parse_fields(k_str)); - let mut vq = parse_fields(vq_str); - if vq.is_empty() { vq.push(Value::Int(0)); } // a bare query id - let q_val = Value::Tuple(vq); - eprintln!("seeding query: key={:?} val_with_q={:?}", q_key, q_val); - inputs[q_idx].update((q_key, q_val), 1); - } - } - } - for i in inputs.iter_mut() { i.advance_to(1); i.flush(); } - while probe.less_than(&1u64) { worker.step(); } - println!("worker {}: {} loaded ({} edges, total {:.2?}, load {:.2?})", index, name, edges, timer.elapsed(), timer_load.elapsed()); - - let mut cursor = 0u64; - let mut round = 0u64; - let limit = rounds.unwrap_or(u64::MAX); - while round < limit { - let timer_round = std::time::Instant::now(); - let time = (round + 2) as u64; - for _ in 0..batch { - let remove_idx = cursor; - let add_idx = edges + cursor; - if (remove_idx as usize) % peers == index { - let input_idx = (remove_idx as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(remove_idx, nodes, arity), -1); - } - if (add_idx as usize) % peers == index { - let input_idx = (add_idx as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(add_idx, nodes, arity), 1); - } - cursor += 1; - } - for i in inputs.iter_mut() { i.advance_to(time); i.flush(); } - let sync = flags.sync.max(1); - if (round + 1) % sync == 0 || round + 1 == limit { - while probe.less_than(&time) { worker.step(); } - } - - round += 1; - if round % 100 == 0 { - println!("worker {}: {} round {} (total {:.2?}, round {:.2?})", index, name, round, timer.elapsed(), timer_round.elapsed()); - } - } - println!("worker {}: {} done ({} rounds, batch {}, total {:.2?})", index, name, round, batch, timer.elapsed()); - }).unwrap(); -} - -fn main() { - // Strip our flags; everything else stays positional (and the tail is - // forwarded to timely). - let raw_args: Vec = std::env::args().collect(); - let (flags, args): (Flags, Vec) = { - let mut it = raw_args.into_iter(); - let prog = it.next().unwrap(); - let mut flags = Flags::default(); - let mut rest: Vec = Vec::new(); - for a in it { - if a == "--explain" { flags.explain = true; } - else if let Some(q) = a.strip_prefix("--query=") { flags.query = Some(q.to_string()); } - else if a == "--debug-demand" { flags.debug_demand = true; } - else if a == "--diag" { flags.diag = true; } - else if let Some(k) = a.strip_prefix("--sync=") { flags.sync = k.parse().expect("--sync=K"); } - else if let Some(b) = a.strip_prefix("--backend=") { - flags.corgi = match b { "corgi" => true, "vec" => false, other => panic!("unknown backend {other:?} (vec|corgi)") }; - } - else { rest.push(a); } - } - let mut out = vec![prog]; out.extend(rest); - (flags, out) - }; - let program = args.get(1).cloned().unwrap_or_else(|| { std::process::exit(0); }); - let arity: usize = args.get(2).cloned().unwrap_or("2".into()).parse().unwrap(); - let nodes: u64 = args.get(3).cloned().unwrap_or("10".into()).parse().unwrap(); - let edges: u64 = args.get(4).cloned().unwrap_or_else(|| (2 * nodes).to_string()).parse().unwrap(); - let batch: u64 = args.get(5).cloned().unwrap_or("1".into()).parse().unwrap(); - let rounds: Option = args.get(6).map(|s| s.parse().unwrap()); - - let source = interactive::load_program(&program); - let stmts = if program.ends_with(".ddp") { - parse::pipe::parse(&source) - } else { - parse::applicative::parse(&source) - }; - let (n_inputs, imports) = interactive::survey_sources(&stmts); - if !imports.is_empty() { - panic!("ddir: program references imports {:?} but this harness has no trace registry.", imports); - } - let name = std::path::Path::new(&program).file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or(program.clone()); - let timely_args: Vec = args.iter().skip(4).cloned().collect(); - run(&name, stmts, n_inputs, nodes, edges, arity, batch, rounds, flags, timely_args); -} diff --git a/interactive/examples/ddir_server.rs b/interactive/examples/ddir_server.rs deleted file mode 100644 index 67bf91d89..000000000 --- a/interactive/examples/ddir_server.rs +++ /dev/null @@ -1,323 +0,0 @@ -//! A live DDIR server: install interpreted dataflows into one running timely -//! computation and let them share results by name. Multi-worker capable. -//! -//! Usage (see `examples/server/` for session scripts and a README): -//! cargo run --release --example ddir_server # stdin, 1 worker -//! cargo run --release --example ddir_server --