From 5dd8cea115e46bff29820ba9cf584ca0e6869903 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 4 Sep 2026 09:22:16 -0400 Subject: [PATCH 1/5] DDIR: run the examples and tests through the server; retire the ddir harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-program harness (`examples/ddir.rs`) duplicated the server's job: build one dataflow, fill its inputs, close epochs. The server now covers what the harness did, and the harness is gone. - `load ` bulk-loads a positional input, sharded across the workers, from a `random:`/`iota:` recipe or a file of integer rows. A `churn=C` recipe keeps changing the input on every `tick` — the harness's batch/rounds loop — through the same per-input generator that drives generated sources (now one map per program, not one slot). - `tick [n]` closes several epochs and reports the wall-clock time. - `--backend=vec|corgi` selects the substrate for the whole server. - `install … explain=[,debug]` applies the explanation rewrite; the query is an ordinary `feed` of the extra input, the demand sets are `peek`s. - `server::evaluate` is the data-in/data-out entry point the test suites use: install, feed, tick, snapshot — the path a live install takes. It replaces the vec and corgi backends' private `evaluate` harnesses, so the corgi gate (at 1–4 workers and over serializing channels) and the explanation tests all run against the server on both backends. - The AoC suite drives the server: `run.sh [vec|corgi]`. vec 33/33; corgi 32/33 (day13 part 1 crashes corgi, as it did before). Also dropped: `survey_sources` (only the harness used it) and the `diagnostics` dev-dependency (only the harness's `--diag` used it). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QDsLC46QQW9aBrksaWad6T --- interactive/README.md | 12 +- interactive/examples/aoc2023/README.md | 17 +- interactive/examples/aoc2023/run.sh | 18 +- interactive/examples/ddir.rs | 248 ------------------------- interactive/examples/ddir_server.rs | 97 ++++++++-- interactive/examples/server/README.md | 45 ++++- interactive/src/backend/corgi.rs | 126 ------------- interactive/src/backend/vec.rs | 77 -------- interactive/src/lib.rs | 39 ---- interactive/src/server.rs | 145 ++++++++++++++- interactive/tests/corgi_backend.rs | 11 +- interactive/tests/explain.rs | 16 +- 12 files changed, 304 insertions(+), 547 deletions(-) delete mode 100644 interactive/examples/ddir.rs diff --git a/interactive/README.md b/interactive/README.md index c72c07436..2ae49e352 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -73,9 +73,17 @@ The flow moves through four steps: 4. The `examples/` directory contains back-ends that execute programs. The `examples/programs/` directory contains example programs, intentionally simple at the moment. -You can run any of them with one of the example harnesses, for example +The one executable is the server (`examples/ddir_server.rs`, documented in `examples/server/README.md`); +a program runs by installing it, loading its inputs, and closing epochs. For example, on a random +graph of 100 nodes and 200 edges, 10 of which change each epoch, for 100 epochs: ``` -cargo run --release --example ddir_vec -- ./examples/programs/reach.ddp 2 100 200 1 100 +cd interactive +printf 'install reach examples/programs/reach.ddp +load reach 0 random:nodes=100,edges=200,churn=10 +feed reach 1 0 +tick 100 +exit +' | cargo run --release --example ddir_server -- --backend=corgi -w4 ``` More generally, you can run diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md index 652f26c91..90f3c251d 100644 --- a/interactive/examples/aoc2023/README.md +++ b/interactive/examples/aoc2023/README.md @@ -17,17 +17,18 @@ mixed-arity inputs. ## Run - cargo build --release --example ddir - ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch + cargo build --release --example ddir_server + ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch + ./run.sh corgi # the same through the Corgi columnar backend `run.sh` first runs `transcribe.py` (python3) to regenerate `gen/`, then -runs each part as `EDGES_FILE=gen/ ddir --backend=vec -dayNN/partN.ddp 10 0 1 0`; the answer is the `Int` on the -`[partN]` inspect line. +runs each part as one server session — `install`, `load` the fact file into +input 0, `tick` — and reads the answer off the `[partN]` inspect line. -`run.sh` asserts the **vec** backend only; corgi currently disagrees or -crashes on 4 of these parts (known open issues, minimized separately) and -is not asserted yet. +The vec backend passes all 33 parts. Corgi passes 32: it needs day05's +arity-padded inputs (`run.sh corgi` transcribes with `--pad`), and day13 +part 1 still crashes it (a batch of one shape reaching an operator pinned at +another — a known open issue). ## Verdicts diff --git a/interactive/examples/aoc2023/run.sh b/interactive/examples/aoc2023/run.sh index ade1193b1..4871249ab 100755 --- a/interactive/examples/aoc2023/run.sh +++ b/interactive/examples/aoc2023/run.sh @@ -1,17 +1,23 @@ #!/bin/sh -# Run every AoC 2023 program on the vec backend and check the answers. -# Usage: ./run.sh [path/to/ddir] (build with: cargo build --release --example ddir) +# Run every AoC 2023 program through the DDIR server and check the answers. +# Usage: ./run.sh [vec|corgi] [path/to/ddir_server] +# (build with: cargo build --release --example ddir_server) cd "$(dirname "$0")" || exit 1 -DDIR=${1:-../../../target/release/examples/ddir} -python3 transcribe.py || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files +BACKEND=${1:-vec} +SERVER=${2:-../../../target/release/examples/ddir_server} +PAD=; [ "$BACKEND" = corgi ] && PAD=--pad # corgi needs day05's uniform-arity copies +python3 transcribe.py $PAD || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files fail=0 while read -r day part expected; do case "$day" in ''|'#'*) continue;; esac dir=day$day inp=gen/$dir/input.txt [ -f "gen/$dir/input$part.txt" ] && inp=gen/$dir/input$part.txt # day05/day15: per-part inputs - arity=$(head -1 "$inp" | awk '{print NF}') - got=$(EDGES_FILE=$inp "$DDIR" --backend=vec "$dir/part$part.ddp" "$arity" 10 0 1 0 2>&1 \ + [ -n "$PAD" ] && [ -f "gen/$dir/input${part}p.txt" ] && inp=gen/$dir/input${part}p.txt + # One session per part: install the program, bulk-load its input, close the + # epoch. The answer is the `Int` on the `[partN]` inspect line. + got=$(printf 'install p %s\nload p 0 %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ + | "$SERVER" --backend="$BACKEND" 2>&1 \ | sed -n "s/.*\\[part$part\\].*Int(\\(-\\{0,1\\}[0-9]*\\)).*/\\1/p") if [ "$got" = "$expected" ]; then echo "day$day part$part: ok ($got)" diff --git a/interactive/examples/ddir.rs b/interactive/examples/ddir.rs deleted file mode 100644 index d262f1d00..000000000 --- a/interactive/examples/ddir.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! The DDIR driver: parse, lower, render on the chosen backend, execute. -//! -//! Usage: `ddir [flags] [batch] [rounds] [timely args]` -//! -//! Inputs: with `EDGES_FILE` set, rows come from that file — one row per line, -//! whitespace-separated `i64` fields, assigned round-robin to the program's -//! inputs (`line % n_inputs`). Otherwise rows are synthesized by `gen_row`. -//! -//! Flags: -//! - `--explain`: apply the explanation rewrite after lowering; the last input -//! becomes the query input (the demand envelope `(key ; val ++ q)`). -//! - `--query=K:V[,q]`: seed the query input with one row (requires --explain). -//! - `--debug-demand`: tap every demand collection with an Inspect. -//! - `--diag`: serve timely/DD diagnostics on port 51371. -//! - `--backend=vec|corgi`: rendering substrate (default `vec`). The corgi -//! backend exchanges by key hash, so both take `-w`. -//! - `--sync=K`: await completion only every K rounds (default 1), letting K -//! timestamps retire with whatever inter-timestamp concurrency the system -//! finds — the open(er)-loop regime DD adapts into under load. - -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -use differential_dataflow::dynamic::pointstamp::PointStamp; -use differential_dataflow::input::Input; - -use interactive::parse; -use interactive::lower; -use interactive::scope_ir as st; -use interactive::ir::{Diff, Value}; -use interactive::backend::vec::{render_tree, Row}; -use interactive::backend::corgi::render_tree_rows; - -#[derive(Clone, Default)] -struct Flags { - explain: bool, - query: Option, - debug_demand: bool, - diag: bool, - corgi: bool, - sync: u64, -} - -fn run( - name: &str, - stmts: Vec, - n_inputs: usize, - nodes: u64, - edges: u64, - arity: usize, - batch: u64, - rounds: Option, - flags: Flags, - timely_args: Vec, -) { - let explain = flags.explain; - let mut tree = lower::lower_tree(stmts); - // --explain: rewrite for self-explanation before optimization (the rules - // assume single-op Linears). Sources are the root's positional inputs; the - // query arrives as one extra input appended after them. - if explain { - let source_shapes: Vec<(usize, usize)> = tree.root.imports.iter().map(|imp| match &imp.from { - st::Source::Input(_) => (arity, 0usize), - other => panic!("ddir --explain: unsupported source {:?}", other), - }).collect(); - let shape = interactive::explain::export_shape(&tree, &source_shapes); - eprintln!("explain: first export shape (k={}, v={}); query is (key[{}] ; val[{}] ++ q)", shape.0, shape.1, shape.0, shape.1); - let options = interactive::explain::Options { debug_inspects: flags.debug_demand }; - tree = interactive::explain::explain_with(&tree, &source_shapes, options); - } - let ops_before = tree.op_count(); - tree.optimize(); - let tree_export_idx = tree.root.exports.iter().position(|e| e.name == "result").unwrap_or(0); - println!("{}: {} ops before optimize, {} after; driving export {:?}", - name, ops_before, tree.op_count(), tree.root.exports[tree_export_idx].name); - let name = name.to_string(); - let edges_file = std::env::var("EDGES_FILE").ok(); - let total_inputs = if explain { n_inputs + 1 } else { n_inputs }; - let query_input_idx = if explain { Some(n_inputs) } else { None }; - - timely::execute_from_args(timely_args.into_iter(), move |worker| { - - // --diag registers timely/DD logging and serves the diagnostics - // WebSocket on worker 0 (port 51371) — see diagnostics/README.md. - let _diag = if flags.diag { - let state = diagnostics::logging::register(worker, false); - if worker.index() == 0 { - Some(diagnostics::server::Server::start(51371, state.sink)) - } else { - drop(state.sink); - None - } - } else { None }; - - let (mut inputs, probe) = worker.dataflow::(|scope| { - let mut handles = Vec::new(); - let mut collections = Vec::new(); - for _ in 0..total_inputs { - let (h, c) = scope.new_collection::<(Row, Row), Diff>(); - handles.push(h); collections.push(c); - } - let mut probe = timely::dataflow::ProbeHandle::new(); - let output = scope.iterative::, _, _>(|inner| { - let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect(); - let root_imports: Vec<_> = tree.root.imports.iter().map(|imp| match &imp.from { - st::Source::Input(n) => entered[*n].clone(), - st::Source::Trace(name) => panic!("ddir: Import {:?} not supported in this harness (no trace registry).", name), - st::Source::Parent(_) => unreachable!("root scope cannot import from a parent"), - }).collect(); - let exports = if flags.corgi { - render_tree_rows(&tree.root, inner, 0, root_imports) - } else { - render_tree(&tree.root, inner, 0, root_imports) - }; - exports[tree_export_idx].clone().leave(scope) - }); - output.probe_with(&mut probe); - (handles, probe) - }); - - let index = worker.index(); - let peers = worker.peers(); - - let timer = std::time::Instant::now(); - let timer_load = std::time::Instant::now(); - // With `EDGES_FILE` set, rows come from the file (one row per line, - // whitespace-separated i64 fields), assigned round-robin to inputs. - // Otherwise `gen_row` synthesizes `edges` rows. - if let Some(path) = &edges_file { - let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("Cannot read {}: {}", path, e)); - for (e, line) in text.lines().filter(|l| !l.trim().is_empty()).enumerate() { - if e % peers == index { - let input_idx = e % n_inputs; - let fields: Vec = line.split_whitespace().map(|t| Value::Int(t.parse::().unwrap())).collect(); - inputs[input_idx].update((Value::Tuple(fields), Value::unit()), 1); - } - } - } else { - for e in 0..edges { - if (e as usize) % peers == index { - let input_idx = (e as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(e, nodes, arity), 1); - } - } - } - // Seed the query input (worker 0 only) from --query="k:v[,q]". The row - // is the flat demand envelope `(key[k] ; val[v] ++ [q])` — at depth 0 - // there is no chain, so the value is just V's fields then the query id. - if let Some(q_idx) = query_input_idx { - if index == 0 { - if let Some(qstr) = flags.query.clone() { - let parse_fields = |s: &str| -> Vec { - if s.is_empty() { vec![] } else { s.split(',').map(|t| Value::Int(t.trim().parse::().unwrap())).collect() } - }; - let (k_str, vq_str) = qstr.split_once(':').unwrap_or((qstr.as_str(), "")); - let q_key = Value::Tuple(parse_fields(k_str)); - let mut vq = parse_fields(vq_str); - if vq.is_empty() { vq.push(Value::Int(0)); } // a bare query id - let q_val = Value::Tuple(vq); - eprintln!("seeding query: key={:?} val_with_q={:?}", q_key, q_val); - inputs[q_idx].update((q_key, q_val), 1); - } - } - } - for i in inputs.iter_mut() { i.advance_to(1); i.flush(); } - while probe.less_than(&1u64) { worker.step(); } - println!("worker {}: {} loaded ({} edges, total {:.2?}, load {:.2?})", index, name, edges, timer.elapsed(), timer_load.elapsed()); - - let mut cursor = 0u64; - let mut round = 0u64; - let limit = rounds.unwrap_or(u64::MAX); - while round < limit { - let timer_round = std::time::Instant::now(); - let time = (round + 2) as u64; - for _ in 0..batch { - let remove_idx = cursor; - let add_idx = edges + cursor; - if (remove_idx as usize) % peers == index { - let input_idx = (remove_idx as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(remove_idx, nodes, arity), -1); - } - if (add_idx as usize) % peers == index { - let input_idx = (add_idx as usize) % n_inputs; - inputs[input_idx].update(interactive::gen_row(add_idx, nodes, arity), 1); - } - cursor += 1; - } - for i in inputs.iter_mut() { i.advance_to(time); i.flush(); } - let sync = flags.sync.max(1); - if (round + 1) % sync == 0 || round + 1 == limit { - while probe.less_than(&time) { worker.step(); } - } - - round += 1; - if round % 100 == 0 { - println!("worker {}: {} round {} (total {:.2?}, round {:.2?})", index, name, round, timer.elapsed(), timer_round.elapsed()); - } - } - println!("worker {}: {} done ({} rounds, batch {}, total {:.2?})", index, name, round, batch, timer.elapsed()); - }).unwrap(); -} - -fn main() { - // Strip our flags; everything else stays positional (and the tail is - // forwarded to timely). - let raw_args: Vec = std::env::args().collect(); - let (flags, args): (Flags, Vec) = { - let mut it = raw_args.into_iter(); - let prog = it.next().unwrap(); - let mut flags = Flags::default(); - let mut rest: Vec = Vec::new(); - for a in it { - if a == "--explain" { flags.explain = true; } - else if let Some(q) = a.strip_prefix("--query=") { flags.query = Some(q.to_string()); } - else if a == "--debug-demand" { flags.debug_demand = true; } - else if a == "--diag" { flags.diag = true; } - else if let Some(k) = a.strip_prefix("--sync=") { flags.sync = k.parse().expect("--sync=K"); } - else if let Some(b) = a.strip_prefix("--backend=") { - flags.corgi = match b { "corgi" => true, "vec" => false, other => panic!("unknown backend {other:?} (vec|corgi)") }; - } - else { rest.push(a); } - } - let mut out = vec![prog]; out.extend(rest); - (flags, out) - }; - let program = args.get(1).cloned().unwrap_or_else(|| { std::process::exit(0); }); - let arity: usize = args.get(2).cloned().unwrap_or("2".into()).parse().unwrap(); - let nodes: u64 = args.get(3).cloned().unwrap_or("10".into()).parse().unwrap(); - let edges: u64 = args.get(4).cloned().unwrap_or_else(|| (2 * nodes).to_string()).parse().unwrap(); - let batch: u64 = args.get(5).cloned().unwrap_or("1".into()).parse().unwrap(); - let rounds: Option = args.get(6).map(|s| s.parse().unwrap()); - - let source = interactive::load_program(&program); - let stmts = if program.ends_with(".ddp") { - parse::pipe::parse(&source) - } else { - parse::applicative::parse(&source) - }; - let (n_inputs, imports) = interactive::survey_sources(&stmts); - if !imports.is_empty() { - panic!("ddir: program references imports {:?} but this harness has no trace registry.", imports); - } - let name = std::path::Path::new(&program).file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or(program.clone()); - let timely_args: Vec = args.iter().skip(4).cloned().collect(); - run(&name, stmts, n_inputs, nodes, edges, arity, batch, rounds, flags, timely_args); -} diff --git a/interactive/examples/ddir_server.rs b/interactive/examples/ddir_server.rs index 67bf91d89..c6b2968ae 100644 --- a/interactive/examples/ddir_server.rs +++ b/interactive/examples/ddir_server.rs @@ -8,6 +8,9 @@ //! cargo run --release --example ddir_server -- -w4 # stdin, 4 workers //! e.g. cargo run --example ddir_server -- interactive/examples/server/sessions/shared_trace.txt //! +//! `--backend=vec|corgi` picks the rendering substrate for every installed +//! program (default `vec`). Any other flag is passed to timely (`-w4`). +//! //! # Where parsing happens //! //! Commands are parsed, lowered, and validated **on the main (intake) thread**, @@ -15,13 +18,14 @@ //! and the server keeps running — bad input can never panic a worker. Only a //! well-typed [`Command`] is handed to worker 0, which injects it into a timely //! `Sequencer`; the resulting total order is replayed on every worker, so -//! install/tick/drop stay collective while `feed` is applied on worker 0 only -//! (the arrangement's exchange pact routes data to key owners). +//! install/load/tick/drop stay collective while `feed` is applied on worker 0 +//! only (the arrangement's exchange pact routes data to key owners). //! //! Commands (one per line; `#` or `--` starts a comment): -//! install +//! install [explain=[,debug]] //! feed [val=] [time=] [diff=] -//! tick +//! load +//! tick [n] //! drop //! peek [key] //! list @@ -34,6 +38,17 @@ //! `inject(2,tuple(3,4))`, etc. A value is one whitespace-delimited token, so //! write terms without spaces. `feed`'s value defaults to unit, `time` to the //! current epoch (use a future `time=` to schedule ahead), and `diff` to +1. +//! +//! `load` fills an input in bulk, sharded across the workers: from a recipe +//! (`random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C]`, `iota:N`) or from a +//! text file of whitespace-separated integer rows. A `churn=C` recipe then +//! replaces `C` rows on every `tick` — so `install`, `load`, `tick 100` is a +//! program under standing change. `tick` reports its wall-clock time. +//! +//! `install … explain=` applies the explanation rewrite (every source +//! has `arity` key fields and no value); the query input is the one after the +//! program's own inputs, fed as `(key ; val ++ q)`. `,debug` taps every demand +//! collection with an inspect. use mimalloc::MiMalloc; @@ -53,10 +68,11 @@ use interactive::parse; use interactive::lower; use interactive::scope_ir as st; use interactive::ir::{eval, Value}; -use interactive::server::{Command, Server}; +use interactive::server::{Command, RenderBackend, Server}; -/// Parse, lower, and optimize a program file (`.ddp` = pipe syntax, else applicative). -fn load(path: &str) -> st::Program { +/// Parse, lower, optionally explain, and optimize a program file (`.ddp` = +/// pipe syntax, else applicative). +fn load(path: &str, explain: Option<(usize, bool)>) -> st::Program { let src = interactive::load_program(path); let stmts = if path.ends_with(".ddp") { parse::pipe::parse(&src) @@ -64,6 +80,12 @@ fn load(path: &str) -> st::Program { parse::applicative::parse(&src) }; let mut prog = lower::lower_tree(stmts); + // The rewrite precedes optimization (its rules assume single-op Linears). + if let Some((arity, debug)) = explain { + let shapes: Vec<(usize, usize)> = prog.root.imports.iter().map(|_| (arity, 0)).collect(); + let options = interactive::explain::Options { debug_inspects: debug }; + prog = interactive::explain::explain_with(&prog, &shapes, options); + } prog.optimize(); prog } @@ -98,10 +120,23 @@ fn panic_msg(e: Box) -> String { fn parse_command(line: &str) -> Result { let toks: Vec<&str> = line.split_whitespace().collect(); match toks[0] { - "install" if toks.len() == 3 => { + "install" if toks.len() == 3 || toks.len() == 4 => { let name = toks[1].to_string(); let file = toks[2].to_string(); - let program = catch_unwind(AssertUnwindSafe(|| load(&file))).map_err(panic_msg)?; + let explain = match toks.get(3) { + None => None, + Some(t) => { + let spec = t.strip_prefix("explain=").ok_or_else(|| format!("install: unrecognized argument {:?}", t))?; + let (arity, debug) = match spec.split_once(',') { + Some((a, "debug")) => (a, true), + Some(_) => return Err(format!("install: explain=[,debug], got {:?}", t)), + None => (spec, false), + }; + let arity = arity.parse().map_err(|_| format!("install: explain= arity must be a number, got {:?}", arity))?; + Some((arity, debug)) + } + }; + let program = catch_unwind(AssertUnwindSafe(|| load(&file, explain))).map_err(panic_msg)?; Ok(Command::Install { name, program }) } "feed" if toks.len() >= 4 => { @@ -124,7 +159,18 @@ fn parse_command(line: &str) -> Result { } Ok(Command::Feed { prog, input, key, val, time, diff }) } - "tick" if toks.len() == 1 => Ok(Command::Tick { n: 1 }), + "load" if toks.len() == 4 => { + let prog = toks[1].to_string(); + let input: usize = toks[2].parse().map_err(|_| format!("load: must be a number, got {:?}", toks[2]))?; + Ok(Command::Load { prog, input, source: toks[3].to_string() }) + } + "tick" if toks.len() <= 2 => { + let n = match toks.get(1) { + Some(n) => n.parse().map_err(|_| format!("tick: [n] must be a number, got {:?}", n))?, + None => 1, + }; + Ok(Command::Tick { n }) + } "bind" | "unbind" if toks.len() == 4 => { let trace = toks[1].to_string(); let prog = toks[2].to_string(); @@ -152,9 +198,10 @@ fn parse_command(line: &str) -> Result { fn print_help() { println!("commands:"); - println!(" install "); + println!(" install [explain=[,debug]]"); println!(" feed [val=] [time=] [diff=]"); - println!(" tick"); + println!(" load (random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C] | iota:N | path)"); + println!(" tick [n]"); println!(" bind (feed the trace's changes back in, each tick)"); println!(" unbind "); println!(" drop "); @@ -164,13 +211,13 @@ fn print_help() { } /// Execute one sequenced command on this worker. Collective commands -/// (`install`/`tick`/`drop`) run on every worker; `feed` and all printing -/// happen on worker 0. Returns `false` for `exit`. +/// (`install`/`load`/`tick`/`drop`) run on every worker; `feed` and all +/// printing happen on worker 0. Returns `false` for `exit`. fn dispatch(cmd: &Command, server: &mut Server, worker: &mut Worker) -> bool { let w0 = worker.index() == 0; match cmd { Command::Install { name, program } => match server.install(worker, name, program) { - Ok(()) => if w0 { println!("installed {:?}", name); }, + Ok(()) => if w0 { println!("installed {:?} ({} ops)", name, program.op_count()); }, Err(e) => if w0 { println!("error: {}", e); }, }, Command::Feed { prog, input, key, val, time, diff } => { @@ -187,11 +234,17 @@ fn dispatch(cmd: &Command, server: &mut Server, worker: &mut Worker) -> bool { } } } + // Collective: each worker feeds its shard of the source. + Command::Load { prog, input, source } => match server.load(worker, prog, *input, source) { + Ok(rows) => if w0 { println!("loaded {} rows from {:?} into {:?} input {}", rows, source, prog, input); }, + Err(e) => if w0 { println!("error: {}", e); }, + }, Command::Tick { n } => { + let timer = Instant::now(); for _ in 0..*n { server.tick(worker); } - if w0 { println!("tick -> epoch {}", server.epoch()); } + if w0 { println!("tick -> epoch {} ({:.2?})", server.epoch(), timer.elapsed()); } } Command::Drop { name } => match server.drop_program(worker, name) { Ok(()) => if w0 { println!("dropped {:?}", name); }, @@ -231,15 +284,19 @@ fn main() { } })); - // First positional arg (if it isn't a flag) is the script; the rest go to - // timely. We always pass a leading dummy so getopts has an argv[0] to skip. + // `--backend=` is ours; the first other positional arg is the script; the + // rest go to timely. We always pass a leading dummy so getopts has an + // argv[0] to skip. let mut it = std::env::args(); let _bin = it.next(); + let mut backend = RenderBackend::Vec; let mut script: Option = None; let mut timely_args: Vec = vec!["ddir_server".to_string()]; let mut saw_positional = false; for a in it { - if !saw_positional && !a.starts_with('-') { + if let Some(b) = a.strip_prefix("--backend=") { + backend = b.parse().unwrap_or_else(|e| { eprintln!("{}", e); std::process::exit(1) }); + } else if !saw_positional && !a.starts_with('-') { script = Some(a); saw_positional = true; } else { @@ -255,7 +312,7 @@ fn main() { let guards = timely::execute_from_args(timely_args.into_iter(), move |worker| { let recv = recv.clone(); let me_zero = worker.index() == 0; - let mut server = Server::new(); + let mut server = Server::with_backend(backend); let mut sequencer: Sequencer = Sequencer::new(worker, Instant::now()); let mut done = false; diff --git a/interactive/examples/server/README.md b/interactive/examples/server/README.md index f1c37959e..6f7f77aa0 100644 --- a/interactive/examples/server/README.md +++ b/interactive/examples/server/README.md @@ -14,9 +14,9 @@ totally ordered across workers by a timely `Sequencer`. ## Two kinds of file (don't mix them up) - **`programs/*.ddp`** — DDIR *programs*: dataflow definitions you `install`. - The ones here are server-oriented (they use `import`/`export`), so unlike the - programs in `../programs/` they are not runnable by the batch `ddir_vec` - harness. + The ones here are server-oriented (they use `import`/`export`); the programs + in `../programs/` read positional `input`s instead, which you fill with + `feed` or `load` (see "Running a program in batch" below). - **`sessions/*.txt`** — *command scripts*: a stream of server commands (`install`/`feed`/`tick`/…) you hand to the server. You do **not** `install` a session; you run the server *on* it. @@ -29,6 +29,9 @@ cargo run --release --example ddir_server -- interactive/examples/server/session # Or interactively (no script arg): type `help`, or `exit`. cargo run --release --example ddir_server + +# Render every installed program on the Corgi columnar backend (default: vec): +cargo run --release --example ddir_server -- --backend=corgi -w4 ``` Paths inside the session scripts are relative to the `interactive/` crate @@ -39,9 +42,11 @@ the repo root with `--example`; adjust if you `cd interactive` first). | command | effect | |---|---| -| `install ` | parse + lower + install a program under `` | +| `install [explain=[,debug]]` | parse + lower + install a program under ``; optionally after the explanation rewrite | | `feed [val=] [time=] [diff=]` | stage an input update | -| `tick` | close the epoch and run to quiescence | +| `load ` | bulk-load an input from a recipe (`random:…`, `iota:N`) or a file of integer rows, sharded across workers | +| `tick [n]` | close `n` epochs (default 1), running to quiescence after each; reports the wall-clock time | +| `bind ` / `unbind …` | feed a trace's changes back into an input at every tick (one-epoch-delayed feedback) | | `drop ` | evict a program (refused if a live program still imports its trace) | | `peek [key]` | print a trace's current contents (consolidated across workers) | | `list` | show traces (+ importer counts) and installed programs | @@ -51,6 +56,36 @@ or any **closed scalar term, written without spaces** (`inject(2,tuple(3,4))`, `list(1,2,3)`) for ADT-shaped data such as ASTs. `feed` defaults to value=unit, `time`=the current epoch (use a future `time=` to schedule ahead), `diff`=+1. +## Running a program in batch + +The programs in `../programs/` (reachability, SCC, stable matching, …) read +positional inputs. A session fills them in bulk and closes epochs; that is the +whole of the old single-program harness, so there is no separate binary: + +``` +install scc ../programs/scc.ddp +load scc 0 random:nodes=100000,edges=200000,churn=100 +tick # the initial load: reported as one epoch's time +tick 100 # 100 epochs of 100 replaced edges each, timed together +peek result +exit +``` + +`load` deals the rows across the workers (each feeds its shard, and the +exchange places every row on its key's owner). A `random:` recipe with +`churn=C` keeps the input changing: every later `tick` retracts the next `C` +rows of its window and adds `C` fresh ones, which is the standing-change regime +programs are benchmarked under. A file source is one row per line of +whitespace-separated integers, each becoming `(Tuple[ints] ; ())`; the +`aoc2023/run.sh` suite drives every AoC program this way. `feed` still works +alongside `load` for the small inputs (roots, queries). + +`install … explain=` applies the explanation rewrite before +optimization, treating every source as `arity` key fields with no value; the +rewritten program has one extra input after its own — the query input, fed as +`(key ; val ++ q)` — and exports one `demand:inputN` trace per source, which +you `peek`. `,debug` taps every demand collection with an inspect. + ## Generated (named) sources A program can `import` a *recipe* name instead of another program's export: diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 0b48a19ef..e149c4211 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -447,129 +447,3 @@ pub fn render_tree_rows<'s>( }) .collect() } - -/// Evaluate `program` on explicit inputs via the **corgi** backend (mirrors [`crate::backend::vec::evaluate`]). -/// -/// Inputs/exports cross the iterative-scope boundary as Vec rows (which support refinement -/// enter/leave); corgi containers exist only INSIDE the dynamic scope, where `Enter`/`Leave` are the -/// same-Time identity. The `ToCorgi`/`FromCorgi` unaries are the only row↔corgi conversions. -pub fn evaluate( - program: &st::Program, - inputs: &[Vec<(Row, Row)>], -) -> std::collections::BTreeMap> { - evaluate_with_workers(program, inputs, 1) -} - -/// [`evaluate`] on `workers` worker threads in one process. -/// -/// The answer must not depend on `workers`: the exchange places each key on one worker, every -/// operator is key-local from there, and each worker's captured output is a disjoint share of the -/// same collection, so summing the shares reproduces the single-worker result exactly. That -/// invariant is the correctness statement for [`CorgiPact`], and the backend gate checks it by -/// running each program at several worker counts. -/// -/// Input rows are dealt round-robin across workers (`row index % peers`) so each row is introduced -/// exactly once globally — where they enter is irrelevant, since `arrange` re-places them. -pub fn evaluate_with_workers( - program: &st::Program, - inputs: &[Vec<(Row, Row)>], - workers: usize, -) -> std::collections::BTreeMap> { - evaluate_with_config(program, inputs, timely::Config::process(workers)) -} - -/// [`evaluate_with_workers`] with the timely configuration named outright. -/// -/// The configuration decides which half of the exchange gets exercised. -/// `Config::process(n)` hands containers between worker threads as typed values — the -/// [`CorgiDistributor`](crate::corgi::exchange::CorgiDistributor)'s partition runs, but no bytes -/// are produced. `CommunicationConfig::ProcessBinary(n)` puts the same threads behind serializing -/// channels, so every container makes the round trip through -/// [the wire format](crate::corgi::bytes) — the multi-*process* path, without the processes. -pub fn evaluate_with_config( - program: &st::Program, - inputs: &[Vec<(Row, Row)>], - config: timely::Config, -) -> std::collections::BTreeMap> { - use std::collections::BTreeMap; - use std::sync::{Arc, Mutex, mpsc::channel}; - use timely::dataflow::operators::core::capture::{Capture, Event}; - use differential_dataflow::input::Input; - use differential_dataflow::dynamic::pointstamp::PointStamp; - - let names: Vec = program.root.exports.iter().map(|e| e.name.clone()).collect(); - let mut txs = Vec::new(); - let mut rxs = Vec::new(); - for _ in &names { - let (tx, rx) = channel::>>(); - txs.push(tx); - rxs.push(rx); - } - // Every worker captures into the same per-export channel. `Sender` is `Send` but not `Sync`, - // and timely's worker closure must be both, so the senders travel behind a mutex and each - // worker clones its own out of it once, at construction. - let txs = Arc::new(Mutex::new(txs)); - - let program = program.clone(); - let inputs: Vec> = inputs.to_vec(); - let guards = timely::execute(config, move |worker| { - let (index, peers) = (worker.index(), worker.peers()); - let txs: Vec<_> = txs.lock().expect("capture senders").iter().cloned().collect(); - let mut handles = worker.dataflow::(|scope| { - let mut handles = Vec::new(); - let mut collections = Vec::new(); - for _ in 0..inputs.len() { - let (h, c) = scope.new_collection::<(Row, Row), Diff>(); - handles.push(h); - collections.push(c); - } - let exports = scope.iterative::, _, _>(|inner| { - // Enter row collections (refinement); rows convert to corgi containers and - // back inside `render_tree_rows`. - let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect(); - let root_imports: Vec<_> = program - .root - .imports - .iter() - .map(|imp| match &imp.from { - st::Source::Input(n) => entered[*n].clone(), - other => panic!("corgi evaluate: unsupported source {other:?}"), - }) - .collect(); - render_tree_rows(&program.root, inner.clone(), 0, root_imports) - .into_iter() - .map(|rows| rows.leave(scope)) - .collect::>() - }); - for (col, tx) in exports.into_iter().zip(txs) { - col.inner.capture_into(tx); - } - handles - }); - for (i, rows) in inputs.iter().enumerate() { - for r in rows.iter().skip(index).step_by(peers) { - handles[i].update(r.clone(), 1); - } - } - }) - .expect("corgi evaluate: worker startup"); - // Join the workers before draining, so every capture has been sent and every cloned sender - // dropped; otherwise the receive loop below would block on a channel that is still open. - guards.join(); - - names - .into_iter() - .zip(rxs) - .map(|(name, rx)| { - let mut acc: BTreeMap<(Row, Row), Diff> = BTreeMap::new(); - for event in rx { - if let Event::Messages(_, data) = event { - for ((k, v), _, d) in data { - *acc.entry((k, v)).or_insert(0) += d; - } - } - } - (name, acc.into_iter().filter(|(_, d)| *d != 0).collect()) - }) - .collect() -} diff --git a/interactive/src/backend/vec.rs b/interactive/src/backend/vec.rs index 2a5001cea..c01cd52b7 100644 --- a/interactive/src/backend/vec.rs +++ b/interactive/src/backend/vec.rs @@ -169,80 +169,3 @@ pub fn render_tree<'s>( ) -> Vec> { crate::backend::render_tree::(s, scope, depth, imports) } - -/// Evaluate `program` on explicit inputs: single worker, in-process, every -/// export returned as its consolidated final contents (rows with non-zero -/// net multiplicity). -/// -/// `inputs[i]` provides the rows of positional input `i`, each with -/// multiplicity +1; all rows are introduced at time 0 and the computation -/// runs to quiescence. This is the data-in/data-out entry point that tests -/// and tools build on — e.g. feeding an explanation's demand-set back -/// through the original program to check the queried output reproduces. -pub fn evaluate( - program: &st::Program, - inputs: &[Vec<(Row, Row)>], -) -> std::collections::BTreeMap> { - use std::sync::mpsc::channel; - use timely::dataflow::operators::core::capture::{Capture, Event}; - use differential_dataflow::input::Input; - - let names: Vec = program.root.exports.iter().map(|e| e.name.clone()).collect(); - let mut txs = Vec::with_capacity(names.len()); - let mut rxs = Vec::with_capacity(names.len()); - for _ in &names { - let (tx, rx) = channel::>>(); - txs.push(tx); - rxs.push(rx); - } - - let program = program.clone(); - let inputs: Vec> = inputs.to_vec(); - timely::execute_directly(move |worker| { - let mut handles = worker.dataflow::(|scope| { - let mut handles = Vec::new(); - let mut collections = Vec::new(); - for _ in 0..inputs.len() { - let (h, c) = scope.new_collection::<(Row, Row), Diff>(); - handles.push(h); - collections.push(c); - } - let exports = scope.iterative::, _, _>(|inner| { - let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect(); - let root_imports: Vec<_> = program.root.imports.iter().map(|imp| match &imp.from { - st::Source::Input(n) => entered[*n].clone(), - other => panic!("evaluate: unsupported source {:?}", other), - }).collect(); - let exports = render_tree(&program.root, inner.clone(), 0, root_imports); - exports.into_iter().map(|c| c.leave(scope)).collect::>() - }); - for (col, tx) in exports.into_iter().zip(txs) { - col.inner.capture_into(tx); - } - handles - }); - for (i, rows) in inputs.iter().enumerate() { - for r in rows { - handles[i].update(r.clone(), 1); - } - } - // Dropping the handles closes the inputs; `execute_directly` then - // steps the worker until the dataflow completes. - }); - - names - .into_iter() - .zip(rxs) - .map(|(name, rx)| { - let mut acc: std::collections::BTreeMap<(Row, Row), Diff> = std::collections::BTreeMap::new(); - for event in rx { - if let Event::Messages(_, data) = event { - for ((k, v), _, d) in data { - *acc.entry((k, v)).or_insert(0) += d; - } - } - } - (name, acc.into_iter().filter(|(_, d)| *d != 0).collect()) - }) - .collect() -} diff --git a/interactive/src/lib.rs b/interactive/src/lib.rs index ba3b3a95b..cd530d93f 100644 --- a/interactive/src/lib.rs +++ b/interactive/src/lib.rs @@ -10,45 +10,6 @@ pub mod server; // the Value reverse model implements `time_le`/`strip` directly over the nested // chain tuple, so explain no longer needs it. -use std::collections::BTreeSet; - -use parse::{Stmt, Expr}; - -/// Survey a program's external sources: the count of positional inputs (one -/// more than the largest `input N` index, zero if none appear) and the set of -/// names referenced by `import "name"`. Two kinds because `import` does not yet -/// subsume `input` — see `ir::Node::Import`; this returns one number when that -/// cutover happens. -pub fn survey_sources(stmts: &[Stmt]) -> (usize, BTreeSet) { - let mut positional = 0usize; - let mut imports = BTreeSet::new(); - walk_stmts(stmts, &mut positional, &mut imports); - (positional, imports) -} - -fn walk_stmts(stmts: &[Stmt], positional: &mut usize, imports: &mut BTreeSet) { - for stmt in stmts { - match stmt { - Stmt::Let(_, expr) | Stmt::Var(_, expr) | Stmt::Export(_, expr) => walk_expr(expr, positional, imports), - Stmt::Scope(_, body) => walk_stmts(body, positional, imports), - } - } -} - -fn walk_expr(expr: &Expr, positional: &mut usize, imports: &mut BTreeSet) { - match expr { - Expr::Input(n) => { *positional = (*positional).max(n + 1); }, - Expr::Import(name) => { imports.insert(name.clone()); }, - Expr::Map(e, _) | Expr::Negate(e) | Expr::Arrange(e) - | Expr::EnterAt(e, _) | Expr::LiftIter(e) | Expr::Filter(e, _) - | Expr::FlatMap(e, _) - | Expr::Reduce(e, _) | Expr::Inspect(e, _) => walk_expr(e, positional, imports), - Expr::Join(l, r, _) => { walk_expr(l, positional, imports); walk_expr(r, positional, imports); }, - Expr::Concat(es) => { for e in es { walk_expr(e, positional, imports); } }, - Expr::Name(_) | Expr::Qualified(_, _) => {}, - } -} - /// Load a program source file. pub fn load_program(path: &str) -> String { std::fs::read_to_string(path).unwrap_or_else(|e| panic!("Cannot read {}: {}", path, e)) diff --git a/interactive/src/server.rs b/interactive/src/server.rs index 4ccdb9b98..f8d0f5461 100644 --- a/interactive/src/server.rs +++ b/interactive/src/server.rs @@ -32,6 +32,9 @@ //! for teardown. //! - **feed** stages an input update at a chosen time (default: the current //! epoch) via `update_at`, so inputs can be scheduled into the future. +//! - **load** fills an input in bulk from a recipe or a file, each worker +//! feeding its own shard; a churning recipe then changes the input on every +//! tick, which is how a program is run under standing change. //! - **tick** advances all inputs to the next epoch, runs to quiescence, then //! lets every trace compact (an importer's own handle holds the shared //! `TraceBox` back to what it still needs). @@ -242,6 +245,14 @@ pub enum Command { input: usize, updates: Vec, }, + /// Bulk-load `source` — a recipe name or a file path — into positional + /// `input` of `prog` at the current epoch. Collective: every worker feeds + /// its own shard of the rows (see [`Server::load`]). + Load { + prog: String, + input: usize, + source: String, + }, /// Close `n` epochs, running to quiescence after each one. Tick { n: u64 }, /// Drop the named program. @@ -284,8 +295,10 @@ struct Installed { /// [`Origin`]. Generated/clock entries advance and drop like any program but /// are not writable by `feed`. origin: Origin, - /// Generator recipe and next row to retract, for changing random sources. - generator: Option<(Recipe, u64)>, + /// Per input: the recipe whose rows it holds and the next row to retract, + /// for inputs that churn each `tick` (a generated `random:` source's own + /// input, or a program input bulk-loaded from such a recipe). + generators: HashMap, } /// A stable, transport-friendly description of one installed dataflow. @@ -563,7 +576,7 @@ impl Server { dataflow_id, probe, origin: Origin::Program, - generator: None, + generators: HashMap::new(), }, ); Ok(()) @@ -607,7 +620,7 @@ impl Server { dataflow_id, probe, origin: Origin::Generated, - generator: Some((recipe, 0)), + generators: HashMap::from([(0usize, (recipe, 0u64))]), }, ); } @@ -645,7 +658,7 @@ impl Server { dataflow_id, probe, origin: Origin::Clock, - generator: None, + generators: HashMap::new(), }, ); } @@ -696,6 +709,70 @@ impl Server { Ok(()) } + /// Bulk-load rows into positional `input` of `prog` at the current epoch. + /// + /// `source` is either a recipe (`random:…`, `iota:N` — the same names an + /// `import` accepts) or the path of a text file with one row per line of + /// whitespace-separated integers (`(Tuple[ints] ; ())`). Collective: every + /// worker must call this, and each feeds only its shard (`row % peers == + /// index`) through its own handle, so the union is the source exactly once + /// and the exchange places each row on its key's owner. + /// + /// A `random:` recipe with `churn=C` keeps churning: every later `tick` + /// retracts the next `C` rows of the window and adds `C` fresh ones, the + /// standing-change regime a program is benchmarked under. Returns the + /// number of rows in the source (across all workers). + pub fn load( + &mut self, + worker: &Worker, + prog: &str, + input: usize, + source: &str, + ) -> Result { + let time = self.epoch; + self.validate_feed(prog, input, time)?; + let (index, peers) = (worker.index(), worker.peers()); + let mine = |e: u64| (e as usize) % peers == index; + let recipe = Recipe::parse(source); + let (total, rows): (u64, Vec<(Value, Value)>) = match recipe { + Some(recipe) => ( + recipe.rows_len(), + (0..recipe.rows_len()).filter(|e| mine(*e)).map(|e| recipe.row(e)).collect(), + ), + None => { + let text = std::fs::read_to_string(source) + .map_err(|e| format!("load: cannot read {:?}: {}", source, e))?; + let mut total = 0; + let mut rows = Vec::new(); + for (e, line) in text.lines().filter(|l| !l.trim().is_empty()).enumerate() { + total += 1; + if !mine(e as u64) { + continue; + } + let fields = line + .split_whitespace() + .map(|t| t.parse::().map(Value::Int)) + .collect::, _>>() + .map_err(|_| format!("load: line {} of {:?} is not a row of integers: {:?}", e + 1, source, line))?; + rows.push((Value::Tuple(fields), Value::unit())); + } + (total, rows) + } + }; + let installed = self + .programs + .get_mut(&canonical_source_name(prog)) + .expect("load target was prevalidated"); + if let Some(recipe @ Recipe::Random { churn: 1.., .. }) = recipe { + installed.generators.insert(input, (recipe, 0)); + } + let handle = installed.inputs.get_mut(&input).expect("load input was prevalidated"); + for row in rows { + handle.update_at(row, time, 1); + } + Ok(total) + } + /// Check everything about an input target that can fail without changing /// its handle. Both singular and batched feeds validate before applying. fn validate_feed(&self, prog: &str, input: usize, time: OuterTime) -> Result<(), String> { @@ -1058,10 +1135,10 @@ impl Server { } // A random source denotes an infinite deterministic row stream. // Each tick replaces `churn` members of its fixed-size window. - if let Some((recipe, cursor)) = &mut installed.generator { + for (input, (recipe, cursor)) in installed.generators.iter_mut() { let recipe = *recipe; if let Recipe::Random { edges, churn, .. } = recipe { - if let Some(h) = installed.inputs.get_mut(&0) { + if let Some(h) = installed.inputs.get_mut(input) { for _ in 0..churn { let old = *cursor; let new = edges + *cursor; @@ -1185,3 +1262,57 @@ impl Default for Server { Server::new() } } + +/// Evaluate `program` on explicit inputs through a throwaway server: every +/// export's consolidated final contents (rows with non-zero net multiplicity), +/// by name. +/// +/// This is the data-in/data-out entry point the test suites build on, and it +/// takes the same path a live install does — `install`, one `feed` of each +/// positional input (`inputs[i]` holds input `i`'s rows, each at multiplicity +/// +1, dealt round-robin across the workers), one `tick`, then a `snapshot` of +/// each export. `config` picks the worker group: `Config::process(n)` hands +/// exchanged containers between threads as typed values, while +/// `CommunicationConfig::ProcessBinary(n)` sends every one through the wire +/// format. The answer must not depend on either choice. +pub fn evaluate( + backend: RenderBackend, + config: timely::Config, + program: &st::Program, + inputs: &[Vec<(Value, Value)>], +) -> std::collections::BTreeMap> { + let program = program.clone(); + let inputs = inputs.to_vec(); + let guards = timely::execute(config, move |worker| { + let mut server = Server::with_backend(backend); + server.install(worker, "evaluate", &program).expect("evaluate: install"); + let has_input = |i: usize| program.root.imports.iter().any(|imp| matches!(imp.from, st::Source::Input(n) if n == i)); + for (input, rows) in inputs.iter().enumerate().filter(|(i, _)| has_input(*i)) { + let shard = rows + .iter() + .skip(worker.index()) + .step_by(worker.peers()) + .map(|(key, val)| InputUpdate { key: key.clone(), val: val.clone(), diff: 1 }) + .collect(); + server.feed_batch("evaluate", input, shard).expect("evaluate: feed"); + } + server.tick(worker); + // `snapshot` gathers to worker 0; the other workers' results are empty. + program + .root + .exports + .iter() + .map(|e| { + let rows = server.snapshot(worker, &e.name).expect("evaluate: snapshot"); + (e.name.clone(), rows.into_iter().map(|(k, v, d)| ((k, v), d)).collect()) + }) + .collect() + }) + .expect("evaluate: worker startup"); + guards + .join() + .into_iter() + .next() + .expect("evaluate: worker 0") + .expect("evaluate: worker 0 returned") +} diff --git a/interactive/tests/corgi_backend.rs b/interactive/tests/corgi_backend.rs index b3ca27b78..c6c835655 100644 --- a/interactive/tests/corgi_backend.rs +++ b/interactive/tests/corgi_backend.rs @@ -1,8 +1,11 @@ //! The corgi backend's correctness gate: each canonical `.ddp` program must evaluate //! identically through the corgi backend and the reference vec backend. +//! +//! Every evaluation goes through the server (`server::evaluate`): install, feed, tick, +//! snapshot — the same path a live install takes. -use interactive::backend::{corgi, vec}; use interactive::ir::Value; +use interactive::server::{evaluate, RenderBackend}; use interactive::{lower, parse}; fn tup(fields: &[i64]) -> Value { @@ -77,14 +80,14 @@ fn assert_backends_agree(prog: &str) { let mut tree = lower::lower_tree(parse::pipe::parse(&src)); tree.optimize(); let inputs = inputs_for(prog); - let want = vec::evaluate(&tree, &inputs); + let want = evaluate(RenderBackend::Vec, timely::Config::process(1), &tree, &inputs); // At every worker count: the exchange places each key on one worker and every operator is // key-local from there, so the answer must not depend on how many workers ran it. 3 is in the // list on purpose — it is not a power of two, so it takes the modulus path rather than the // mask, and it cannot divide these inputs evenly. for workers in [1, 2, 3, 4] { assert_eq!( - corgi::evaluate_with_workers(&tree, &inputs, workers), + evaluate(RenderBackend::Corgi, timely::Config::process(workers), &tree, &inputs), want, "corgi backend at {workers} worker(s) disagrees with the vec backend on {prog}", ); @@ -93,7 +96,7 @@ fn assert_backends_agree(prog: &str) { // round trip through the wire format. This is the multi-process path: `Config::process` above // hands containers between threads as typed values and never encodes a byte. assert_eq!( - corgi::evaluate_with_config(&tree, &inputs, serializing(3)), + evaluate(RenderBackend::Corgi, serializing(3), &tree, &inputs), want, "corgi backend over serializing channels disagrees with the vec backend on {prog}", ); diff --git a/interactive/tests/explain.rs b/interactive/tests/explain.rs index cf55f1ade..226466635 100644 --- a/interactive/tests/explain.rs +++ b/interactive/tests/explain.rs @@ -1,5 +1,5 @@ //! End-to-end semantic tests for the explanation rewrite, built on -//! `backend::vec::evaluate` (explicit inputs in, every export out). The +//! `server::evaluate` (explicit inputs in, every export out). The //! sufficiency properties use that row execution to evaluate the rewrite; a //! final section cross-checks the behavior shared by the row and corgi //! implementations for these programs. @@ -11,12 +11,12 @@ //! regenerate the queried output row. Tests marked `#[ignore]` are heavier sweeps meant for //! `cargo test --release -- --ignored`. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; -use interactive::backend::corgi::evaluate as corgi_evaluate; -use interactive::backend::vec::{evaluate, Row}; +use interactive::backend::vec::Row; use interactive::ir::Value; use interactive::scope_ir::Program; +use interactive::server::{self, RenderBackend}; use interactive::{explain, lower, parse}; /// SCC with the scc edge-set itself as the result, so individual output @@ -101,6 +101,12 @@ fn row(fields: &[i64]) -> Row { Value::Tuple(fields.iter().map(|&n| Value::Int(n)).collect()) } +/// Run `p` on `inputs` through the server on the vec backend: every export's +/// consolidated rows, by name. +fn evaluate(p: &Program, inputs: &[Vec<(Row, Row)>]) -> BTreeMap> { + server::evaluate(RenderBackend::Vec, timely::Config::process(1), p, inputs) +} + /// Run `p` on `inputs` and return one export's rows (asserting positive /// multiplicities — a set-like result). fn export_rows(p: &Program, inputs: &[Vec<(Row, Row)>], export: &str) -> BTreeSet<(Row, Row)> { @@ -508,7 +514,7 @@ fn assert_explained_backends_agree( ex_inputs.push(query_rows(queries)); let by_vec = evaluate(&ex, &ex_inputs); - let by_corgi = corgi_evaluate(&ex, &ex_inputs); + let by_corgi = server::evaluate(RenderBackend::Corgi, timely::Config::process(1), &ex, &ex_inputs); assert_eq!( by_vec.keys().collect::>(), by_corgi.keys().collect::>(), From d875d6e0cc92364f058ac1650d26bf25044cbb0e Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 4 Sep 2026 09:25:46 -0400 Subject: [PATCH 2/5] Corgi: a literal-condition `if` compiles only its live branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `if(1, $1, 0)` is the idiom for carrying a collected List as one field (a bare `$1` would splice it). Corgi's typer demanded that both branches share a shape, so the dead `0` made the whole program a type error — AoC day 13 part 1 was the one part corgi could not run. With a literal condition the taken branch is known, so only it is compiled; the vec backend evaluated it that way already. Pinned by `tests/programs/if_literal.ddp`; AoC is now 33/33 on both backends. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QDsLC46QQW9aBrksaWad6T --- interactive/examples/aoc2023/README.md | 6 ++---- interactive/src/corgi/logic.rs | 6 ++++++ interactive/tests/corgi_backend.rs | 2 ++ interactive/tests/programs/if_literal.ddp | 5 +++++ 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 interactive/tests/programs/if_literal.ddp diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md index 90f3c251d..8bd19a888 100644 --- a/interactive/examples/aoc2023/README.md +++ b/interactive/examples/aoc2023/README.md @@ -25,10 +25,8 @@ mixed-arity inputs. runs each part as one server session — `install`, `load` the fact file into input 0, `tick` — and reads the answer off the `[partN]` inspect line. -The vec backend passes all 33 parts. Corgi passes 32: it needs day05's -arity-padded inputs (`run.sh corgi` transcribes with `--pad`), and day13 -part 1 still crashes it (a batch of one shape reaching an operator pinned at -another — a known open issue). +Both backends pass all 33 parts. Corgi needs day05's arity-padded inputs +(`run.sh corgi` transcribes with `--pad`). ## Verdicts diff --git a/interactive/src/corgi/logic.rs b/interactive/src/corgi/logic.rs index 434539b71..3b54e17ac 100644 --- a/interactive/src/corgi/logic.rs +++ b/interactive/src/corgi/logic.rs @@ -356,6 +356,12 @@ pub fn compile( }) } Term::If { cond, then, els } => { + // A literal condition takes one branch, so only that branch needs a shape. This is + // the `if(1, $1, 0)` idiom: it carries a whole value as ONE field where a bare `$1` + // would splice, and its dead branch need not agree in shape. + if let Term::Int(c) = **cond { + return compile(if c != 0 { then } else { els }, b, env, env_shapes, anchor, expected); + } // `Select` blends per row and is shape-generic; the branches must share one shape, // and a branch that cannot fix its own (a bare `None`) takes the other's. let (ts, es) = branch_shapes(then, els, env_shapes, expected)?; diff --git a/interactive/tests/corgi_backend.rs b/interactive/tests/corgi_backend.rs index c6c835655..914a78923 100644 --- a/interactive/tests/corgi_backend.rs +++ b/interactive/tests/corgi_backend.rs @@ -45,6 +45,7 @@ fn inputs_for(prog: &str) -> Vec> { // sum_skew: any keyed pairs — the skew is in the program, not the data. "sum_skew" => vec![rows(&[&[1, 10], &[2, 20], &[2, 21], &[3, 30]])], "case_ops" => vec![rows(&[&[1, 10], &[2, 20], &[3, 14], &[3, 30]])], + "if_literal" => vec![rows(&[&[1, 10], &[1, 20], &[2, 30]])], // pair_keys: composite keys with overlap, fanout, and one-sided keys on both sides. "pair_keys" => vec![ rows(&[&[1, 1, 10], &[1, 2, 20], &[2, 1, 30], &[2, 1, 31], &[9, 9, 90]]), @@ -123,6 +124,7 @@ fn serializing(n: usize) -> timely::Config { #[test] fn empty_batch() { assert_backends_agree("empty_batch"); } #[test] fn sum_skew() { assert_backends_agree("sum_skew"); } #[test] fn case_ops() { assert_backends_agree("case_ops"); } +#[test] fn if_literal() { assert_backends_agree("if_literal"); } #[test] fn tour() { assert_backends_agree("tour"); } #[test] fn pair_keys() { assert_backends_agree("pair_keys"); } #[test] fn signed_min() { assert_backends_agree("signed_min"); } diff --git a/interactive/tests/programs/if_literal.ddp b/interactive/tests/programs/if_literal.ddp new file mode 100644 index 000000000..eebaa524c --- /dev/null +++ b/interactive/tests/programs/if_literal.ddp @@ -0,0 +1,5 @@ +-- Pins the literal-condition `if`. `if(1, $1, 0)` carries a collected List as +-- ONE field (a bare `$1` would splice its elements), and corgi compiles only +-- the live branch, so the dead branch need not agree in shape. +let lists = input 0 | key($0[0] ; $0[1]) | collect; +export "result" = lists | map($0[0] ; if(1, $1, 0), if(0, 7, $0[0])); From e610a6106d47fa74b289fb5ad3be4d4eb3892cd4 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Sat, 5 Sep 2026 17:00:26 -0400 Subject: [PATCH 3/5] DDIR server: `feed from ` on both front ends The live server crate (`interactive/server`, the networked `ddir_server`) matches every server command exhaustively, so the new `Command::Load` variant broke its build. It now handles it: `feed from ` fills an input from a source the server reads itself, each worker taking its shard, so no row crosses the wire. The example driver's command is spelled the same way (it was `load`, which in the live server's protocol means install). A parser test and a four-worker integration step cover it; the AoC suite passes on both backends with the new spelling. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QDsLC46QQW9aBrksaWad6T --- interactive/README.md | 2 +- interactive/examples/aoc2023/README.md | 4 +-- interactive/examples/aoc2023/run.sh | 2 +- interactive/examples/ddir_server.rs | 32 +++++++++++----------- interactive/examples/server/README.md | 14 +++++----- interactive/server/README.md | 8 ++++-- interactive/server/src/cmd.rs | 35 +++++++++++++++++++++++++ interactive/server/src/loop_.rs | 4 +++ interactive/server/tests/multiworker.rs | 21 +++++++++++++++ interactive/src/server.rs | 6 ++--- 10 files changed, 96 insertions(+), 32 deletions(-) diff --git a/interactive/README.md b/interactive/README.md index 2ae49e352..2613c7f83 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -79,7 +79,7 @@ graph of 100 nodes and 200 edges, 10 of which change each epoch, for 100 epochs: ``` cd interactive printf 'install reach examples/programs/reach.ddp -load reach 0 random:nodes=100,edges=200,churn=10 +feed reach 0 from random:nodes=100,edges=200,churn=10 feed reach 1 0 tick 100 exit diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md index 8bd19a888..b05e4f780 100644 --- a/interactive/examples/aoc2023/README.md +++ b/interactive/examples/aoc2023/README.md @@ -22,8 +22,8 @@ mixed-arity inputs. ./run.sh corgi # the same through the Corgi columnar backend `run.sh` first runs `transcribe.py` (python3) to regenerate `gen/`, then -runs each part as one server session — `install`, `load` the fact file into -input 0, `tick` — and reads the answer off the `[partN]` inspect line. +runs each part as one server session — `install`, `feed … from` the fact +file into input 0, `tick` — and reads the answer off the `[partN]` inspect line. Both backends pass all 33 parts. Corgi needs day05's arity-padded inputs (`run.sh corgi` transcribes with `--pad`). diff --git a/interactive/examples/aoc2023/run.sh b/interactive/examples/aoc2023/run.sh index 4871249ab..3c3587339 100755 --- a/interactive/examples/aoc2023/run.sh +++ b/interactive/examples/aoc2023/run.sh @@ -16,7 +16,7 @@ while read -r day part expected; do [ -n "$PAD" ] && [ -f "gen/$dir/input${part}p.txt" ] && inp=gen/$dir/input${part}p.txt # One session per part: install the program, bulk-load its input, close the # epoch. The answer is the `Int` on the `[partN]` inspect line. - got=$(printf 'install p %s\nload p 0 %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ + got=$(printf 'install p %s\nfeed p 0 from %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ | "$SERVER" --backend="$BACKEND" 2>&1 \ | sed -n "s/.*\\[part$part\\].*Int(\\(-\\{0,1\\}[0-9]*\\)).*/\\1/p") if [ "$got" = "$expected" ]; then diff --git a/interactive/examples/ddir_server.rs b/interactive/examples/ddir_server.rs index c6b2968ae..94504bb2f 100644 --- a/interactive/examples/ddir_server.rs +++ b/interactive/examples/ddir_server.rs @@ -18,13 +18,14 @@ //! and the server keeps running — bad input can never panic a worker. Only a //! well-typed [`Command`] is handed to worker 0, which injects it into a timely //! `Sequencer`; the resulting total order is replayed on every worker, so -//! install/load/tick/drop stay collective while `feed` is applied on worker 0 -//! only (the arrangement's exchange pact routes data to key owners). +//! install/tick/drop and `feed … from` stay collective while a row `feed` is +//! applied on worker 0 only (the arrangement's exchange pact routes data to +//! key owners). //! //! Commands (one per line; `#` or `--` starts a comment): //! install [explain=[,debug]] //! feed [val=] [time=] [diff=] -//! load +//! feed from //! tick [n] //! drop //! peek [key] @@ -39,11 +40,12 @@ //! write terms without spaces. `feed`'s value defaults to unit, `time` to the //! current epoch (use a future `time=` to schedule ahead), and `diff` to +1. //! -//! `load` fills an input in bulk, sharded across the workers: from a recipe -//! (`random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C]`, `iota:N`) or from a -//! text file of whitespace-separated integer rows. A `churn=C` recipe then -//! replaces `C` rows on every `tick` — so `install`, `load`, `tick 100` is a -//! program under standing change. `tick` reports its wall-clock time. +//! `feed … from ` fills an input in bulk, sharded across the workers: +//! from a recipe (`random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C]`, +//! `iota:N`) or from a text file of whitespace-separated integer rows. A +//! `churn=C` recipe then replaces `C` rows on every `tick` — so `install`, +//! `feed … from`, `tick 100` is a program under standing change. `tick` +//! reports its wall-clock time. //! //! `install … explain=` applies the explanation rewrite (every source //! has `arity` key fields and no value); the query input is the one after the @@ -142,6 +144,9 @@ fn parse_command(line: &str) -> Result { "feed" if toks.len() >= 4 => { let prog = toks[1].to_string(); let input: usize = toks[2].parse().map_err(|_| format!("feed: must be a number, got {:?}", toks[2]))?; + if let ["from", source] = toks[3..] { + return Ok(Command::Load { prog, input, source: source.to_string() }); + } let key = catch_unwind(AssertUnwindSafe(|| parse_value(toks[3]))).map_err(panic_msg)?; let mut val = Value::unit(); let mut time = None; @@ -159,11 +164,6 @@ fn parse_command(line: &str) -> Result { } Ok(Command::Feed { prog, input, key, val, time, diff }) } - "load" if toks.len() == 4 => { - let prog = toks[1].to_string(); - let input: usize = toks[2].parse().map_err(|_| format!("load: must be a number, got {:?}", toks[2]))?; - Ok(Command::Load { prog, input, source: toks[3].to_string() }) - } "tick" if toks.len() <= 2 => { let n = match toks.get(1) { Some(n) => n.parse().map_err(|_| format!("tick: [n] must be a number, got {:?}", n))?, @@ -200,7 +200,7 @@ fn print_help() { println!("commands:"); println!(" install [explain=[,debug]]"); println!(" feed [val=] [time=] [diff=]"); - println!(" load (random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C] | iota:N | path)"); + println!(" feed from (random:nodes=N,edges=E[,arity=A][,seed=S][,churn=C] | iota:N | path)"); println!(" tick [n]"); println!(" bind (feed the trace's changes back in, each tick)"); println!(" unbind "); @@ -211,8 +211,8 @@ fn print_help() { } /// Execute one sequenced command on this worker. Collective commands -/// (`install`/`load`/`tick`/`drop`) run on every worker; `feed` and all -/// printing happen on worker 0. Returns `false` for `exit`. +/// (`install`/`feed … from`/`tick`/`drop`) run on every worker; a row `feed` +/// and all printing happen on worker 0. Returns `false` for `exit`. fn dispatch(cmd: &Command, server: &mut Server, worker: &mut Worker) -> bool { let w0 = worker.index() == 0; match cmd { diff --git a/interactive/examples/server/README.md b/interactive/examples/server/README.md index 6f7f77aa0..2ef563323 100644 --- a/interactive/examples/server/README.md +++ b/interactive/examples/server/README.md @@ -16,7 +16,7 @@ totally ordered across workers by a timely `Sequencer`. - **`programs/*.ddp`** — DDIR *programs*: dataflow definitions you `install`. The ones here are server-oriented (they use `import`/`export`); the programs in `../programs/` read positional `input`s instead, which you fill with - `feed` or `load` (see "Running a program in batch" below). + `feed` (see "Running a program in batch" below). - **`sessions/*.txt`** — *command scripts*: a stream of server commands (`install`/`feed`/`tick`/…) you hand to the server. You do **not** `install` a session; you run the server *on* it. @@ -44,7 +44,7 @@ the repo root with `--example`; adjust if you `cd interactive` first). |---|---| | `install [explain=[,debug]]` | parse + lower + install a program under ``; optionally after the explanation rewrite | | `feed [val=] [time=] [diff=]` | stage an input update | -| `load ` | bulk-load an input from a recipe (`random:…`, `iota:N`) or a file of integer rows, sharded across workers | +| `feed from ` | fill an input from a recipe (`random:…`, `iota:N`) or a file of integer rows, sharded across workers | | `tick [n]` | close `n` epochs (default 1), running to quiescence after each; reports the wall-clock time | | `bind ` / `unbind …` | feed a trace's changes back into an input at every tick (one-epoch-delayed feedback) | | `drop ` | evict a program (refused if a live program still imports its trace) | @@ -64,21 +64,21 @@ whole of the old single-program harness, so there is no separate binary: ``` install scc ../programs/scc.ddp -load scc 0 random:nodes=100000,edges=200000,churn=100 +feed scc 0 from random:nodes=100000,edges=200000,churn=100 tick # the initial load: reported as one epoch's time tick 100 # 100 epochs of 100 replaced edges each, timed together peek result exit ``` -`load` deals the rows across the workers (each feeds its shard, and the -exchange places every row on its key's owner). A `random:` recipe with +`feed … from` deals the rows across the workers (each feeds its shard, and +the exchange places every row on its key's owner). A `random:` recipe with `churn=C` keeps the input changing: every later `tick` retracts the next `C` rows of its window and adds `C` fresh ones, which is the standing-change regime programs are benchmarked under. A file source is one row per line of whitespace-separated integers, each becoming `(Tuple[ints] ; ())`; the -`aoc2023/run.sh` suite drives every AoC program this way. `feed` still works -alongside `load` for the small inputs (roots, queries). +`aoc2023/run.sh` suite drives every AoC program this way. A row `feed` still +serves the small inputs (roots, queries). `install … explain=` applies the explanation rewrite before optimization, treating every source as `arity` key fields with no value; the diff --git a/interactive/server/README.md b/interactive/server/README.md index 6d552e850..b962f9d65 100644 --- a/interactive/server/README.md +++ b/interactive/server/README.md @@ -42,8 +42,12 @@ Between commands, blank lines and `#` comment lines are skipped, so command scripts can be piped to stdin (see `demo/`). The useful commands are `load`, `drop`, `list`, `feed`, `bind`, `unbind`, -`peek`, `tail`, `stop`, `tick`, and `exit`. `load` accepts an inline -pipe-syntax program: +`peek`, `tail`, `stop`, `tick`, and `exit`. `feed from ` +fills an input from a source the server reads itself — a recipe such as +`random:nodes=N,edges=E,churn=C` or `iota:N`, or a file of integer rows — with +each worker taking its shard, so no row crosses the wire; a `churn=C` recipe +then replaces `C` rows on every `tick`. `load` accepts an inline pipe-syntax +program: load graph begin let edges = import "random:nodes=8,edges=12,seed=1,churn=1"; diff --git a/interactive/server/src/cmd.rs b/interactive/server/src/cmd.rs index df8a06ec6..cc6d9044e 100644 --- a/interactive/server/src/cmd.rs +++ b/interactive/server/src/cmd.rs @@ -77,6 +77,14 @@ pub enum Cmd { input: usize, updates: Vec, }, + /// Fill one program input from a source the server reads itself — a recipe + /// (`random:…`, `iota:N`) or a file of integer rows — each worker taking its + /// shard, so no row crosses the wire. `feed from `. + Source { + prog: String, + input: usize, + source: String, + }, /// Bind a trace's changes into `prog`'s positional `input`, delivered at /// each tick one epoch delayed — the write path for installed programs. Bind { @@ -203,6 +211,7 @@ pub fn prepare(command: Cmd) -> Result { input, updates, }, + Cmd::Source { prog, input, source } => ServerCommand::Load { prog, input, source }, Cmd::Bind { trace, prog, input } => ServerCommand::Bind { trace, prog, input }, Cmd::Unbind { trace, prog, input } => ServerCommand::Unbind { trace, prog, input }, Cmd::Query { .. } => { @@ -712,6 +721,17 @@ fn parse_cmd(cmd: &str, args: &[&str]) -> ParseOutcome { input, }; } + // Syntax: `feed from ` — the server sources the rows. + if let [prog, input, "from", source] = args { + return match input.parse() { + Ok(input) => ParseOutcome::Cmd(Cmd::Source { + prog: (*prog).to_string(), + input, + source: (*source).to_string(), + }), + Err(_) => ParseOutcome::Err(format!("feed: must be a number, got {input:?}")), + }; + } // Syntax: `feed [val=] [time=] [diff=]` // A ``/`` is a comma-separated integer row (`1,2` → tuple; // `_`/empty → unit) or a closed scalar term written without @@ -852,6 +872,21 @@ mod tests { out } + #[test] + fn feed_from_sources_rows_server_side() { + let mut p = LineParser::default(); + let out = feed_all(&mut p, &["r1 feed world 0 from iota:3\n"]); + match &out[..] { + [(reqid, Ok(Cmd::Source { prog, input, source }))] => { + assert_eq!(reqid.as_str(), "r1"); + assert_eq!((prog.as_str(), *input, source.as_str()), ("world", 0, "iota:3")); + } + other => panic!("unexpected parse: {other:?}"), + } + let out = feed_all(&mut p, &["r2 feed world x from iota:3\n"]); + assert!(matches!(&out[..], [(_, Err(_))]), "a bad input index must be rejected: {out:?}"); + } + #[test] fn simple_commands() { let mut p = LineParser::new(); diff --git a/interactive/server/src/loop_.rs b/interactive/server/src/loop_.rs index 231848dfe..c95897483 100644 --- a/interactive/server/src/loop_.rs +++ b/interactive/server/src/loop_.rs @@ -331,6 +331,10 @@ fn dispatch( server.epoch() ) }), + // Collective: every worker sources its own shard. + ServerCommand::Load { prog, input, source } => server + .load(worker, &prog, input, &source) + .map(|rows| format!("loaded {rows} rows into {prog:?} input {input}")), ServerCommand::Bind { trace, prog, input } => server .bind(worker, &trace, &prog, input) .map(|()| format!("bound {:?} -> {:?} input {}", trace, prog, input)), diff --git a/interactive/server/tests/multiworker.rs b/interactive/server/tests/multiworker.rs index 539ff94ea..3ff80f4eb 100644 --- a/interactive/server/tests/multiworker.rs +++ b/interactive/server/tests/multiworker.rs @@ -174,6 +174,27 @@ fn assert_backend(backend: &str) { Some("time=1 diff=1 key=Tuple([Int(8)]) val=Tuple([Int(10)])") ); + // Server-side sourcing: every worker feeds its shard of the recipe, so the + // union is the source exactly once however many workers there are. Its own + // program, because a recipe's rows carry a unit value and `world`'s carry + // an integer, and one input holds one shape. + request(&mut writer, &mut reader, "r9", "r9 stop r6\n"); + request( + &mut writer, + &mut reader, + "r10", + "r10 load counted begin\nlet rows = input 0;\nexport \"counted\" = rows;\nr10 end-load\n", + ); + request(&mut writer, &mut reader, "r11", "r11 feed counted 0 from iota:5\n"); + request(&mut writer, &mut reader, "r12", "r12 tick\n"); + let rows = request(&mut writer, &mut reader, "r13", "r13 peek counted\n"); + assert_eq!( + rows, + (0..5) + .map(|n| format!("diff=1 key=Tuple([Int({n})]) val=Tuple([])")) + .collect::>() + ); + drop(reader); drop(writer); server.stop(); diff --git a/interactive/src/server.rs b/interactive/src/server.rs index f8d0f5461..cd84c3fce 100644 --- a/interactive/src/server.rs +++ b/interactive/src/server.rs @@ -245,9 +245,9 @@ pub enum Command { input: usize, updates: Vec, }, - /// Bulk-load `source` — a recipe name or a file path — into positional - /// `input` of `prog` at the current epoch. Collective: every worker feeds - /// its own shard of the rows (see [`Server::load`]). + /// Fill positional `input` of `prog` from `source` — a recipe name or a + /// file path — at the current epoch. Collective: every worker feeds its + /// own shard of the rows (see [`Server::load`]). Load { prog: String, input: usize, From e4088232d86956466874521c074d974baa45251e Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Sat, 5 Sep 2026 17:19:09 -0400 Subject: [PATCH 4/5] =?UTF-8?q?DDIR:=20one=20server=20binary=20=E2=80=94?= =?UTF-8?q?=20the=20live=20crate=20takes=20over=20from=20the=20example=20d?= =?UTF-8?q?river?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two front ends drove `interactive::server`, both building a binary named `ddir_server`: the stdin/script example driver (a Sequencer and a 1 ms poll) and the live `ddir-server` crate (stdin, TCP and WebSocket sessions, request ids, `tail`; parks between commands). The crate is the superset, so the driver's four remaining abilities move into it and the driver is deleted: - `load from [explain=[,debug]] [name=binding ...]` installs a program file (`.ddp` pipe syntax, else applicative), read on the session thread like an inline body. `explain=` applies the explanation rewrite before optimization, replacing the reserved `--explain`; the query input is the one after the program's own, the demand sets its exports. - `peek [key]` filters to one key (the dispatch already could). - `tick [n]` reports the wall-clock time alongside the epoch. - The crate installs the same global allocator the driver ran on. The session scripts, the AoC suite and the READMEs pipe into `cargo run -p ddir-server` (`DDIR_WORKERS`, `DDIR_BACKEND`); every session replays, AoC is 33/33 on both backends, and end of input stops the server. `install` is spelled `load` throughout, the crate's word. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QDsLC46QQW9aBrksaWad6T --- interactive/Cargo.toml | 1 - interactive/README.md | 11 +- interactive/examples/aoc2023/README.md | 7 +- interactive/examples/aoc2023/run.sh | 12 +- interactive/examples/ddir_server.rs | 380 ------------------ interactive/examples/server/README.md | 44 +- .../examples/server/sessions/arrange_idem.txt | 2 +- .../examples/server/sessions/clock.txt | 2 +- .../examples/server/sessions/derived.txt | 2 +- interactive/examples/server/sessions/drop.txt | 6 +- .../examples/server/sessions/generated.txt | 4 +- interactive/examples/server/sessions/peek.txt | 4 +- .../examples/server/sessions/shared_trace.txt | 4 +- .../examples/server/sessions/values.txt | 2 +- interactive/server/Cargo.toml | 1 + interactive/server/README.md | 20 +- interactive/server/src/cmd.rs | 191 ++++++--- interactive/server/src/loop_.rs | 6 +- interactive/server/src/main.rs | 6 + 19 files changed, 211 insertions(+), 494 deletions(-) delete mode 100644 interactive/examples/ddir_server.rs diff --git a/interactive/Cargo.toml b/interactive/Cargo.toml index adb8ea20a..c598919ff 100644 --- a/interactive/Cargo.toml +++ b/interactive/Cargo.toml @@ -16,7 +16,6 @@ columnar = { workspace = true } # The columnar kernels for the interpreted backend, pinned by git rev. corgi = { git = "https://github.com/frankmcsherry/WIP", rev = "de0f2ac91d31ac63641035e5a5bb1b5640fe9424", features = ["serde"] } differential-dataflow = { workspace = true } -mimalloc = "0.1.48" serde = { version = "1.0", features = ["derive"] } smallvec = "1.15.1" timely = { workspace = true } diff --git a/interactive/README.md b/interactive/README.md index 2613c7f83..b29d57025 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -73,17 +73,18 @@ The flow moves through four steps: 4. The `examples/` directory contains back-ends that execute programs. The `examples/programs/` directory contains example programs, intentionally simple at the moment. -The one executable is the server (`examples/ddir_server.rs`, documented in `examples/server/README.md`); -a program runs by installing it, loading its inputs, and closing epochs. For example, on a random -graph of 100 nodes and 200 edges, 10 of which change each epoch, for 100 epochs: +The one executable is the server (the `ddir-server` crate in `server/`, documented in +`server/README.md`; the session scripts in `examples/server/` show it at work); a program runs by +loading it, feeding its inputs, and closing epochs. For example, on a random graph of 100 nodes and +200 edges, 10 of which change each epoch, for 100 epochs, on four workers of the Corgi backend: ``` cd interactive -printf 'install reach examples/programs/reach.ddp +printf 'load reach from examples/programs/reach.ddp feed reach 0 from random:nodes=100,edges=200,churn=10 feed reach 1 0 tick 100 exit -' | cargo run --release --example ddir_server -- --backend=corgi -w4 +' | DDIR_BACKEND=corgi DDIR_WORKERS=4 cargo run --release -p ddir-server ``` More generally, you can run diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md index b05e4f780..26d9d32ab 100644 --- a/interactive/examples/aoc2023/README.md +++ b/interactive/examples/aoc2023/README.md @@ -17,13 +17,14 @@ mixed-arity inputs. ## Run - cargo build --release --example ddir_server + cargo build --release -p ddir-server ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch ./run.sh corgi # the same through the Corgi columnar backend `run.sh` first runs `transcribe.py` (python3) to regenerate `gen/`, then -runs each part as one server session — `install`, `feed … from` the fact -file into input 0, `tick` — and reads the answer off the `[partN]` inspect line. +runs each part as one server session piped into `ddir_server` — `load` the +program from its file, `feed … from` the fact file into input 0, `tick` — and +reads the answer off the `[partN]` inspect line. Both backends pass all 33 parts. Corgi needs day05's arity-padded inputs (`run.sh corgi` transcribes with `--pad`). diff --git a/interactive/examples/aoc2023/run.sh b/interactive/examples/aoc2023/run.sh index 3c3587339..bf253f55c 100755 --- a/interactive/examples/aoc2023/run.sh +++ b/interactive/examples/aoc2023/run.sh @@ -1,10 +1,10 @@ #!/bin/sh # Run every AoC 2023 program through the DDIR server and check the answers. # Usage: ./run.sh [vec|corgi] [path/to/ddir_server] -# (build with: cargo build --release --example ddir_server) +# (build with: cargo build --release -p ddir-server) cd "$(dirname "$0")" || exit 1 BACKEND=${1:-vec} -SERVER=${2:-../../../target/release/examples/ddir_server} +SERVER=${2:-../../../target/release/ddir_server} PAD=; [ "$BACKEND" = corgi ] && PAD=--pad # corgi needs day05's uniform-arity copies python3 transcribe.py $PAD || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files fail=0 @@ -14,10 +14,10 @@ while read -r day part expected; do inp=gen/$dir/input.txt [ -f "gen/$dir/input$part.txt" ] && inp=gen/$dir/input$part.txt # day05/day15: per-part inputs [ -n "$PAD" ] && [ -f "gen/$dir/input${part}p.txt" ] && inp=gen/$dir/input${part}p.txt - # One session per part: install the program, bulk-load its input, close the - # epoch. The answer is the `Int` on the `[partN]` inspect line. - got=$(printf 'install p %s\nfeed p 0 from %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ - | "$SERVER" --backend="$BACKEND" 2>&1 \ + # One session per part: load the program, feed its input from the fact + # file, close the epoch. The answer is the `Int` on the `[partN]` inspect line. + got=$(printf 'load p from %s\nfeed p 0 from %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \ + | DDIR_BACKEND="$BACKEND" "$SERVER" 2>&1 \ | sed -n "s/.*\\[part$part\\].*Int(\\(-\\{0,1\\}[0-9]*\\)).*/\\1/p") if [ "$got" = "$expected" ]; then echo "day$day part$part: ok ($got)" diff --git a/interactive/examples/ddir_server.rs b/interactive/examples/ddir_server.rs deleted file mode 100644 index 94504bb2f..000000000 --- a/interactive/examples/ddir_server.rs +++ /dev/null @@ -1,380 +0,0 @@ -//! A live DDIR server: install interpreted dataflows into one running timely -//! computation and let them share results by name. Multi-worker capable. -//! -//! Usage (see `examples/server/` for session scripts and a README): -//! cargo run --release --example ddir_server # stdin, 1 worker -//! cargo run --release --example ddir_server --