diff --git a/interactive/Cargo.toml b/interactive/Cargo.toml
index adb8ea20a..c598919ff 100644
--- a/interactive/Cargo.toml
+++ b/interactive/Cargo.toml
@@ -16,7 +16,6 @@ columnar = { workspace = true }
# The columnar kernels for the interpreted backend, pinned by git rev.
corgi = { git = "https://github.com/frankmcsherry/WIP", rev = "de0f2ac91d31ac63641035e5a5bb1b5640fe9424", features = ["serde"] }
differential-dataflow = { workspace = true }
-mimalloc = "0.1.48"
serde = { version = "1.0", features = ["derive"] }
smallvec = "1.15.1"
timely = { workspace = true }
diff --git a/interactive/README.md b/interactive/README.md
index c72c07436..b29d57025 100644
--- a/interactive/README.md
+++ b/interactive/README.md
@@ -73,9 +73,18 @@ The flow moves through four steps:
4. The `examples/` directory contains back-ends that execute programs.
The `examples/programs/` directory contains example programs, intentionally simple at the moment.
-You can run any of them with one of the example harnesses, for example
+The one executable is the server (the `ddir-server` crate in `server/`, documented in
+`server/README.md`; the session scripts in `examples/server/` show it at work); a program runs by
+loading it, feeding its inputs, and closing epochs. For example, on a random graph of 100 nodes and
+200 edges, 10 of which change each epoch, for 100 epochs, on four workers of the Corgi backend:
```
-cargo run --release --example ddir_vec -- ./examples/programs/reach.ddp 2 100 200 1 100
+cd interactive
+printf 'load reach from examples/programs/reach.ddp
+feed reach 0 from random:nodes=100,edges=200,churn=10
+feed reach 1 0
+tick 100
+exit
+' | DDIR_BACKEND=corgi DDIR_WORKERS=4 cargo run --release -p ddir-server
```
More generally, you can run
diff --git a/interactive/examples/aoc2023/README.md b/interactive/examples/aoc2023/README.md
index 652f26c91..26d9d32ab 100644
--- a/interactive/examples/aoc2023/README.md
+++ b/interactive/examples/aoc2023/README.md
@@ -17,17 +17,17 @@ mixed-arity inputs.
## Run
- cargo build --release --example ddir
- ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch
+ cargo build --release -p ddir-server
+ ./run.sh # every part on the vec backend vs expected.txt; nonzero exit on any mismatch
+ ./run.sh corgi # the same through the Corgi columnar backend
`run.sh` first runs `transcribe.py` (python3) to regenerate `gen/`, then
-runs each part as `EDGES_FILE=gen/ ddir --backend=vec
-dayNN/partN.ddp 10 0 1 0`; the answer is the `Int` on the
-`[partN]` inspect line.
+runs each part as one server session piped into `ddir_server` — `load` the
+program from its file, `feed … from` the fact file into input 0, `tick` — and
+reads the answer off the `[partN]` inspect line.
-`run.sh` asserts the **vec** backend only; corgi currently disagrees or
-crashes on 4 of these parts (known open issues, minimized separately) and
-is not asserted yet.
+Both backends pass all 33 parts. Corgi needs day05's arity-padded inputs
+(`run.sh corgi` transcribes with `--pad`).
## Verdicts
diff --git a/interactive/examples/aoc2023/run.sh b/interactive/examples/aoc2023/run.sh
index ade1193b1..bf253f55c 100755
--- a/interactive/examples/aoc2023/run.sh
+++ b/interactive/examples/aoc2023/run.sh
@@ -1,17 +1,23 @@
#!/bin/sh
-# Run every AoC 2023 program on the vec backend and check the answers.
-# Usage: ./run.sh [path/to/ddir] (build with: cargo build --release --example ddir)
+# Run every AoC 2023 program through the DDIR server and check the answers.
+# Usage: ./run.sh [vec|corgi] [path/to/ddir_server]
+# (build with: cargo build --release -p ddir-server)
cd "$(dirname "$0")" || exit 1
-DDIR=${1:-../../../target/release/examples/ddir}
-python3 transcribe.py || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files
+BACKEND=${1:-vec}
+SERVER=${2:-../../../target/release/ddir_server}
+PAD=; [ "$BACKEND" = corgi ] && PAD=--pad # corgi needs day05's uniform-arity copies
+python3 transcribe.py $PAD || exit 1 # dense dayNN/input.txt -> gen/dayNN/ fact files
fail=0
while read -r day part expected; do
case "$day" in ''|'#'*) continue;; esac
dir=day$day
inp=gen/$dir/input.txt
[ -f "gen/$dir/input$part.txt" ] && inp=gen/$dir/input$part.txt # day05/day15: per-part inputs
- arity=$(head -1 "$inp" | awk '{print NF}')
- got=$(EDGES_FILE=$inp "$DDIR" --backend=vec "$dir/part$part.ddp" "$arity" 10 0 1 0 2>&1 \
+ [ -n "$PAD" ] && [ -f "gen/$dir/input${part}p.txt" ] && inp=gen/$dir/input${part}p.txt
+ # One session per part: load the program, feed its input from the fact
+ # file, close the epoch. The answer is the `Int` on the `[partN]` inspect line.
+ got=$(printf 'load p from %s\nfeed p 0 from %s\ntick\nexit\n' "$dir/part$part.ddp" "$inp" \
+ | DDIR_BACKEND="$BACKEND" "$SERVER" 2>&1 \
| sed -n "s/.*\\[part$part\\].*Int(\\(-\\{0,1\\}[0-9]*\\)).*/\\1/p")
if [ "$got" = "$expected" ]; then
echo "day$day part$part: ok ($got)"
diff --git a/interactive/examples/ddir.rs b/interactive/examples/ddir.rs
deleted file mode 100644
index d262f1d00..000000000
--- a/interactive/examples/ddir.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-//! The DDIR driver: parse, lower, render on the chosen backend, execute.
-//!
-//! Usage: `ddir [flags] [batch] [rounds] [timely args]`
-//!
-//! Inputs: with `EDGES_FILE` set, rows come from that file — one row per line,
-//! whitespace-separated `i64` fields, assigned round-robin to the program's
-//! inputs (`line % n_inputs`). Otherwise rows are synthesized by `gen_row`.
-//!
-//! Flags:
-//! - `--explain`: apply the explanation rewrite after lowering; the last input
-//! becomes the query input (the demand envelope `(key ; val ++ q)`).
-//! - `--query=K:V[,q]`: seed the query input with one row (requires --explain).
-//! - `--debug-demand`: tap every demand collection with an Inspect.
-//! - `--diag`: serve timely/DD diagnostics on port 51371.
-//! - `--backend=vec|corgi`: rendering substrate (default `vec`). The corgi
-//! backend exchanges by key hash, so both take `-w`.
-//! - `--sync=K`: await completion only every K rounds (default 1), letting K
-//! timestamps retire with whatever inter-timestamp concurrency the system
-//! finds — the open(er)-loop regime DD adapts into under load.
-
-use mimalloc::MiMalloc;
-
-#[global_allocator]
-static GLOBAL: MiMalloc = MiMalloc;
-
-use differential_dataflow::dynamic::pointstamp::PointStamp;
-use differential_dataflow::input::Input;
-
-use interactive::parse;
-use interactive::lower;
-use interactive::scope_ir as st;
-use interactive::ir::{Diff, Value};
-use interactive::backend::vec::{render_tree, Row};
-use interactive::backend::corgi::render_tree_rows;
-
-#[derive(Clone, Default)]
-struct Flags {
- explain: bool,
- query: Option,
- debug_demand: bool,
- diag: bool,
- corgi: bool,
- sync: u64,
-}
-
-fn run(
- name: &str,
- stmts: Vec,
- n_inputs: usize,
- nodes: u64,
- edges: u64,
- arity: usize,
- batch: u64,
- rounds: Option,
- flags: Flags,
- timely_args: Vec,
-) {
- let explain = flags.explain;
- let mut tree = lower::lower_tree(stmts);
- // --explain: rewrite for self-explanation before optimization (the rules
- // assume single-op Linears). Sources are the root's positional inputs; the
- // query arrives as one extra input appended after them.
- if explain {
- let source_shapes: Vec<(usize, usize)> = tree.root.imports.iter().map(|imp| match &imp.from {
- st::Source::Input(_) => (arity, 0usize),
- other => panic!("ddir --explain: unsupported source {:?}", other),
- }).collect();
- let shape = interactive::explain::export_shape(&tree, &source_shapes);
- eprintln!("explain: first export shape (k={}, v={}); query is (key[{}] ; val[{}] ++ q)", shape.0, shape.1, shape.0, shape.1);
- let options = interactive::explain::Options { debug_inspects: flags.debug_demand };
- tree = interactive::explain::explain_with(&tree, &source_shapes, options);
- }
- let ops_before = tree.op_count();
- tree.optimize();
- let tree_export_idx = tree.root.exports.iter().position(|e| e.name == "result").unwrap_or(0);
- println!("{}: {} ops before optimize, {} after; driving export {:?}",
- name, ops_before, tree.op_count(), tree.root.exports[tree_export_idx].name);
- let name = name.to_string();
- let edges_file = std::env::var("EDGES_FILE").ok();
- let total_inputs = if explain { n_inputs + 1 } else { n_inputs };
- let query_input_idx = if explain { Some(n_inputs) } else { None };
-
- timely::execute_from_args(timely_args.into_iter(), move |worker| {
-
- // --diag registers timely/DD logging and serves the diagnostics
- // WebSocket on worker 0 (port 51371) — see diagnostics/README.md.
- let _diag = if flags.diag {
- let state = diagnostics::logging::register(worker, false);
- if worker.index() == 0 {
- Some(diagnostics::server::Server::start(51371, state.sink))
- } else {
- drop(state.sink);
- None
- }
- } else { None };
-
- let (mut inputs, probe) = worker.dataflow::(|scope| {
- let mut handles = Vec::new();
- let mut collections = Vec::new();
- for _ in 0..total_inputs {
- let (h, c) = scope.new_collection::<(Row, Row), Diff>();
- handles.push(h); collections.push(c);
- }
- let mut probe = timely::dataflow::ProbeHandle::new();
- let output = scope.iterative::, _, _>(|inner| {
- let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect();
- let root_imports: Vec<_> = tree.root.imports.iter().map(|imp| match &imp.from {
- st::Source::Input(n) => entered[*n].clone(),
- st::Source::Trace(name) => panic!("ddir: Import {:?} not supported in this harness (no trace registry).", name),
- st::Source::Parent(_) => unreachable!("root scope cannot import from a parent"),
- }).collect();
- let exports = if flags.corgi {
- render_tree_rows(&tree.root, inner, 0, root_imports)
- } else {
- render_tree(&tree.root, inner, 0, root_imports)
- };
- exports[tree_export_idx].clone().leave(scope)
- });
- output.probe_with(&mut probe);
- (handles, probe)
- });
-
- let index = worker.index();
- let peers = worker.peers();
-
- let timer = std::time::Instant::now();
- let timer_load = std::time::Instant::now();
- // With `EDGES_FILE` set, rows come from the file (one row per line,
- // whitespace-separated i64 fields), assigned round-robin to inputs.
- // Otherwise `gen_row` synthesizes `edges` rows.
- if let Some(path) = &edges_file {
- let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("Cannot read {}: {}", path, e));
- for (e, line) in text.lines().filter(|l| !l.trim().is_empty()).enumerate() {
- if e % peers == index {
- let input_idx = e % n_inputs;
- let fields: Vec = line.split_whitespace().map(|t| Value::Int(t.parse::().unwrap())).collect();
- inputs[input_idx].update((Value::Tuple(fields), Value::unit()), 1);
- }
- }
- } else {
- for e in 0..edges {
- if (e as usize) % peers == index {
- let input_idx = (e as usize) % n_inputs;
- inputs[input_idx].update(interactive::gen_row(e, nodes, arity), 1);
- }
- }
- }
- // Seed the query input (worker 0 only) from --query="k:v[,q]". The row
- // is the flat demand envelope `(key[k] ; val[v] ++ [q])` — at depth 0
- // there is no chain, so the value is just V's fields then the query id.
- if let Some(q_idx) = query_input_idx {
- if index == 0 {
- if let Some(qstr) = flags.query.clone() {
- let parse_fields = |s: &str| -> Vec {
- if s.is_empty() { vec![] } else { s.split(',').map(|t| Value::Int(t.trim().parse::().unwrap())).collect() }
- };
- let (k_str, vq_str) = qstr.split_once(':').unwrap_or((qstr.as_str(), ""));
- let q_key = Value::Tuple(parse_fields(k_str));
- let mut vq = parse_fields(vq_str);
- if vq.is_empty() { vq.push(Value::Int(0)); } // a bare query id
- let q_val = Value::Tuple(vq);
- eprintln!("seeding query: key={:?} val_with_q={:?}", q_key, q_val);
- inputs[q_idx].update((q_key, q_val), 1);
- }
- }
- }
- for i in inputs.iter_mut() { i.advance_to(1); i.flush(); }
- while probe.less_than(&1u64) { worker.step(); }
- println!("worker {}: {} loaded ({} edges, total {:.2?}, load {:.2?})", index, name, edges, timer.elapsed(), timer_load.elapsed());
-
- let mut cursor = 0u64;
- let mut round = 0u64;
- let limit = rounds.unwrap_or(u64::MAX);
- while round < limit {
- let timer_round = std::time::Instant::now();
- let time = (round + 2) as u64;
- for _ in 0..batch {
- let remove_idx = cursor;
- let add_idx = edges + cursor;
- if (remove_idx as usize) % peers == index {
- let input_idx = (remove_idx as usize) % n_inputs;
- inputs[input_idx].update(interactive::gen_row(remove_idx, nodes, arity), -1);
- }
- if (add_idx as usize) % peers == index {
- let input_idx = (add_idx as usize) % n_inputs;
- inputs[input_idx].update(interactive::gen_row(add_idx, nodes, arity), 1);
- }
- cursor += 1;
- }
- for i in inputs.iter_mut() { i.advance_to(time); i.flush(); }
- let sync = flags.sync.max(1);
- if (round + 1) % sync == 0 || round + 1 == limit {
- while probe.less_than(&time) { worker.step(); }
- }
-
- round += 1;
- if round % 100 == 0 {
- println!("worker {}: {} round {} (total {:.2?}, round {:.2?})", index, name, round, timer.elapsed(), timer_round.elapsed());
- }
- }
- println!("worker {}: {} done ({} rounds, batch {}, total {:.2?})", index, name, round, batch, timer.elapsed());
- }).unwrap();
-}
-
-fn main() {
- // Strip our flags; everything else stays positional (and the tail is
- // forwarded to timely).
- let raw_args: Vec = std::env::args().collect();
- let (flags, args): (Flags, Vec) = {
- let mut it = raw_args.into_iter();
- let prog = it.next().unwrap();
- let mut flags = Flags::default();
- let mut rest: Vec = Vec::new();
- for a in it {
- if a == "--explain" { flags.explain = true; }
- else if let Some(q) = a.strip_prefix("--query=") { flags.query = Some(q.to_string()); }
- else if a == "--debug-demand" { flags.debug_demand = true; }
- else if a == "--diag" { flags.diag = true; }
- else if let Some(k) = a.strip_prefix("--sync=") { flags.sync = k.parse().expect("--sync=K"); }
- else if let Some(b) = a.strip_prefix("--backend=") {
- flags.corgi = match b { "corgi" => true, "vec" => false, other => panic!("unknown backend {other:?} (vec|corgi)") };
- }
- else { rest.push(a); }
- }
- let mut out = vec![prog]; out.extend(rest);
- (flags, out)
- };
- let program = args.get(1).cloned().unwrap_or_else(|| { std::process::exit(0); });
- let arity: usize = args.get(2).cloned().unwrap_or("2".into()).parse().unwrap();
- let nodes: u64 = args.get(3).cloned().unwrap_or("10".into()).parse().unwrap();
- let edges: u64 = args.get(4).cloned().unwrap_or_else(|| (2 * nodes).to_string()).parse().unwrap();
- let batch: u64 = args.get(5).cloned().unwrap_or("1".into()).parse().unwrap();
- let rounds: Option = args.get(6).map(|s| s.parse().unwrap());
-
- let source = interactive::load_program(&program);
- let stmts = if program.ends_with(".ddp") {
- parse::pipe::parse(&source)
- } else {
- parse::applicative::parse(&source)
- };
- let (n_inputs, imports) = interactive::survey_sources(&stmts);
- if !imports.is_empty() {
- panic!("ddir: program references imports {:?} but this harness has no trace registry.", imports);
- }
- let name = std::path::Path::new(&program).file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or(program.clone());
- let timely_args: Vec = args.iter().skip(4).cloned().collect();
- run(&name, stmts, n_inputs, nodes, edges, arity, batch, rounds, flags, timely_args);
-}
diff --git a/interactive/examples/ddir_server.rs b/interactive/examples/ddir_server.rs
deleted file mode 100644
index 67bf91d89..000000000
--- a/interactive/examples/ddir_server.rs
+++ /dev/null
@@ -1,323 +0,0 @@
-//! A live DDIR server: install interpreted dataflows into one running timely
-//! computation and let them share results by name. Multi-worker capable.
-//!
-//! Usage (see `examples/server/` for session scripts and a README):
-//! cargo run --release --example ddir_server # stdin, 1 worker
-//! cargo run --release --example ddir_server --