diff --git a/AGENTS.md b/AGENTS.md index 8fc7d20b..1c6fc398 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -487,6 +487,14 @@ full cold-read/resource acceptance gates listed in `docs/spec/README.md`. and candidate external-boundary proof remain separate obligations; local loose objects and retired download membership cannot justify retirement. See the cost and remaining-evidence rows in the linked docs. +- **D50 (2026-10-26): The JetStream sink's cursor lives in its own stream.** `walgit nats` publishes the + `ref` events of D32 into NATS JetStream and resumes from the last message on each repository's subject + (`Walgit-Seq` header, inclusive, then drop what the stream already holds). That cursor is the sink's state, + not walgit's: Principle I holds because wiping every walgit instance loses nothing, and wiping the stream + republishes the WAL window with older history backfilled from the WAL. The event model is the `walgit-events` + crate shared with the webhook bridge, whose bucket cursor (`events/cursor.json`) is unchanged. Wake-ups + arrive over NATS and the periodic sweep stays the backstop; the service API is the NATS micro protocol, no HTTP. + ## 5. Working rules - **No backwards compatibility (pre-1.0, banner at top):** change the shape and delete the old one in the same diff --git a/Cargo.lock b/Cargo.lock index 93065712..219881a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,42 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-nats" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f92759f418774b78c36ef07885849150ce950c67d9bcdeff8a9c253f57d9a1" +dependencies = [ + "base64", + "bytes", + "futures-util", + "memchr", + "nkeys", + "nuid", + "pin-project", + "portable-atomic", + "rand 0.10.2", + "regex", + "ring", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki 0.103.14", + "serde", + "serde_json", + "serde_nanos", + "serde_repr", + "thiserror", + "time", + "tokio", + "tokio-rustls 0.26.4", + "tokio-stream", + "tokio-util", + "tokio-websockets", + "tracing", + "tryhard", + "url", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -1437,6 +1473,7 @@ dependencies = [ "ed25519", "serde", "sha2 0.10.9", + "signature", "subtle", "zeroize", ] @@ -3732,6 +3769,21 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "nkeys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879011babc47a1c7fdf5a935ae3cfe94f34645ca0cac1c7f6424b36fc743d1bf" +dependencies = [ + "data-encoding", + "ed25519", + "ed25519-dalek", + "getrandom 0.2.17", + "log", + "rand 0.8.7", + "signatory", +] + [[package]] name = "nom" version = "7.1.3" @@ -3757,6 +3809,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nuid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83" +dependencies = [ + "rand 0.8.7", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -4908,6 +4969,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_nanos" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93142f0367a4cc53ae0fead1bcda39e85beccfad3dcd717656cacab94b12985" +dependencies = [ + "serde", +] + [[package]] name = "serde_path_to_error" version = "0.1.20" @@ -4919,6 +4989,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -5005,6 +5086,12 @@ dependencies = [ "sha1 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5068,6 +5155,18 @@ dependencies = [ "libc", ] +[[package]] +name = "signatory" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" +dependencies = [ + "pkcs8", + "rand_core 0.6.4", + "signature", + "zeroize", +] + [[package]] name = "signature" version = "2.2.0" @@ -5429,6 +5528,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52efb639344a7c6adb8e62c6f3d2c19c001ff1b79a5041ba1c6ed42e19c6aa5" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-sink", + "http 1.5.0", + "httparse", + "rand 0.10.2", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -5656,6 +5776,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tryhard" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe58ebd5edd976e0fe0f8a14d2a04b7c81ef153ea9a54eebc42e67c2c23b4e5" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "typenum" version = "1.20.1" @@ -5798,6 +5928,7 @@ dependencies = [ "tracing-subscriber", "walgit-config", "walgit-git", + "walgit-nats", "walgit-proto", "walgit-server", "walgit-store", @@ -5818,6 +5949,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "walgit-events" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "walgit-git", + "walgit-proto", +] + [[package]] name = "walgit-git" version = "0.1.0" @@ -5857,6 +5998,30 @@ dependencies = [ "walgit-proto", ] +[[package]] +name = "walgit-nats" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "bytes", + "futures", + "nkeys", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tracing", + "walgit-config", + "walgit-events", + "walgit-git", + "walgit-nats", + "walgit-proto", + "walgit-store", + "walgit-wal", +] + [[package]] name = "walgit-proto" version = "0.1.0" @@ -5876,6 +6041,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "async-nats", "async-trait", "axum", "base64", @@ -5928,7 +6094,9 @@ dependencies = [ "tracing-subscriber", "uuid", "walgit-config", + "walgit-events", "walgit-git", + "walgit-nats", "walgit-proto", "walgit-store", "walgit-wal", diff --git a/Cargo.toml b/Cargo.toml index 2a1c3be7..ca3f2483 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,9 @@ members = [ "crates/walgit-config", "crates/walgit-git", "crates/walgit-wal", + "crates/walgit-events", "crates/walgit-server", + "crates/walgit-nats", "crates/walgit-cli", ] @@ -65,7 +67,9 @@ walgit-store = { path = "crates/walgit-store" } walgit-config = { path = "crates/walgit-config" } walgit-git = { path = "crates/walgit-git" } walgit-wal = { path = "crates/walgit-wal" } +walgit-events = { path = "crates/walgit-events" } walgit-server = { path = "crates/walgit-server" } +walgit-nats = { path = "crates/walgit-nats" } anyhow = "1" thiserror = "2" @@ -136,9 +140,11 @@ parking_lot = "0.12" moka = { version = "0.12", features = ["sync"] } mimalloc = "0.1" tempfile = "3" +nkeys = "0.4" flate2 = "1" async-compression = { version = "0.4", features = ["tokio", "gzip"] } pin-project-lite = "0.2" +async-nats = "0.51" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "json"] } base64 = "0.22" jsonwebtoken = { version = "11.0.0", features = ["rust_crypto"] } diff --git a/crates/walgit-cli/Cargo.toml b/crates/walgit-cli/Cargo.toml index f7231266..a65684d4 100644 --- a/crates/walgit-cli/Cargo.toml +++ b/crates/walgit-cli/Cargo.toml @@ -24,6 +24,7 @@ walgit-config.workspace = true walgit-git.workspace = true walgit-wal.workspace = true walgit-server.workspace = true +walgit-nats.workspace = true gix-hash.workspace = true anyhow.workspace = true tokio.workspace = true diff --git a/crates/walgit-cli/src/bin/walgit.rs b/crates/walgit-cli/src/bin/walgit.rs index e4acf99b..08e900ff 100644 --- a/crates/walgit-cli/src/bin/walgit.rs +++ b/crates/walgit-cli/src/bin/walgit.rs @@ -1,4 +1,4 @@ -//! `walgit` — the full CLI (serve | compact | repo | wal | synth | import | mirror | config). +//! `walgit` — the full CLI (serve | compact | repo | wal | nats | synth | import | mirror | config). fn main() -> anyhow::Result<()> { walgit_cli::main() } diff --git a/crates/walgit-cli/src/lib.rs b/crates/walgit-cli/src/lib.rs index c7b49b35..d6f55c24 100644 --- a/crates/walgit-cli/src/lib.rs +++ b/crates/walgit-cli/src/lib.rs @@ -109,6 +109,8 @@ enum Command { #[command(subcommand)] action: RepoAction, }, + /// Publish the WAL's ref events into NATS `JetStream` (`[events.nats]`, `docs/EVENTS.md`). + Nats, /// WAL inspection and rewind. Wal { #[command(subcommand)] @@ -471,6 +473,7 @@ async fn dispatch(command: Command, cfg: Config) -> Result<()> { } => compact::run(repo, all, once, base, &cfg).await, Command::Repo { action } => repo::run(action, &cfg).await, Command::Wal { action } => wal_cmd::run(action, &cfg).await, + Command::Nats => walgit_nats::run(cfg.clone()).await, Command::Mirror { from, to, diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 6a156eb3..b17cf8ad 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -637,6 +637,8 @@ pub struct EventsConfig { /// warns when a sweep finds unpublished entries. `0` = off. #[serde(with = "humantime_serde")] pub sweep_interval: Duration, + /// The `JetStream` publisher (`walgit nats`, `docs/EVENTS.md`). Read only by that subcommand. + pub nats: NatsEventsConfig, } impl Default for EventsConfig { @@ -645,10 +647,95 @@ impl Default for EventsConfig { webhook_url: None, webhook_secret: None, sweep_interval: Duration::from_mins(5), + nats: NatsEventsConfig::default(), } } } +/// Where a cold repository (no message on its subject yet) starts publishing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum NatsStart { + /// The oldest readable seq (`min_seq - 1`), as the webhook bridge does. + #[default] + Oldest, + /// Nothing older than the head the publisher first sees. + Head, +} + +/// `[events.nats]`: `walgit nats` publishes the bridge's `ref` events into `JetStream`. +/// The resume cursor is the last message on each repository's subject, so there is no cursor +/// object in the bucket. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct NatsEventsConfig { + /// Server URLs (`nats://host:4222`); the client connects to the first reachable one. + pub urls: Vec, + /// The `JetStream` stream holding the events (subjects `.ref.>`). + pub stream: String, + /// Create the stream when missing (file storage, `duplicate_window`). An existing stream is used as is. + pub ensure_stream: bool, + /// Deduplication window of a created stream (`Nats-Msg-Id`). + #[serde(with = "humantime_serde")] + pub duplicate_window: Duration, + /// Events go to `.ref..`. + pub subject_prefix: String, + /// Core subject bucket notifications are published to (S3 event JSON, `{"key"}` or `{"repo"}`). + pub notify_subject: String, + /// Queue group of the notify subscription and the service endpoints: instances share the work. + pub queue_group: String, + pub start: NatsStart, +} + +impl Default for NatsEventsConfig { + fn default() -> Self { + NatsEventsConfig { + urls: vec!["nats://127.0.0.1:4222".into()], + stream: "WALGIT_EVENTS".into(), + ensure_stream: true, + duplicate_window: Duration::from_hours(1), + subject_prefix: "walgit".into(), + notify_subject: "walgit.notify".into(), + queue_group: "walgit-events".into(), + start: NatsStart::Oldest, + } + } +} + +impl NatsEventsConfig { + fn validate(&self) -> Result<()> { + anyhow::ensure!(!self.urls.is_empty(), "events.nats.urls must not be empty"); + anyhow::ensure!( + self.urls.iter().all(|u| !u.trim().is_empty()), + "events.nats.urls must not contain empty entries" + ); + let plain = |s: &str| { + !s.is_empty() + && !s + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '*' | '>')) + }; + let dotted = |s: &str| plain(s) && s.split('.').all(|t| !t.is_empty()); + anyhow::ensure!( + plain(&self.stream) && !self.stream.contains(['.', '/', '\\']), + "events.nats.stream must be a stream name (no '.', '/', '\\', '*', '>' or whitespace)" + ); + anyhow::ensure!( + dotted(&self.subject_prefix), + "events.nats.subject_prefix must be dot-separated tokens (no '*', '>' or whitespace)" + ); + anyhow::ensure!( + dotted(&self.notify_subject), + "events.nats.notify_subject must be a literal subject (no '*', '>' or whitespace)" + ); + anyhow::ensure!( + plain(&self.queue_group), + "events.nats.queue_group must not be empty or contain '*', '>' or whitespace" + ); + Ok(()) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct TelemetryConfig { @@ -1252,6 +1339,7 @@ impl Config { "events.webhook_url must be an http(s) URL" ); } + self.events.nats.validate()?; Ok(()) } @@ -1740,6 +1828,34 @@ webhook_secret = "s" let err = Config::parse("[events]\nwebhook_url = \"ftp://x\"\n").unwrap_err(); assert!(err.to_string().contains("webhook_url"), "{err}"); } + + #[test] + fn events_nats_section_parses_and_validates() { + let d = Config::parse("").unwrap().events.nats; + assert_eq!((d.stream.as_str(), d.start), ("WALGIT_EVENTS", NatsStart::Oldest)); + let c = Config::parse( + r#" +[events.nats] +urls = ["nats://a:4222", "nats://b:4222"] +start = "head" +subject_prefix = "acme.git" +"#, + ) + .unwrap(); + assert_eq!(c.events.nats.start, NatsStart::Head); + assert_eq!(c.events.nats.urls.len(), 2); + for bad in [ + "subject_prefix = \"a.*\"", + "subject_prefix = \"a..b\"", + "notify_subject = \"x.>\"", + "stream = \"a.b\"", + "queue_group = \"\"", + "urls = []", + ] { + let err = Config::parse(&format!("[events.nats]\n{bad}\n")).unwrap_err(); + assert!(err.to_string().contains("events.nats"), "{bad}: {err}"); + } + } } /// Negotiated static delivery thresholds. Foundation only: no URI emission yet. diff --git a/crates/walgit-events/Cargo.toml b/crates/walgit-events/Cargo.toml new file mode 100644 index 00000000..3b36cd67 --- /dev/null +++ b/crates/walgit-events/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "walgit-events" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +walgit-proto.workspace = true +walgit-git.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/walgit-events/src/lib.rs b/crates/walgit-events/src/lib.rs new file mode 100644 index 00000000..de2be5bb --- /dev/null +++ b/crates/walgit-events/src/lib.rs @@ -0,0 +1,389 @@ +//! `ref` event shapes and the WAL → event conversion. Contract: +//! `docs/EVENTS.md`. Shared by the producers (the webhook bridge in +//! `walgit-server`, the `JetStream` publisher in `walgit-nats`): they tail each +//! repo's WAL from a durable cursor and convert committed PUSH / `REF_UPDATE` +//! entries with [`refs_from_entries`]. Nothing on the push path knows events +//! exist, and neither does the WAL crate (D32). +//! +//! Wire facts consumers rely on (the doc is normative, this is the code): +//! `old`/`new` are always full zero OIDs on create/delete, never empty; +//! dedup key = `(repo, _walgit.seq, ref_name)`; order by `_walgit.seq`. + +#![allow(clippy::indexing_slicing)] + +use serde::Serialize; +use walgit_git::RepoId; +use walgit_proto::v1::{EntryKind, LogEntry}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Action { + Create, + Update, + Delete, +} + +impl Action { + pub fn as_str(self) -> &'static str { + match self { + Action::Create => "create", + Action::Update => "update", + Action::Delete => "delete", + } + } +} + +fn absent(oid: &str) -> bool { + oid.is_empty() || oid.bytes().all(|b| b == b'0') +} + +/// `create` / `update` / `delete` from the oid pair; `None` = no-op +/// (`0→0`, `X→X`) which must not emit. +pub fn classify(old: &str, new: &str) -> Option { + match (absent(old), absent(new)) { + (true, true) => None, + (true, false) => Some(Action::Create), + (false, true) => Some(Action::Delete), + (false, false) if old == new => None, + (false, false) => Some(Action::Update), + } +} + +/// The zero OID of the same length as `other` (sha1: 40, sha256: 64). The +/// WAL records "absent" as `""` (import-built txns) or the wire's zero OID +/// (pushes); the event always carries the zero OID. +fn zero_like(other: &str) -> String { + "0".repeat(other.len()) +} + +pub fn ref_type(name: &str) -> &'static str { + if name.starts_with("refs/heads/") { + "branch" + } else if name.starts_with("refs/tags/") { + "tag" + } else { + "" + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct WalgitExt { + pub schema_version: u32, + /// WAL seq of the committed entry, as a string (uint64 convention). + pub seq: String, + /// `push` | `ref_update`. + pub entry_kind: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub request_id: String, +} + +/// Top level byte-compatible with the common `RefEvent` shape; walgit adds `repo` +/// and `_walgit`. +#[derive(Debug, Clone, Serialize)] +pub struct RefEvent { + pub action: Action, + pub ref_type: &'static str, + pub ref_name: String, + pub old: String, + pub new: String, + pub pusher: String, + pub correlation_id: String, + pub repo: String, + #[serde(rename = "_walgit")] + pub walgit: WalgitExt, +} + +impl RefEvent { + /// Message key (`/`): per-ref order where a transport keeps it. + pub fn key(&self) -> String { + format!("{}/{}", self.repo, self.ref_name) + } + /// Transport headers (`x_walgit_`): filtering without parsing bodies. + /// Always a strict subset of the body. + pub fn headers(&self) -> [(&'static str, &str); 7] { + [ + ("repo", &self.repo), + ("action", self.action.as_str()), + ("ref_type", self.ref_type), + ("ref_name", &self.ref_name), + ("pusher", &self.pusher), + ("seq", &self.walgit.seq), + ("correlation_id", &self.correlation_id), + ] + } +} + +/// `ref` events for the PUSH / `REF_UPDATE` entries in `entries`, in seq order. +pub fn refs_from_entries(repo: &RepoId, entries: &[LogEntry], out: &mut Vec) { + let repo = repo.to_string(); + for entry in entries { + let entry_kind = match EntryKind::try_from(entry.kind) { + Ok(EntryKind::Push) => "push", + Ok(EntryKind::RefUpdate) => "ref_update", + // COMPACT / CHECKPOINT never move refs. + _ => continue, + }; + let Some(txn) = &entry.txn else { continue }; + let pusher = entry.meta.get("principal").cloned().unwrap_or_default(); + let request_id = entry.meta.get("request_id").cloned().unwrap_or_default(); + for u in &txn.updates { + if !u.new_symbolic_target.is_empty() { + continue; // HEAD retarget: no oid transition to announce + } + let Some(action) = classify(&u.old_oid, &u.new_oid) else { + continue; + }; + let (old, new) = match action { + Action::Create => (zero_like(&u.new_oid), u.new_oid.clone()), + Action::Delete => (u.old_oid.clone(), zero_like(&u.old_oid)), + Action::Update => (u.old_oid.clone(), u.new_oid.clone()), + }; + out.push(RefEvent { + action, + ref_type: ref_type(&u.name), + ref_name: u.name.clone(), + old, + new, + pusher: pusher.clone(), + correlation_id: request_id.clone(), + repo: repo.clone(), + walgit: WalgitExt { + schema_version: 1, + seq: entry.seq.to_string(), + entry_kind: entry_kind.to_string(), + request_id: request_id.clone(), + }, + }); + } + } +} + +/// The bucket object keys a notification body names, `store.prefix` included as the +/// notifications carry it. Store-agnostic. +pub fn notified_keys(v: &serde_json::Value, store_prefix: &str) -> Vec { + let mut keys = Vec::new(); + // GCS → Pub/Sub push envelope. + let attrs = &v["message"]["attributes"]; + if attrs["eventType"] == "OBJECT_FINALIZE" + && let Some(k) = attrs["objectId"].as_str() + { + keys.push(k.to_string()); + } + // S3 event notification (also what MinIO/rustfs/Ceph emit; MinIO spells the name `s3:ObjectCreated:*`). + if let Some(records) = v["Records"].as_array() { + for r in records { + if r["eventName"] + .as_str() + .is_some_and(|e| e.strip_prefix("s3:").unwrap_or(e).starts_with("ObjectCreated")) + && let Some(k) = r["s3"]["object"]["key"].as_str() + { + keys.push(percent_decode(k)); + } + } + } + // Plain shapes for your own glue. + if let Some(k) = v["key"].as_str() { + keys.push(k.to_string()); + } + // A repository id names no object: use the key its manifest has in this bucket. + if let Some(r) = v["repo"].as_str() { + keys.push(format!("{store_prefix}repos/{r}/manifest.pb")); + } + keys +} + +/// S3 URL-encodes keys in notifications (`+` is a space, `%XX` a byte). +fn percent_decode(key: &str) -> String { + let bytes = key.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + match b { + b'+' => out.push(b' '), + b'%' => { + let hex = bytes + .get(i + 1..i + 3) + .and_then(|h| std::str::from_utf8(h).ok()) + .and_then(|h| u8::from_str_radix(h, 16).ok()); + if let Some(decoded) = hex { + out.push(decoded); + i += 2; + } else { + out.push(b); + } + } + _ => out.push(b), + } + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// `prefix/repos///manifest.pb` → `o/r`: the repository whose commit point +/// `object` is. `None` for every other key. +pub fn manifest_repo(object: &str, store_prefix: &str) -> Option { + let rel = object + .strip_prefix(store_prefix)? + .strip_prefix("repos/")? + .strip_suffix("/manifest.pb")?; + let (owner, name) = rel.split_once('/')?; + if name.contains('/') { + return None; + } + RepoId::new(owner, name).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn classify_shapes() { + assert_eq!(classify("", ""), None); + assert_eq!(classify("0000", "0000"), None); + assert_eq!(classify("", "abc"), Some(Action::Create)); + assert_eq!(classify("abc", ""), Some(Action::Delete)); + assert_eq!(classify("abc", "abc"), None); + assert_eq!(classify("abc", "def"), Some(Action::Update)); + assert_eq!(ref_type("refs/heads/main"), "branch"); + assert_eq!(ref_type("refs/tags/v1"), "tag"); + assert_eq!(ref_type("refs/notes/commits"), ""); + } + + /// The golden body shape (docs/EVENTS.md). Field order and names are the + /// contract; changing this test is changing the wire. + #[test] + fn ref_event_golden() { + let ev = RefEvent { + action: Action::Update, + ref_type: "branch", + ref_name: "refs/heads/main".into(), + old: "48a0637".into(), + new: "cb38da1".into(), + pusher: "alice@example.com".into(), + correlation_id: "d1f916f7".into(), + repo: "acme/monorepo".into(), + walgit: WalgitExt { + schema_version: 1, + seq: "42".into(), + entry_kind: "push".into(), + request_id: "d1f916f7".into(), + }, + }; + let got = serde_json::to_string(&ev).unwrap(); + let want = concat!( + "{\"action\":\"update\",\"ref_type\":\"branch\",\"ref_name\":\"refs/heads/main\",", + "\"old\":\"48a0637\",\"new\":\"cb38da1\",\"pusher\":\"alice@example.com\",", + "\"correlation_id\":\"d1f916f7\",\"repo\":\"acme/monorepo\",", + "\"_walgit\":{\"schema_version\":1,\"seq\":\"42\",\"entry_kind\":\"push\",", + "\"request_id\":\"d1f916f7\"}}" + ); + assert_eq!(got, want); + assert_eq!(ev.key(), "acme/monorepo/refs/heads/main"); + } + + fn log_entry(kind: EntryKind, name: &str, old: &str, new: &str, sym: &str) -> LogEntry { + let mut meta = HashMap::new(); + meta.insert("principal".into(), "alice".into()); + meta.insert("request_id".into(), "cid".into()); + LogEntry { + seq: 9, + kind: kind as i32, + txn: Some(walgit_proto::v1::RefTransaction { + updates: vec![walgit_proto::v1::RefUpdate { + name: name.into(), + old_oid: old.into(), + new_oid: new.into(), + new_symbolic_target: sym.into(), + ..Default::default() + }], + ..Default::default() + }), + meta, + ..Default::default() + } + } + + #[test] + fn manifest_object_names() { + let prefix = "prefix/"; + let id = manifest_repo("prefix/repos/acme/monorepo/manifest.pb", prefix).unwrap(); + assert_eq!(id.to_string(), "acme/monorepo"); + // Every body shape names the same repository under a non-empty prefix. + for body in [ + serde_json::json!({"repo": "acme/monorepo"}), + serde_json::json!({"key": "prefix/repos/acme/monorepo/manifest.pb"}), + serde_json::json!({"message": {"attributes": { + "eventType": "OBJECT_FINALIZE", + "objectId": "prefix/repos/acme/monorepo/manifest.pb"}}}), + serde_json::json!({"Records": [{"eventName": "ObjectCreated:Put", + "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), + serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), + ] { + let ids: Vec<_> = notified_keys(&body, prefix) + .iter() + .filter_map(|k| manifest_repo(k, prefix)) + .map(|id| id.to_string()) + .collect(); + assert_eq!(ids, ["acme/monorepo"], "{body}"); + } + for other in [ + "repos/t/r/manifest.pb", + "prefix/repos/t/r/wal/abc.pack", + "prefix/repos/t/r/events/cursor.json", + "walgit-go/repos/t/r/manifest.pb", + "prefix/repos/t/manifest.pb", + "prefix/repos/t/r/x/manifest.pb", + ] { + assert!(manifest_repo(other, prefix).is_none(), "{other}"); + } + } + + #[test] + fn s3_keys_are_percent_decoded() { + assert_eq!(percent_decode("a%2Fb+c%zz%4"), "a/b c%zz%4"); + } + + const SHA: &str = "4a06219d6a5ba9d3e3d1a8e1e2b6f0c9d8e7f6a5"; + + #[test] + fn conversion_emits_ref_skips_compact_noop_symbolic() { + let repo = walgit_git::RepoId::new("t", "r").unwrap(); + let mut out = Vec::new(); + refs_from_entries( + &repo, + &[ + log_entry(EntryKind::Compact, "refs/heads/main", "", SHA, ""), + log_entry(EntryKind::Push, "refs/heads/main", SHA, SHA, ""), + log_entry(EntryKind::Push, "HEAD", "", "", "refs/heads/main"), + log_entry(EntryKind::Push, "refs/heads/main", "", SHA, ""), + log_entry(EntryKind::Push, "refs/heads/main", SHA, &"0".repeat(40), ""), + ], + &mut out, + ); + assert_eq!(out.len(), 2, "compact/noop/symbolic must not emit"); + let r = &out[0]; + assert_eq!(r.action, Action::Create); + assert_eq!(r.ref_name, "refs/heads/main"); + assert_eq!( + r.old, + "0".repeat(40), + "absent old is the zero OID, never empty" + ); + assert_eq!(r.new, SHA); + assert_eq!( + (r.pusher.as_str(), r.correlation_id.as_str()), + ("alice", "cid") + ); + assert_eq!((r.repo.as_str(), r.walgit.seq.as_str()), ("t/r", "9")); + assert_eq!(r.walgit.entry_kind, "push"); + let d = &out[1]; + assert_eq!(d.action, Action::Delete); + assert_eq!( + (d.old.as_str(), d.new.as_str()), + (SHA, "0".repeat(40).as_str()) + ); + } +} diff --git a/crates/walgit-nats/Cargo.toml b/crates/walgit-nats/Cargo.toml new file mode 100644 index 00000000..2aa9311d --- /dev/null +++ b/crates/walgit-nats/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "walgit-nats" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +walgit-events.workspace = true +walgit-config.workspace = true +walgit-store.workspace = true +walgit-git.workspace = true +walgit-wal.workspace = true +async-nats.workspace = true +anyhow.workspace = true +thiserror.workspace = true +bytes.workspace = true +futures.workspace = true +tokio.workspace = true +tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile = { workspace = true, optional = true } + +[features] +# `testing::start_nats`: a throwaway `nats-server -js` for integration tests. +testing = ["dep:tempfile"] + +[dev-dependencies] +walgit-nats = { path = ".", features = ["testing"] } +walgit-proto.workspace = true +nkeys.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/walgit-nats/src/auth.rs b/crates/walgit-nats/src/auth.rs new file mode 100644 index 00000000..3397f753 --- /dev/null +++ b/crates/walgit-nats/src/auth.rs @@ -0,0 +1,220 @@ +//! How `walgit nats` authenticates to NATS, resolved from the standard `NATS_*` environment +//! variables so secrets never sit in `walgit.toml`. + +use std::fmt; +use std::path::PathBuf; + +use anyhow::Context; + +pub const ENV_NATS_CREDS: &str = "NATS_CREDS"; +pub const ENV_NATS_NKEY: &str = "NATS_NKEY"; +pub const ENV_NATS_USER: &str = "NATS_USER"; +pub const ENV_NATS_PASSWORD: &str = "NATS_PASSWORD"; +pub const ENV_NATS_TOKEN: &str = "NATS_TOKEN"; + +/// NATS authentication method. +/// +/// When resolved from environment variables, priority order is: +/// 1. Credentials file (`NATS_CREDS`) +/// 2. `NKey` seed (`NATS_NKEY`) +/// 3. User/Password (`NATS_USER` + `NATS_PASSWORD`) +/// 4. Token (`NATS_TOKEN`) +/// 5. No auth +#[derive(Clone)] +pub enum NatsAuth { + Credentials(PathBuf), + NKey(String), + UserPassword { user: String, password: String }, + Token(String), + None, +} + +/// `NKey`, `password`, and `Token` are bearer credentials and must not be logged. +impl fmt::Debug for NatsAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Credentials(path) => f.debug_tuple("Credentials").field(path).finish(), + Self::NKey(_) => f.debug_tuple("NKey").field(&"").finish(), + Self::UserPassword { user, .. } => f + .debug_struct("UserPassword") + .field("user", user) + .field("password", &"") + .finish(), + Self::Token(_) => f.debug_tuple("Token").field(&"").finish(), + Self::None => f.write_str("None"), + } + } +} + +impl NatsAuth { + pub fn description(&self) -> &'static str { + match self { + Self::Credentials(_) => "credentials file", + Self::NKey(_) => "NKey", + Self::UserPassword { .. } => "user/password", + Self::Token(_) => "token", + Self::None => "none", + } + } + + /// Resolve from the process environment. + pub fn from_process_env() -> Self { + Self::from_env(|name| std::env::var(name).ok()) + } + + /// Resolve through `var`, in the priority order of [`NatsAuth`]. + pub fn from_env(var: impl Fn(&str) -> Option) -> Self { + if let Some(path) = var(ENV_NATS_CREDS) { + return Self::Credentials(PathBuf::from(path)); + } + if let Some(seed) = var(ENV_NATS_NKEY) { + return Self::NKey(seed); + } + if let (Some(user), Some(password)) = (var(ENV_NATS_USER), var(ENV_NATS_PASSWORD)) { + return Self::UserPassword { user, password }; + } + if let Some(token) = var(ENV_NATS_TOKEN) { + return Self::Token(token); + } + Self::None + } + + pub(crate) async fn connect_options(&self) -> anyhow::Result { + Ok(match self { + Self::Credentials(path) => { + async_nats::ConnectOptions::with_credentials_file(path.clone()) + .await + .with_context(|| format!("{ENV_NATS_CREDS}: {}", path.display()))? + } + Self::NKey(seed) => async_nats::ConnectOptions::with_nkey(seed.clone()), + Self::UserPassword { user, password } => { + async_nats::ConnectOptions::with_user_and_password(user.clone(), password.clone()) + } + Self::Token(token) => async_nats::ConnectOptions::with_token(token.clone()), + Self::None => async_nats::ConnectOptions::new(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn resolve(vars: &[(&str, &str)]) -> NatsAuth { + let vars: HashMap = vars + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + NatsAuth::from_env(|name| vars.get(name).cloned()) + } + + #[test] + fn no_variables_mean_no_auth() { + assert!(matches!(resolve(&[]), NatsAuth::None)); + } + + #[test] + fn credentials_take_priority() { + let auth = resolve(&[ + ("NATS_CREDS", "/path/to/creds"), + ("NATS_NKEY", "some-nkey"), + ("NATS_TOKEN", "some-token"), + ]); + assert!( + matches!(auth, NatsAuth::Credentials(p) if p == std::path::Path::new("/path/to/creds")) + ); + } + + #[test] + fn nkey_over_user_password_and_token() { + let auth = resolve(&[ + ("NATS_NKEY", "my-nkey"), + ("NATS_USER", "user"), + ("NATS_PASSWORD", "pass"), + ("NATS_TOKEN", "tok"), + ]); + assert!(matches!(auth, NatsAuth::NKey(k) if k == "my-nkey")); + } + + #[test] + fn user_password_over_token() { + let auth = resolve(&[ + ("NATS_USER", "user"), + ("NATS_PASSWORD", "pass"), + ("NATS_TOKEN", "tok"), + ]); + assert!(matches!( + auth, + NatsAuth::UserPassword { user, password } if user == "user" && password == "pass" + )); + } + + #[test] + fn token_when_nothing_else() { + assert!( + matches!(resolve(&[("NATS_TOKEN", "my-token")]), NatsAuth::Token(t) if t == "my-token") + ); + } + + #[test] + fn user_without_password_is_no_auth() { + assert!(matches!(resolve(&[("NATS_USER", "user")]), NatsAuth::None)); + assert!(matches!( + resolve(&[("NATS_PASSWORD", "pass")]), + NatsAuth::None + )); + } + + #[test] + fn description_matches_variant() { + assert_eq!( + NatsAuth::Credentials("/a".into()).description(), + "credentials file" + ); + assert_eq!(NatsAuth::NKey("k".into()).description(), "NKey"); + assert_eq!( + NatsAuth::UserPassword { + user: "u".into(), + password: "p".into() + } + .description(), + "user/password" + ); + assert_eq!(NatsAuth::Token("t".into()).description(), "token"); + assert_eq!(NatsAuth::None.description(), "none"); + } + + #[test] + fn debug_redacts_credentials() { + let nkey = format!("{:?}", NatsAuth::NKey("super-secret-nkey".into())); + assert!(!nkey.contains("super-secret-nkey")); + assert!(nkey.contains("")); + + let user_password = format!( + "{:?}", + NatsAuth::UserPassword { + user: "alice".into(), + password: "super-secret-password".into(), + } + ); + assert!(user_password.contains("alice")); + assert!(!user_password.contains("super-secret-password")); + assert!(user_password.contains("")); + + let token = format!("{:?}", NatsAuth::Token("super-secret-token".into())); + assert!(!token.contains("super-secret-token")); + assert!(token.contains("")); + } + + #[test] + fn debug_shows_non_secret_variants() { + let creds = format!( + "{:?}", + NatsAuth::Credentials(PathBuf::from("/etc/nats/app.creds")) + ); + assert!(creds.contains("/etc/nats/app.creds")); + assert_eq!(format!("{:?}", NatsAuth::None), "None"); + } +} diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs new file mode 100644 index 00000000..80679848 --- /dev/null +++ b/crates/walgit-nats/src/lib.rs @@ -0,0 +1,140 @@ +//! `walgit nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with +//! the cursor in the stream, wake-ups over NATS and a `$SRV` service API instead of HTTP. +//! The event model is shared with the webhook bridge through `walgit-events`. + +pub mod auth; +pub mod message; +pub mod publisher; +pub mod service; +pub mod subject; +#[cfg(feature = "testing")] +pub mod testing; +pub mod wake; + +use std::str::FromStr; +use std::sync::Arc; + +use anyhow::Context; +use async_nats::jetstream::{self, stream}; +use walgit_config::{Config, NatsEventsConfig}; + +use crate::auth::NatsAuth; +use crate::publisher::Publisher; +use crate::subject::SubjectPrefix; + +/// Connect to `[events.nats] urls` with `auth`. Only the method is logged, never the secret. +pub async fn connect(cfg: &NatsEventsConfig, auth: &NatsAuth) -> anyhow::Result { + let addrs = cfg + .urls + .iter() + .map(|u| async_nats::ServerAddr::from_str(u).with_context(|| format!("events.nats.urls: {u}"))) + .collect::>>()?; + tracing::info!(urls = ?cfg.urls, auth = auth.description(), "connecting to NATS"); + auth.connect_options() + .await? + .name("walgit") + .connect(addrs) + .await + .with_context(|| format!("connecting to NATS (auth: {})", auth.description())) +} + +/// The stream holding the events: created when missing (file storage, `.ref.>`, +/// `duplicate_window`) if `ensure_stream`, otherwise it must exist. +pub async fn open_stream( + js: &jetstream::Context, + cfg: &NatsEventsConfig, + prefix: &SubjectPrefix, +) -> anyhow::Result { + if cfg.ensure_stream { + js.get_or_create_stream(stream::Config { + name: cfg.stream.clone(), + subjects: vec![prefix.stream_filter()], + storage: stream::StorageType::File, + duplicate_window: cfg.duplicate_window, + ..stream::Config::default() + }) + .await + .with_context(|| format!("ensuring stream {}", cfg.stream)) + } else { + js.get_stream(&cfg.stream) + .await + .with_context(|| format!("stream {} (events.nats.ensure_stream is off)", cfg.stream)) + } +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + match signal(SignalKind::terminate()) { + Ok(mut term) => { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } + return; + } + Err(e) => tracing::warn!(error = %e, "no SIGTERM handler"), + } + } + let _ = tokio::signal::ctrl_c().await; +} + +/// The config the publisher's registry runs on. This process never applies a push itself, so a +/// wake-up must always see the manifest as it is in the bucket (`freshness_ttl` would let it +/// skip the GET and miss the push it was woken for), and it only reads manifest and log, so +/// pack prefetch would pull bytes nobody reads. Both hold whatever `[wal]` says. +pub fn publisher_config(cfg: &Config) -> Config { + let mut cfg = cfg.clone(); + cfg.wal.freshness_ttl = std::time::Duration::ZERO; + cfg.wal.prefetch_packs = false; + cfg +} + +/// `walgit nats`: publish until SIGTERM/ctrl-c, or fail when a worker stops. +pub async fn run(cfg: Arc) -> anyhow::Result<()> { + let nats = &cfg.events.nats; + let prefix = SubjectPrefix::new(&nats.subject_prefix)?; + let store = walgit_store::open_store(&cfg).await?; + let registry = walgit_wal::Registry::new(store, Arc::new(publisher_config(&cfg))); + + let client = crate::connect(nats, &NatsAuth::from_process_env()).await?; + let js = async_nats::jetstream::new(client.clone()); + let stream = crate::open_stream(&js, nats, &prefix).await?; + let publisher = Publisher::new(registry, js, stream, prefix.clone(), nats.start); + tracing::info!(stream = %nats.stream, prefix = %prefix, start = ?nats.start, "walgit-nats ready"); + + let mut tasks = tokio::task::JoinSet::new(); + tasks.spawn(crate::wake::notify_loop( + client.clone(), + nats.notify_subject.clone(), + nats.queue_group.clone(), + cfg.store_prefix(), + publisher.clone(), + )); + tasks.spawn({ + let (client, queue_group, publisher) = + (client.clone(), nats.queue_group.clone(), publisher.clone()); + async move { crate::service::serve(client, &prefix, &queue_group, publisher).await } + }); + tasks.spawn({ + let publisher = publisher.clone(); + let every = cfg.events.sweep_interval; + async move { + crate::wake::sweep_loop(publisher, every).await; + std::future::pending::>().await + } + }); + + tokio::select! { + () = shutdown_signal() => tracing::info!("shutting down"), + Some(done) = tasks.join_next() => { + // A loop ending is a failure: the subscription or the service closed under us. + done.context("worker panicked")?.context("worker stopped")?; + anyhow::bail!("a worker stopped unexpectedly"); + } + } + tasks.shutdown().await; + client.flush().await.ok(); + Ok(()) +} diff --git a/crates/walgit-nats/src/message.rs b/crates/walgit-nats/src/message.rs new file mode 100644 index 00000000..c2fcdbfe --- /dev/null +++ b/crates/walgit-nats/src/message.rs @@ -0,0 +1,208 @@ +//! One `RefEvent` as a `JetStream` message: body = the webhook's event JSON, headers +//! `Walgit-Seq`, `Walgit-Repo` and `Nats-Msg-Id = ::`. + +use std::fmt; + +use async_nats::header::NATS_MESSAGE_ID; +use async_nats::{HeaderMap, jetstream::message::PublishMessage}; +use bytes::Bytes; +use walgit_events::RefEvent; + +use crate::subject::RepoSubject; + +pub const HEADER_SEQ: &str = "Walgit-Seq"; +pub const HEADER_REPO: &str = "Walgit-Repo"; + +/// `::`: `JetStream`'s dedup key and, with the repository, the consumer's. +/// A repository id never contains `:` and a seq is a number, so the first two colons delimit +/// the ref name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MsgId { + repo: String, + seq: u64, + ref_name: String, +} + +impl MsgId { + pub fn new(repo: &str, seq: u64, ref_name: &str) -> Self { + MsgId { + repo: repo.to_string(), + seq, + ref_name: ref_name.to_string(), + } + } + + pub fn of(event: &RefEvent) -> Option { + Some(MsgId::new( + &event.repo, + event.walgit.seq.parse().ok()?, + &event.ref_name, + )) + } + + pub fn parse(raw: &str) -> Option { + let mut parts = raw.splitn(3, ':'); + let repo = parts.next()?; + let seq = parts.next()?.parse().ok()?; + let ref_name = parts.next()?; + Some(MsgId::new(repo, seq, ref_name)) + } + + pub fn seq(&self) -> u64 { + self.seq + } + + pub fn ref_name(&self) -> &str { + &self.ref_name + } +} + +impl fmt::Display for MsgId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}:{}", self.repo, self.seq, self.ref_name) + } +} + +/// The last event on a repository's subject, as the stream remembers it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LastPublished { + /// Stream sequence of the message (the `Nats-Expected-Last-Subject-Sequence` token). + pub stream_seq: u64, + pub id: MsgId, +} + +impl LastPublished { + pub fn from_headers(stream_seq: u64, headers: &HeaderMap) -> Option { + let id = MsgId::parse(headers.get(NATS_MESSAGE_ID)?.as_str())?; + let seq: u64 = headers.get(HEADER_SEQ)?.as_str().parse().ok()?; + (seq == id.seq()).then_some(LastPublished { stream_seq, id }) + } +} + +/// The events of `last`'s entry that were not published yet, and everything after it. One WAL +/// entry can yield several events, so a crash mid-entry leaves a tail to finish; resuming from the +/// entry (inclusive) and dropping what the stream already holds makes it exact. +pub fn after_last_published(events: Vec, last: &LastPublished) -> Vec { + let seq = last.id.seq().to_string(); + let published = events + .iter() + .position(|e| e.walgit.seq == seq && e.ref_name == last.id.ref_name()); + match published { + Some(i) => events.into_iter().skip(i + 1).collect(), + None => events, + } +} + +pub struct EventMessage { + pub subject: RepoSubject, + pub id: MsgId, + publish: PublishMessage, +} + +impl EventMessage { + pub fn new(subject: RepoSubject, event: &RefEvent) -> anyhow::Result { + let id = MsgId::of(event) + .ok_or_else(|| anyhow::anyhow!("event seq {:?} is not a number", event.walgit.seq))?; + let publish = PublishMessage::build() + .payload(Bytes::from(serde_json::to_vec(event)?)) + .message_id(id.to_string()) + .header(HEADER_SEQ, event.walgit.seq.as_str()) + .header(HEADER_REPO, event.repo.as_str()); + Ok(EventMessage { + subject, + id, + publish, + }) + } + + /// Fails the publish unless the subject's last message is `stream_seq` (`0` = none yet). + #[must_use] + pub fn expecting_last_subject_sequence(mut self, stream_seq: u64) -> Self { + self.publish = self.publish.expected_last_subject_sequence(stream_seq); + self + } + + pub fn into_publish(self) -> PublishMessage { + self.publish + } +} + +#[cfg(test)] +mod tests { + use super::*; + use walgit_events::{Action, WalgitExt}; + + fn event(seq: u64, ref_name: &str) -> RefEvent { + RefEvent { + action: Action::Update, + ref_type: "branch", + ref_name: ref_name.into(), + old: "a".into(), + new: "b".into(), + pusher: "p".into(), + correlation_id: "c".into(), + repo: "acme/repo".into(), + walgit: WalgitExt { + schema_version: 1, + seq: seq.to_string(), + entry_kind: "push".into(), + request_id: "c".into(), + }, + } + } + + #[test] + fn msg_id_round_trips_with_colons_in_the_ref() { + let id = MsgId::new("acme/repo", 42, "refs/heads/a:b"); + assert_eq!(id.to_string(), "acme/repo:42:refs/heads/a:b"); + assert_eq!(MsgId::parse(&id.to_string()), Some(id)); + assert_eq!(MsgId::parse("acme/repo:x:refs/heads/a"), None); + assert_eq!(MsgId::parse("acme/repo:1"), None); + } + + #[test] + fn resume_drops_only_what_the_stream_already_holds() { + let events = vec![ + event(5, "refs/heads/a"), + event(5, "refs/heads/b"), + event(5, "refs/heads/c"), + event(6, "refs/heads/a"), + ]; + let last = |seq, name: &str| LastPublished { + stream_seq: 9, + id: MsgId::new("acme/repo", seq, name), + }; + let names = |v: Vec| { + v.into_iter() + .map(|e| format!("{}:{}", e.walgit.seq, e.ref_name)) + .collect::>() + }; + // Crash after b: c and the next entry remain. + assert_eq!( + names(after_last_published(events.clone(), &last(5, "refs/heads/b"))), + ["5:refs/heads/c", "6:refs/heads/a"] + ); + // The whole entry was published. + assert_eq!( + names(after_last_published(events.clone(), &last(5, "refs/heads/c"))), + ["6:refs/heads/a"] + ); + // Nothing new at all. + assert!(after_last_published(events.clone(), &last(6, "refs/heads/a")).is_empty()); + // The message is not among the events (entry rewritten or folded): publish everything. + assert_eq!( + after_last_published(events.clone(), &last(5, "refs/heads/zzz")).len(), + 4 + ); + } + + #[test] + fn message_carries_the_contract_headers() { + let prefix = crate::subject::SubjectPrefix::new("walgit").unwrap(); + let repo = walgit_git::RepoId::new("acme", "repo").unwrap(); + let ev = event(7, "refs/heads/main"); + let msg = EventMessage::new(RepoSubject::new(&prefix, &repo), &ev).unwrap(); + assert_eq!(msg.id.to_string(), "acme/repo:7:refs/heads/main"); + assert_eq!(msg.subject.as_str(), "walgit.ref.acme.repo"); + } +} diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs new file mode 100644 index 00000000..51814c93 --- /dev/null +++ b/crates/walgit-nats/src/publisher.rs @@ -0,0 +1,342 @@ +//! The `JetStream` publisher: the WAL → `.ref..`. +//! +//! `catch_up(repo)`: the cursor is the stream. The last message on the repo's subject names the +//! WAL seq (`Walgit-Seq`) and ref it reached; the publisher reads the log from that entry +//! (inclusive: one entry can yield several events and a crash mid-entry must not drop the rest), +//! drops the events the stream already holds, and publishes the remainder one at a time, waiting +//! for each `PubAck`. The first publish of a catch-up carries +//! `Nats-Expected-Last-Subject-Sequence`, so a second publisher that got there first makes this one +//! stop (the next wake-up retries); every message carries `Nats-Msg-Id` (`::`) so +//! `JetStream` dedups within its window. Consumers still dedup on `(repo, _walgit.seq, ref_name)`. +//! +//! The cursor is the sink's state, not walgit's (D50): wiping walgit loses nothing, wiping the +//! stream republishes the WAL window (`events.nats.start` decides where a cold repo begins). + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context; +use async_nats::jetstream::{ + self, + context::PublishErrorKind, + stream::{LastRawMessageErrorKind, Stream}, +}; +use walgit_config::NatsStart; +use walgit_events::refs_from_entries; +use walgit_git::RepoId; + +use crate::message::{EventMessage, LastPublished, MsgId, after_last_published}; +use crate::subject::{RepoSubject, SubjectPrefix}; + +/// What one `catch_up` did. +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct CatchUp { + pub repo: String, + /// The WAL seq the stream had reached for this repository (or the cold start point). + pub from_seq: u64, + pub head_seq: u64, + /// Messages published now (`JetStream`-deduplicated ones included). + pub emitted: usize, + /// `[first, last]` seqs folded into a checkpoint before they were read: not emitted, counted; + /// consumers backfill from the WAL. + #[serde(skip_serializing_if = "Option::is_none")] + pub gap: Option<[u64; 2]>, + /// Another publisher advanced the subject first; this catch-up stopped and the next retries. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub conflict: bool, +} + +#[derive(Debug, Default)] +pub struct Counters { + pub published: AtomicU64, + pub deduplicated: AtomicU64, + pub gap_entries: AtomicU64, + pub conflicts: AtomicU64, + pub rewinds: AtomicU64, + pub sweep_found: AtomicU64, +} + +impl Counters { + pub fn snapshot(&self) -> serde_json::Value { + let n = |c: &AtomicU64| c.load(Ordering::Relaxed); + serde_json::json!({ + "published": n(&self.published), + "deduplicated": n(&self.deduplicated), + "gap_entries": n(&self.gap_entries), + "conflicts": n(&self.conflicts), + "rewinds": n(&self.rewinds), + "sweep_found": n(&self.sweep_found), + }) + } +} + +pub struct Publisher { + registry: Arc, + js: jetstream::Context, + stream: Stream, + prefix: SubjectPrefix, + start: NatsStart, + serial: tokio::sync::Mutex<()>, + /// Where each repository was fully published up to by this process: the head and the stream's + /// last message for its subject. A wake-up that finds both unchanged spares the log read. The + /// stream half is what notices a purged or recreated stream. Never a source of truth. + done: std::sync::Mutex)>>, + /// The head a cold repository was first seen at (`start = "head"`), and for a recreated one, + /// which old incarnation it was seen behind. + cold_marks: std::sync::Mutex>, + /// The stream sequence of the message each repository's gap was counted behind: the stream + /// keeps no cursor past it, so a head moved only by entries without ref events would otherwise + /// count the same gap again. + gaps_seen: std::sync::Mutex>, + pub counters: Counters, +} + +/// A cold repository's first sighting. `behind` is the stream sequence of the old incarnation's +/// last message when the repository was recreated: the same rewind seen again keeps its mark, so +/// pushes after the first sighting are published while the old message still tops the subject. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ColdMark { + head: u64, + behind: Option, +} + +fn locked(m: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl Publisher { + pub fn new( + registry: Arc, + js: jetstream::Context, + stream: Stream, + prefix: SubjectPrefix, + start: NatsStart, + ) -> Arc { + Arc::new(Publisher { + registry, + js, + stream, + prefix, + start, + serial: tokio::sync::Mutex::new(()), + done: std::sync::Mutex::default(), + cold_marks: std::sync::Mutex::default(), + gaps_seen: std::sync::Mutex::default(), + counters: Counters::default(), + }) + } + + pub fn registry(&self) -> &Arc { + &self.registry + } + + async fn last_published(&self, subject: &RepoSubject) -> anyhow::Result> { + match self + .stream + .get_last_raw_message_by_subject(subject.as_str()) + .await + { + Ok(m) => { + let last = LastPublished::from_headers(m.sequence, &m.headers); + anyhow::ensure!( + last.is_some(), + "last message on {subject} (stream seq {}) lacks Walgit-Seq / Nats-Msg-Id", + m.sequence + ); + Ok(last) + } + Err(e) if matches!(e.kind(), LastRawMessageErrorKind::NoMessageFound) => Ok(None), + Err(e) => Err(e).with_context(|| format!("reading the last message on {subject}")), + } + } + + /// Publish everything committed after what the stream holds for `id`. + pub async fn catch_up(&self, id: &RepoId) -> anyhow::Result { + let _serial = self.serial.lock().await; + let handle = self.registry.open(id).await?; + let manifest = { + let guard = handle.sync_refs().await?; + let m = handle.manifest(); + drop(guard); + m + }; + let head = manifest.head_seq; + // Entries below `min_seq` are folded into a checkpoint and no longer in the manifest's + // log window; `min_seq - 1` is the newest seq that is lost to us. + let readable_from = manifest.min_seq.saturating_sub(1); + let lowest = readable_from + 1; + + let mut report = CatchUp { + repo: id.to_string(), + from_seq: head, + head_seq: head, + ..CatchUp::default() + }; + + // One cheap request on every wake-up: the stream, not this process, says what is published. + let subject = RepoSubject::new(&self.prefix, id); + let last = self.last_published(&subject).await?; + if locked(&self.done) + .get(id) + .is_some_and(|(done_head, done_last)| *done_head == head && *done_last == last) + { + return Ok(report); + } + + // The stream is ahead of the WAL: the repository was deleted and created again (or the log + // rewound). What is on the subject belongs to the old incarnation, so it names no cursor; + // its stream sequence still guards the first publish. + let rewound = last.as_ref().is_some_and(|l| l.id.seq() > head); + let behind = last.as_ref().filter(|_| rewound).map(|l| l.stream_seq); + let seen = ColdMark { head, behind }; + let first_sighting = rewound + && locked(&self.cold_marks) + .get(id) + .is_none_or(|mark| mark.behind != behind); + if first_sighting { + self.counters.rewinds.fetch_add(1, Ordering::Relaxed); + locked(&self.cold_marks).insert(id.clone(), seen); + tracing::warn!(repo = %id, published = last.as_ref().map_or(0, |l| l.id.seq()), head, + "the stream is ahead of the WAL: the repository was recreated; publishing it as a new one"); + } + let basis = if rewound { None } else { last.as_ref() }; + + // First seq to read; `resume` is the entry the stream may hold only part of. + let (first, resume) = match basis { + Some(l) if l.id.seq() >= lowest => (l.id.seq(), Some(l)), + Some(l) => { + // The newest published entry is already folded away: what follows it up to the + // log window is lost to us. + let published = l.id.seq(); + let first_sighting = published < readable_from + && locked(&self.gaps_seen).insert(id.clone(), l.stream_seq) != Some(l.stream_seq); + if first_sighting { + report.gap = Some([published + 1, readable_from]); + self.counters + .gap_entries + .fetch_add(readable_from - published, Ordering::Relaxed); + tracing::warn!(repo = %id, first = published + 1, last = readable_from, + "entries folded into a checkpoint before they were published; consumers backfill from the WAL"); + } + (lowest, None) + } + None => match self.start { + NatsStart::Oldest => (lowest, None), + NatsStart::Head => { + let mark = locked(&self.cold_marks).entry(id.clone()).or_insert(seen).head; + (mark.saturating_add(1).max(lowest), None) + } + }, + }; + report.from_seq = resume.map_or(first.saturating_sub(1), |l| l.id.seq()); + + let mut last_sent: Option = None; + if first <= head { + let entries = handle.read_log(&manifest, first, Some(head)).await?; + let mut events = Vec::new(); + refs_from_entries(id, &entries, &mut events); + if let Some(l) = resume { + events = after_last_published(events, l); + } + // Whatever is on the subject now (a gap leaves older messages there too). + let mut expected = last.as_ref().map_or(0, |l| l.stream_seq); + for (i, event) in events.iter().enumerate() { + last_sent = MsgId::of(event); + let mut msg = EventMessage::new(subject.clone(), event)?; + if i == 0 { + msg = msg.expecting_last_subject_sequence(expected); + } + let (msg_id, subj) = (msg.id.clone(), msg.subject.to_string()); + let ack = self + .js + .send_publish(subj, msg.into_publish()) + .await + .with_context(|| format!("publishing {msg_id}"))?; + match ack.await { + Ok(ack) => { + expected = ack.sequence; + report.emitted += 1; + if ack.duplicate { + self.counters.deduplicated.fetch_add(1, Ordering::Relaxed); + } else { + self.counters.published.fetch_add(1, Ordering::Relaxed); + } + // One structured line per published event, as the bridge logs. + if let Ok(body) = serde_json::to_string(event) { + tracing::info!(target: "walgit::event", + event_type = "ref", repo = %event.repo, event = %body, "event"); + } + } + Err(e) if e.kind() == PublishErrorKind::WrongLastSequence => { + self.counters.conflicts.fetch_add(1, Ordering::Relaxed); + report.conflict = true; + tracing::warn!(repo = %id, "another publisher advanced {subject}; stopping this catch-up, the next wake-up retries"); + return Ok(report); + } + Err(e) => return Err(e).with_context(|| format!("PubAck for {msg_id}")), + } + } + } + + // Done only when the stream confirms its last message is the one just sent: a broker + // that swallowed it as a duplicate of a purged message (the dedup table outlives a purge) + // leaves the repository open, and the retry lands once the dedup window has passed. + let settled = if report.emitted > 0 { + self.last_published(&subject).await? + } else { + last + }; + let confirmed = last_sent + .as_ref() + .is_none_or(|sent| settled.as_ref().is_some_and(|s| s.id == *sent)); + if confirmed { + locked(&self.done).insert(id.clone(), (head, settled)); + } + if report.emitted > 0 { + tracing::info!(repo = %id, from = report.from_seq, head, emitted = report.emitted, "published"); + } + Ok(report) + } + + /// `catch_up` that treats a repository deleted since its notification as nothing to do. + pub async fn catch_up_if_present(&self, id: &RepoId) -> anyhow::Result> { + match self.catch_up(id).await { + Err(e) + if matches!( + e.downcast_ref::(), + Some(walgit_wal::WalError::NotFound) + ) => + { + Ok(None) + } + r => r.map(Some), + } + } + + /// Every repository (a `list` + one conditional manifest GET each): the backstop behind the + /// notifications and the health check. `warn_found`: finding work here means they are not + /// flowing (the startup sweep legitimately finds work and passes `false`). + pub async fn sweep(&self, warn_found: bool) { + let repos = match self.registry.list().await { + Ok(r) => r, + Err(e) => { + tracing::warn!(error = %e, "sweep list failed"); + return; + } + }; + for id in repos { + match self.catch_up_if_present(&id).await { + Ok(Some(c)) if c.emitted > 0 && warn_found => { + self.counters + .sweep_found + .fetch_add(c.emitted as u64, Ordering::Relaxed); + tracing::warn!(repo = %id, emitted = c.emitted, + "sweep found unpublished entries: are the bucket notifications flowing to the notify subject?"); + } + Ok(_) => {} + Err(e) => tracing::warn!(repo = %id, error = %format!("{e:#}"), "sweep catch-up failed"), + } + } + } +} diff --git a/crates/walgit-nats/src/service.rs b/crates/walgit-nats/src/service.rs new file mode 100644 index 00000000..5e311e41 --- /dev/null +++ b/crates/walgit-nats/src/service.rs @@ -0,0 +1,76 @@ +//! The NATS service API (`$SRV` micro protocol) in place of an HTTP listener: +//! `.events.catch-up` takes `{"repo":"owner/name"}` and answers with the catch-up report; +//! `$SRV.PING|INFO|STATS` come with the protocol (stats carry this publisher's counters). + +use std::str::FromStr; +use std::sync::Arc; + +use async_nats::service::ServiceExt; +use async_nats::service::error::Error as ServiceError; +use bytes::Bytes; +use futures::StreamExt; +use walgit_git::RepoId; + +use crate::publisher::Publisher; +use crate::subject::SubjectPrefix; + +pub const SERVICE_NAME: &str = "walgit-events"; + +#[derive(serde::Deserialize)] +struct CatchUpRequest { + repo: String, +} + +fn failure(code: usize, status: &impl ToString) -> ServiceError { + ServiceError { + status: status.to_string(), + code, + } +} + +async fn handle(publisher: &Publisher, payload: &[u8]) -> Result { + let req: CatchUpRequest = + serde_json::from_slice(payload).map_err(|e| failure(400, &format!("request: {e}")))?; + let id = RepoId::from_str(&req.repo).map_err(|e| failure(400, &e))?; + match publisher.catch_up_if_present(&id).await { + Ok(Some(report)) => serde_json::to_vec(&report) + .map(Bytes::from) + .map_err(|e| failure(500, &e)), + Ok(None) => Err(failure(404, &format!("no repository {id}"))), + Err(e) => Err(failure(500, &format!("{e:#}"))), + } +} + +/// Serve until the endpoint closes. Instances share requests through the service queue group. +pub async fn serve( + client: async_nats::Client, + prefix: &SubjectPrefix, + queue_group: &str, + publisher: Arc, +) -> anyhow::Result<()> { + let stats = publisher.clone(); + let service = client + .service_builder() + .description("walgit ref events into JetStream") + .queue_group(queue_group) + .stats_handler(move |_endpoint, _stats| stats.counters.snapshot()) + .start(SERVICE_NAME, env!("CARGO_PKG_VERSION")) + .await + .map_err(|e| anyhow::anyhow!("starting the {SERVICE_NAME} service: {e}"))?; + let mut endpoint = service + .group(prefix.child("events")) + .endpoint("catch-up") + .await + .map_err(|e| anyhow::anyhow!("registering catch-up: {e}"))?; + tracing::info!(subject = %prefix.child("events.catch-up"), "service ready"); + while let Some(request) = endpoint.next().await { + let publisher = publisher.clone(); + tokio::spawn(async move { + let answer = handle(&publisher, &request.message.payload).await; + if let Err(e) = request.respond(answer).await { + tracing::warn!(error = %e, "catch-up reply failed"); + } + }); + } + Ok(()) +} diff --git a/crates/walgit-nats/src/subject.rs b/crates/walgit-nats/src/subject.rs new file mode 100644 index 00000000..7fd5ea9c --- /dev/null +++ b/crates/walgit-nats/src/subject.rs @@ -0,0 +1,240 @@ +//! Subjects of the `JetStream` events: `.ref..`. +//! +//! A NATS subject token may not contain `.`, `*`, `>` or whitespace, and a repository +//! name may contain `.` (`RepoId` allows `[A-Za-z0-9._-]`). Each of owner and repo is +//! therefore one [`Token`]: every byte outside `[A-Za-z0-9_-]` becomes `%XX` (uppercase hex of +//! the UTF-8 byte, `%` itself included), so `acme.io` is `acme%2Eio` and the encoding is +//! reversible token by token. + +use std::fmt::{self, Write as _}; + +use walgit_git::RepoId; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SubjectError { + #[error("subject prefix {0:?} must be dot-separated tokens without '*', '>' or whitespace")] + InvalidPrefix(String), + #[error("{0:?} is not a repository subject under this prefix")] + NotARepoSubject(String), +} + +/// One encoded subject token (never empty, never contains `.`, `*`, `>` or whitespace). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token(String); + +impl Token { + pub fn encode(raw: &str) -> Self { + let mut out = String::with_capacity(raw.len()); + for b in raw.bytes() { + if b.is_ascii_alphanumeric() || b == b'_' || b == b'-' { + out.push(char::from(b)); + } else { + let _ = write!(out, "%{b:02X}"); + } + } + Token(out) + } + + /// `None` when `encoded` is not what [`Token::encode`] produces. + pub fn decode(encoded: &str) -> Option { + let bytes = encoded.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + match b { + b'%' => { + let hex = bytes.get(i + 1..i + 3)?; + if !hex + .iter() + .all(|h| h.is_ascii_digit() || (b'A'..=b'F').contains(h)) + { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(hex).ok()?, 16).ok()?); + i += 3; + } + _ if b.is_ascii_alphanumeric() || b == b'_' || b == b'-' => { + out.push(b); + i += 1; + } + _ => return None, + } + } + if out.is_empty() { + return None; + } + String::from_utf8(out).ok() + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Token { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// The configured `events.nats.subject_prefix`: dot-separated, literal tokens. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubjectPrefix(String); + +impl SubjectPrefix { + pub fn new(raw: &str) -> Result { + let literal = |t: &str| { + !t.is_empty() + && !t + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '*' | '>')) + }; + if raw.split('.').all(literal) { + Ok(SubjectPrefix(raw.to_string())) + } else { + Err(SubjectError::InvalidPrefix(raw.to_string())) + } + } + + /// `.ref.>`: what the stream captures. + pub fn stream_filter(&self) -> String { + format!("{}.ref.>", self.0) + } + + /// `.` for the service API. + pub fn child(&self, name: &str) -> String { + format!("{}.{name}", self.0) + } + + fn ref_root(&self) -> String { + format!("{}.ref.", self.0) + } +} + +impl fmt::Display for SubjectPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// `.ref..`: one repository's events, in WAL order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoSubject(String); + +impl RepoSubject { + pub fn new(prefix: &SubjectPrefix, repo: &RepoId) -> Self { + RepoSubject(format!( + "{}{}.{}", + prefix.ref_root(), + Token::encode(repo.owner()), + Token::encode(repo.name()) + )) + } + + pub fn parse(prefix: &SubjectPrefix, subject: &str) -> Result { + let err = || SubjectError::NotARepoSubject(subject.to_string()); + let rest = subject.strip_prefix(&prefix.ref_root()).ok_or_else(err)?; + let (owner, name) = rest.split_once('.').ok_or_else(err)?; + let owner = Token::decode(owner).ok_or_else(err)?; + let name = Token::decode(name).ok_or_else(err)?; + RepoId::new(owner, name).map_err(|_| err()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for RepoSubject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn prefix() -> SubjectPrefix { + SubjectPrefix::new("walgit").unwrap() + } + + #[test] + fn dots_are_escaped_and_round_trip() { + let repo = RepoId::new("acme.io", "mono.repo-v2_x").unwrap(); + let subject = RepoSubject::new(&prefix(), &repo); + assert_eq!(subject.as_str(), "walgit.ref.acme%2Eio.mono%2Erepo-v2_x"); + assert_eq!(RepoSubject::parse(&prefix(), subject.as_str()).unwrap(), repo); + } + + #[test] + fn token_never_carries_a_nats_metacharacter() { + let t = Token::encode("a.b*c>d e\t%é/"); + assert!(!t.as_str().contains(['.', '*', '>', ' ', '\t', '/'])); + assert_eq!(t.as_str(), "a%2Eb%2Ac%3Ed%20e%09%25%C3%A9%2F"); + assert_eq!(Token::decode(t.as_str()).unwrap(), "a.b*c>d e\t%é/"); + } + + #[test] + fn decode_refuses_what_encode_never_emits() { + for bad in ["", "a.b", "a%2", "a%zz", "a%2e", "a b", "a*", "%FF"] { + assert_eq!(Token::decode(bad), None, "{bad:?}"); + } + } + + #[test] + fn parse_refuses_foreign_subjects() { + for bad in [ + "walgit.ref.acme", + "walgit.ref.acme.repo.extra", + "other.ref.acme.repo", + "walgit.refs.acme.repo", + "walgit.ref..repo", + "walgit.ref.%2E%2E.repo", + ] { + assert!(RepoSubject::parse(&prefix(), bad).is_err(), "{bad}"); + } + } + + #[test] + fn prefix_validation_and_filter() { + assert_eq!(prefix().stream_filter(), "walgit.ref.>"); + let p = SubjectPrefix::new("acme.git").unwrap(); + assert_eq!(p.stream_filter(), "acme.git.ref.>"); + assert_eq!(p.child("events"), "acme.git.events"); + for bad in ["", ".a", "a.", "a..b", "a.*", "a.>", "a b"] { + assert!(SubjectPrefix::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn odd_names_round_trip_and_never_carry_a_metacharacter() { + let names = [ + "a", "-", "_", "a.b.c", "*", ">", "%", "%2E", "%%", "a b", "tab\tnl\n", "é", "日本語", + "emoji\u{1F600}", "a/b\\c", "\u{0}", "..", "*.>", "Z%zz", + ]; + for raw in names { + let t = Token::encode(raw); + assert!(!t.as_str().is_empty(), "{raw:?}"); + assert!( + t.as_str() + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'%')), + "{raw:?} -> {t}" + ); + assert_eq!(Token::decode(t.as_str()).as_deref(), Some(raw), "{raw:?}"); + } + let mut seen = std::collections::HashSet::new(); + assert!(names.iter().all(|n| seen.insert(Token::encode(n).to_string())), "injective"); + } + + #[test] + fn every_single_byte_string_round_trips() { + for c in (1u32..0x250).filter_map(char::from_u32) { + let raw = c.to_string(); + let t = Token::encode(&raw); + assert_eq!(Token::decode(t.as_str()).as_deref(), Some(raw.as_str()), "{c:?}"); + assert!(!t.as_str().contains(['.', '*', '>']), "{c:?}"); + } + } +} diff --git a/crates/walgit-nats/src/testing.rs b/crates/walgit-nats/src/testing.rs new file mode 100644 index 00000000..9d039456 --- /dev/null +++ b/crates/walgit-nats/src/testing.rs @@ -0,0 +1,109 @@ +//! A throwaway `nats-server -js` for tests (feature `testing`). Shared by this crate's tests and +//! `walgit-server`'s, so neither copies the launcher. + +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub struct NatsServer { + child: Child, + port: u16, + args: Vec, + pub url: String, + store: tempfile::TempDir, +} + +impl NatsServer { + fn spawn(port: u16, store: &std::path::Path, args: &[String]) -> Child { + #[allow(clippy::expect_used)] + Command::new("nats-server") + .args(["-js", "-a", "127.0.0.1", "-p", &port.to_string(), "-sd"]) + .arg(store) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn nats-server") + } + + /// Kill the server at once (no drain), leaving its store dir for [`NatsServer::restart`]. + pub fn stop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } + + /// Start the server again on the same port and store dir: streams survive, clients reconnect. + /// + /// # Panics + /// When the server does not accept connections within 15 s. + pub async fn restart(&mut self) { + self.stop(); + self.child = Self::spawn(self.port, self.store.path(), &self.args); + wait_ready(self.port).await; + } +} + +async fn wait_ready(port: u16) { + let t0 = Instant::now(); + while tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_err() + { + assert!( + t0.elapsed() < Duration::from_secs(15), + "nats-server did not start" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +impl Drop for NatsServer { + fn drop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } +} + +/// Start `nats-server -js` on a free loopback port with a temp store dir. `None` (after saying so +/// on stderr) when the binary is not on PATH, so tests skip instead of failing. +/// +/// # Panics +/// When the server cannot be spawned or does not accept connections within 15 s. +#[allow(clippy::expect_used)] +pub async fn start_nats() -> Option { + start_nats_with(&[]).await +} + +/// [`start_nats`] with extra `nats-server` arguments (`--auth`, `--user`/`--pass`, ...). +/// +/// # Panics +/// When the server cannot be spawned or does not accept connections within 15 s. +#[allow(clippy::expect_used)] +pub async fn start_nats_with(args: &[&str]) -> Option { + if !Command::new("nats-server") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + { + eprintln!("skipping: nats-server is not available on PATH"); + return None; + } + let port = std::net::TcpListener::bind("127.0.0.1:0") + .expect("free port") + .local_addr() + .expect("local addr") + .port(); + let store = tempfile::tempdir().expect("tempdir"); + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let child = NatsServer::spawn(port, store.path(), &args); + let server = NatsServer { + child, + port, + args, + url: format!("nats://127.0.0.1:{port}"), + store, + }; + wait_ready(port).await; + Some(server) +} diff --git a/crates/walgit-nats/src/wake.rs b/crates/walgit-nats/src/wake.rs new file mode 100644 index 00000000..d10dc643 --- /dev/null +++ b/crates/walgit-nats/src/wake.rs @@ -0,0 +1,135 @@ +//! Wake-ups: a bucket notification on the notify subject, and the periodic sweep. +//! Both only ever call `catch_up`, so both are idempotent. + +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use futures::StreamExt; +use walgit_events::{manifest_repo, notified_keys}; +use walgit_git::RepoId; + +use crate::publisher::Publisher; + +/// The repositories a notification body names: an S3 event (`Records[].s3.object.key`, what +/// `MinIO` and rustfs publish to NATS natively), a GCS envelope, `{"key": "..."}` or +/// `{"repo": "owner/name"}`. Anything else names nothing. +pub fn notified_repos(payload: &[u8], store_prefix: &str) -> BTreeSet { + let Ok(value) = serde_json::from_slice::(payload) else { + return BTreeSet::new(); + }; + notified_keys(&value, store_prefix) + .iter() + .filter_map(|k| manifest_repo(k, store_prefix)) + .collect() +} + +/// Subscribe to `subject` in `queue_group` (instances share the notifications) until the +/// subscription ends. A failed catch-up is logged, not retried: core NATS does not redeliver, and +/// the sweep is the backstop. +pub async fn notify_loop( + client: async_nats::Client, + subject: String, + queue_group: String, + store_prefix: String, + publisher: Arc, +) -> anyhow::Result<()> { + let mut sub = client.queue_subscribe(subject.clone(), queue_group).await?; + tracing::info!(%subject, "listening for bucket notifications"); + while let Some(msg) = sub.next().await { + for id in notified_repos(&msg.payload, &store_prefix) { + if let Err(e) = publisher.catch_up_if_present(&id).await { + tracing::warn!(repo = %id, error = %format!("{e:#}"), "notified catch-up failed; the sweep retries"); + } + } + } + Ok(()) +} + +/// One sweep at startup (quiet: it legitimately finds work), then every `every` (`0` = never again). +pub async fn sweep_loop(publisher: Arc, every: Duration) { + publisher.sweep(false).await; + if every.is_zero() { + return; + } + let mut tick = tokio::time::interval(every); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + tick.tick().await; + loop { + tick.tick().await; + publisher.sweep(true).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_notification_shape_names_the_repo() { + let key = "pre/repos/acme/mono/manifest.pb"; + for body in [ + serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", "s3": {"object": {"key": key}}}]}), + serde_json::json!({"key": key}), + serde_json::json!({"repo": "acme/mono"}), + ] { + let repos = notified_repos(body.to_string().as_bytes(), "pre/"); + let names: Vec<_> = repos.iter().map(ToString::to_string).collect(); + assert_eq!(names, ["acme/mono"], "{body}"); + } + assert!(notified_repos(b"not json", "pre/").is_empty()); + let other = serde_json::json!({"key": "pre/repos/acme/mono/wal/x.pack"}); + assert!(notified_repos(other.to_string().as_bytes(), "pre/").is_empty()); + } + + fn names(body: &serde_json::Value, prefix: &str) -> Vec { + notified_repos(body.to_string().as_bytes(), prefix) + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn an_s3_event_with_several_records_names_each_repo_once() { + let rec = |name: &str, key: &str| { + serde_json::json!({"eventName": name, "s3": {"object": {"key": key}}}) + }; + let body = serde_json::json!({"Records": [ + rec("s3:ObjectCreated:Put", "repos/a/one/manifest.pb"), + rec("s3:ObjectCreated:CompleteMultipartUpload", "repos/b/two/manifest.pb"), + rec("ObjectCreated:Put", "repos/c/three/manifest.pb"), + rec("s3:ObjectCreated:Put", "repos/a/one/manifest.pb"), + rec("s3:ObjectRemoved:Delete", "repos/d/gone/manifest.pb"), + rec("s3:ObjectCreated:Put", "repos/e/four/log/1.pb"), + rec("s3:ObjectCreated:Put", "repos/a%2Dx/five/manifest.pb"), + ]}); + assert_eq!(names(&body, ""), ["a/one", "a-x/five", "b/two", "c/three"]); + } + + #[test] + fn a_gcs_envelope_counts_only_finalize_events() { + let env = |event: &str, id: &str| { + serde_json::json!({"message": {"attributes": {"eventType": event, "objectId": id}}}) + }; + assert_eq!( + names(&env("OBJECT_FINALIZE", "repos/a/one/manifest.pb"), ""), + ["a/one"] + ); + for event in ["OBJECT_DELETE", "OBJECT_ARCHIVE", "OBJECT_METADATA_UPDATE", ""] { + assert!(names(&env(event, "repos/a/one/manifest.pb"), "").is_empty(), "{event}"); + } + } + + #[test] + fn the_store_prefix_scopes_what_is_notified() { + let key = |k: &str| serde_json::json!({"key": k}); + assert_eq!(names(&key("pre/fix/repos/a/one/manifest.pb"), "pre/fix/"), ["a/one"]); + assert!(names(&key("repos/a/one/manifest.pb"), "pre/fix/").is_empty()); + assert!(names(&key("other/repos/a/one/manifest.pb"), "pre/fix/").is_empty()); + assert_eq!(names(&serde_json::json!({"repo": "a/one"}), "pre/fix/"), ["a/one"]); + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "pre/fix/repos/a/one/manifest.pb"}}}]}); + assert_eq!(names(&s3, "pre/fix/"), ["a/one"]); + assert!(names(&s3, "").is_empty()); + } +} diff --git a/crates/walgit-nats/tests/auth.rs b/crates/walgit-nats/tests/auth.rs new file mode 100644 index 00000000..a2d5a746 --- /dev/null +++ b/crates/walgit-nats/tests/auth.rs @@ -0,0 +1,125 @@ +//! `walgit nats` authenticates with whatever the `NATS_*` variables resolve to, against a real +//! `nats-server` that requires it. + +use std::time::Duration; + +use walgit_config::NatsEventsConfig; +use walgit_nats::auth::NatsAuth; +use walgit_nats::testing::{NatsServer, start_nats_with}; + +fn cfg(nats: &NatsServer) -> NatsEventsConfig { + NatsEventsConfig { + urls: vec![nats.url.clone()], + ..NatsEventsConfig::default() + } +} + +fn env(vars: &[(&str, &str)]) -> NatsAuth { + let vars: Vec<(String, String)> = vars + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + NatsAuth::from_env(|name| vars.iter().find(|(k, _)| k == name).map(|(_, v)| v.clone())) +} + +async fn connects(nats: &NatsServer, auth: &NatsAuth) -> bool { + let attempt = tokio::time::timeout(Duration::from_secs(5), async { + let client = walgit_nats::connect(&cfg(nats), auth).await?; + client.flush().await?; + anyhow::Ok(()) + }); + matches!(attempt.await, Ok(Ok(()))) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_token_from_the_environment_authenticates() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + assert!(connects(&nats, &env(&[("NATS_TOKEN", "s3cret-token")])).await); + assert!(!connects(&nats, &env(&[("NATS_TOKEN", "wrong")])).await); + assert!(!connects(&nats, &env(&[])).await); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_user_and_password_from_the_environment_authenticate() { + let Some(nats) = start_nats_with(&["--user", "walgit", "--pass", "s3cret-pass"]).await else { + return; + }; + assert!( + connects( + &nats, + &env(&[("NATS_USER", "walgit"), ("NATS_PASSWORD", "s3cret-pass")]) + ) + .await + ); + assert!( + !connects( + &nats, + &env(&[("NATS_USER", "walgit"), ("NATS_PASSWORD", "wrong")]) + ) + .await + ); + assert!( + !connects(&nats, &env(&[("NATS_USER", "walgit")])).await, + "a user alone is no auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_nkey_seed_from_the_environment_authenticates() { + let user = nkeys::KeyPair::new_user(); + let other = nkeys::KeyPair::new_user(); + let dir = tempfile::tempdir().unwrap(); + let conf = dir.path().join("nats.conf"); + std::fs::write( + &conf, + format!( + "authorization {{ users = [ {{ nkey: {} }} ] }}\n", + user.public_key() + ), + ) + .unwrap(); + let Some(nats) = start_nats_with(&["-c", conf.to_str().unwrap()]).await else { + return; + }; + assert!(connects(&nats, &env(&[("NATS_NKEY", &user.seed().unwrap())])).await); + assert!(!connects(&nats, &env(&[("NATS_NKEY", &other.seed().unwrap())])).await); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_credentials_file_wins_and_a_missing_one_fails_naming_the_variable() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + let auth = env(&[ + ("NATS_CREDS", "/nonexistent/walgit.creds"), + ("NATS_TOKEN", "s3cret-token"), + ]); + assert!(matches!(auth, NatsAuth::Credentials(_))); + let err = walgit_nats::connect(&cfg(&nats), &auth).await.unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("NATS_CREDS") && msg.contains("/nonexistent/walgit.creds"), + "{msg}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_connection_names_the_method_but_never_the_secret() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + let auth = env(&[("NATS_TOKEN", "leaked-if-printed")]); + let err = tokio::time::timeout( + Duration::from_secs(5), + walgit_nats::connect(&cfg(&nats), &auth), + ) + .await + .expect("a rejected credential fails fast") + .unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("auth: token"), "{msg}"); + assert!(!msg.contains("leaked-if-printed"), "{msg}"); + assert!(!format!("{auth:?}").contains("leaked-if-printed")); +} diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs new file mode 100644 index 00000000..e1327212 --- /dev/null +++ b/crates/walgit-nats/tests/jetstream.rs @@ -0,0 +1,1299 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used, clippy::expect_used)] + +//! The `JetStream` publisher against a real `nats-server -js` and the in-memory walgit store: +//! the stream is the cursor (resume without republishing), crash-mid-entry completion, the +//! wake-ups and the service API. Skips (passes) when `nats-server` is not on PATH. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_nats::jetstream; +use futures::StreamExt; +use walgit_config::{Config, NatsStart}; +use walgit_events::refs_from_entries; +use walgit_git::{ObjectFormat, RepoId}; +use walgit_nats::auth::NatsAuth; +use walgit_nats::message::EventMessage; +use walgit_nats::publisher::Publisher; +use walgit_nats::subject::{RepoSubject, SubjectPrefix}; +use walgit_nats::testing::{NatsServer, start_nats}; +use walgit_proto::v1::{RefTransaction, RefUpdate}; +use async_nats::jetstream::message::PublishMessage; + +const ZERO: &str = "0000000000000000000000000000000000000000"; +const A: &str = "1111111111111111111111111111111111111111"; +const B: &str = "2222222222222222222222222222222222222222"; +const C: &str = "3333333333333333333333333333333333333333"; + +struct Rig { + cfg: Arc, + store: Arc, + registry: Arc, + client: async_nats::Client, + js: jetstream::Context, + prefix: SubjectPrefix, + _cache: tempfile::TempDir, +} + +impl Rig { + async fn new(nats: &NatsServer, start: NatsStart) -> Self { + Self::with_window(nats, start, None).await + } + + /// `window`: the stream's `duplicate_window` (default: the config's one hour). + async fn with_window(nats: &NatsServer, start: NatsStart, window: Option) -> Self { + let cache = tempfile::tempdir().unwrap(); + let mut cfg = Config::default(); + cfg.store.backend = walgit_config::StoreBackend::Memory; + cfg.cache.dir = cache.path().to_path_buf(); + cfg.events.nats.urls = vec![nats.url.clone()]; + cfg.events.nats.start = start; + if let Some(window) = window { + cfg.events.nats.duplicate_window = window; + } + let cfg = Arc::new(cfg); + let store = Arc::new(walgit_store::memory::MemoryStore::new()); + let registry = walgit_wal::Registry::new(store.clone(), cfg.clone()); + let client = walgit_nats::connect(&cfg.events.nats, &NatsAuth::None).await.unwrap(); + let js = jetstream::ContextBuilder::new() + .timeout(Duration::from_secs(3)) + .build(client.clone()); + let prefix = SubjectPrefix::new(&cfg.events.nats.subject_prefix).unwrap(); + Rig { + cfg, + store, + registry, + client, + js, + prefix, + _cache: cache, + } + } + + /// A publisher with no memory of earlier ones, as after a restart. + async fn publisher(&self, start: NatsStart) -> Arc { + let stream = walgit_nats::open_stream(&self.js, &self.cfg.events.nats, &self.prefix) + .await + .unwrap(); + Publisher::new( + self.registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + start, + ) + } + + /// What `walgit nats` runs: its own registry over the same store (it never sees the + /// writer's pushes except through the bucket) on the operator's `[wal]` settings, as + /// `run` adapts them. Returns the registry's cache dir guard too. + async fn operator_publisher( + &self, + operator: impl FnOnce(&mut Config), + ) -> (Arc, Arc, tempfile::TempDir) { + let cache = tempfile::tempdir().unwrap(); + let mut cfg = (*self.cfg).clone(); + cfg.cache.dir = cache.path().to_path_buf(); + operator(&mut cfg); + let registry = walgit_wal::Registry::new( + self.store.clone(), + Arc::new(walgit_nats::publisher_config(&cfg)), + ); + let stream = walgit_nats::open_stream(&self.js, &self.cfg.events.nats, &self.prefix) + .await + .unwrap(); + let publisher = Publisher::new( + registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + NatsStart::Oldest, + ); + (publisher, registry, cache) + } + + async fn push(&self, id: &RepoId, updates: &[(&str, &str, &str)]) -> u64 { + let handle = self.registry.open(id).await.unwrap(); + let txn = RefTransaction { + updates: updates + .iter() + .map(|(name, old, new)| RefUpdate { + name: (*name).into(), + old_oid: (*old).into(), + new_oid: (*new).into(), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + let meta = HashMap::from([ + ("principal".to_string(), "alice".to_string()), + ("request_id".to_string(), "rid".to_string()), + ]); + let r = handle.publish_ref_update(txn, meta).await.unwrap(); + assert!(r.seq > 0, "push rejected: {:?}", r.per_ref); + r.seq + } + + /// The events (one per ref) that WAL entry `seq` yields. + async fn events_of(&self, id: &RepoId, seq: u64) -> Vec { + let handle = self.registry.open(id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, seq, Some(seq)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(id, &entries, &mut events); + events + } + + /// Publish a message on the repo's subject as walgit would, but for a ref the entry does not + /// contain: the stream's last message for the subject then names something `catch_up` cannot + /// find in the entry, which forces it to republish the entry. Returns its stream sequence. + async fn inject(&self, id: &RepoId, seq: u64, ref_name: &str) -> u64 { + let msg = PublishMessage::build() + .payload("{}".into()) + .message_id(format!("{id}:{seq}:{ref_name}")) + .header("Walgit-Seq", seq.to_string().as_str()) + .header("Walgit-Repo", id.to_string().as_str()); + let subject = RepoSubject::new(&self.prefix, id).to_string(); + self.js + .send_publish(subject, msg) + .await + .unwrap() + .await + .unwrap() + .sequence + } + + /// `Nats-Msg-Id` of every message in the stream, in stream order. + async fn ids(&self) -> Vec { + let stream = self.stream().await; + let state = stream.get_info().await.unwrap().state; + let mut out = Vec::new(); + if state.messages == 0 { + return out; + } + for seq in state.first_sequence..=state.last_sequence { + if let Ok(raw) = stream.get_raw_message(seq).await { + out.push( + raw.headers + .get("Nats-Msg-Id") + .map(|v| v.as_str().to_string()) + .unwrap_or_default(), + ); + } + } + out + } + + async fn wait_connected(&self) { + let t0 = Instant::now(); + while self.client.connection_state() != async_nats::connection::State::Connected { + assert!(t0.elapsed() < Duration::from_secs(20), "client did not reconnect"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + fn spawn_notify(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + let nats_cfg = &self.cfg.events.nats; + tokio::spawn(walgit_nats::wake::notify_loop( + self.client.clone(), + nats_cfg.notify_subject.clone(), + nats_cfg.queue_group.clone(), + self.cfg.store_prefix(), + publisher.clone(), + )) + } + + fn spawn_service(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + let (client, prefix, publisher) = (self.client.clone(), self.prefix.clone(), publisher.clone()); + let queue_group = self.cfg.events.nats.queue_group.clone(); + tokio::spawn(async move { + walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await + }) + } + + async fn notify(&self, body: &str) { + self.client + .publish(self.cfg.events.nats.notify_subject.clone(), body.to_string().into()) + .await + .unwrap(); + } + + async fn stream(&self) -> jetstream::stream::Stream { + self.js + .get_stream(&self.cfg.events.nats.stream) + .await + .unwrap() + } + + async fn message_count(&self) -> u64 { + self.stream().await.get_info().await.unwrap().state.messages + } + + async fn wait_for_messages(&self, n: u64) { + let t0 = Instant::now(); + while self.message_count().await < n { + assert!(t0.elapsed() < Duration::from_secs(10), "timed out waiting for {n} messages"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stream_is_the_cursor() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("acme.io", "mono.repo").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + rig.push(&id, &[("refs/heads/main", A, B)]).await; + + let publisher = rig.publisher(NatsStart::Oldest).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.head_seq, r.emitted, r.conflict), (0, 2, 2, false)); + assert_eq!(rig.message_count().await, 2); + + let stream = rig.stream().await; + let first = stream.get_raw_message(1).await.unwrap(); + assert_eq!(first.subject.as_str(), "walgit.ref.acme%2Eio.mono%2Erepo"); + assert_eq!(first.headers.get("Walgit-Seq").unwrap().as_str(), "1"); + assert_eq!(first.headers.get("Walgit-Repo").unwrap().as_str(), "acme.io/mono.repo"); + assert_eq!( + first.headers.get("Nats-Msg-Id").unwrap().as_str(), + "acme.io/mono.repo:1:refs/heads/main" + ); + let body: serde_json::Value = serde_json::from_slice(&first.payload).unwrap(); + assert_eq!(body["action"], "create"); + assert_eq!(body["old"], ZERO); + assert_eq!(body["new"], A); + assert_eq!(body["repo"], "acme.io/mono.repo"); + assert_eq!(body["pusher"], "alice"); + assert_eq!(body["_walgit"]["seq"], "1"); + let second = stream.get_raw_message(2).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&second.payload).unwrap(); + assert_eq!((body["action"].as_str(), body["_walgit"]["seq"].as_str()), (Some("update"), Some("2"))); + + // Restart: a publisher with no memory resumes from the stream and publishes nothing. + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted), (2, 0)); + assert_eq!(rig.message_count().await, 2, "resume must not republish"); + + // New pushes are published, and only those. + rig.push(&id, &[("refs/heads/main", B, ZERO)]).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 3, 1)); + assert_eq!(rig.message_count().await, 3); + let third = stream.get_raw_message(3).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&third.payload).unwrap(); + assert_eq!((body["action"].as_str(), body["new"].as_str()), (Some("delete"), Some(ZERO))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_crash_mid_entry_finishes_that_entry_only() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + // One entry, three refs; the "crash" leaves only the first event in the stream. + let seq = rig + .push( + &id, + &[ + ("refs/heads/a", ZERO, B), + ("refs/heads/b", ZERO, B), + ("refs/tags/c", ZERO, C), + ], + ) + .await; + let handle = rig.registry.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, seq, Some(seq)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(&id, &entries, &mut events); + assert_eq!(events.len(), 3); + let msg = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &events[0]).unwrap(); + let subject = msg.subject.to_string(); + rig.js + .send_publish(subject, msg.into_publish()) + .await + .unwrap() + .await + .unwrap(); + assert_eq!(rig.message_count().await, 2); + + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted), (seq, 2), "the rest of the entry, nothing twice"); + assert_eq!(rig.message_count().await, 4); + let last = rig.stream().await.get_raw_message(4).await.unwrap(); + assert_eq!( + last.headers.get("Nats-Msg-Id").unwrap().as_str(), + format!("t/r:{seq}:refs/tags/c") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn head_start_publishes_nothing_older() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Head).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); + assert_eq!(rig.message_count().await, 0); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 1); + let m = rig.stream().await.get_raw_message(1).await.unwrap(); + assert_eq!(m.headers.get("Walgit-Seq").unwrap().as_str(), "2"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_publisher_that_moved_first_stops_this_one() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + publisher.catch_up(&id).await.unwrap(); + + // The subject moves (another instance published a different entry) between our read of the + // last message and our first publish: emulate by publishing a foreign message afterwards, + // with a stale expectation. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + let handle = rig.registry.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, 2, Some(2)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(&id, &entries, &mut events); + let stale = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &events[0]) + .unwrap() + .expecting_last_subject_sequence(0); + let subject = stale.subject.to_string(); + let err = rig + .js + .send_publish(subject, stale.into_publish()) + .await + .unwrap() + .await + .unwrap_err(); + assert_eq!(err.kind(), jetstream::context::PublishErrorKind::WrongLastSequence); + assert_eq!(rig.message_count().await, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn notifications_and_the_service_api_wake_the_publisher() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let publisher = rig.publisher(NatsStart::Oldest).await; + + let nats_cfg = &rig.cfg.events.nats; + let notify = tokio::spawn(walgit_nats::wake::notify_loop( + rig.client.clone(), + nats_cfg.notify_subject.clone(), + nats_cfg.queue_group.clone(), + rig.cfg.store_prefix(), + publisher.clone(), + )); + let service = tokio::spawn({ + let (client, prefix, publisher) = + (rig.client.clone(), rig.prefix.clone(), publisher.clone()); + let queue_group = nats_cfg.queue_group.clone(); + async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } + }); + tokio::time::sleep(Duration::from_millis(500)).await; + + // An S3 bucket notification, as the object store publishes it. + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "repos/t/r/manifest.pb"}}}]}); + rig.client + .publish(nats_cfg.notify_subject.clone(), s3.to_string().into()) + .await + .unwrap(); + rig.wait_for_messages(1).await; + + // {"repo": ...} through the same subject. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.client + .publish(nats_cfg.notify_subject.clone(), r#"{"repo":"t/r"}"#.into()) + .await + .unwrap(); + rig.wait_for_messages(2).await; + + // The service endpoint. + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!((report["repo"].as_str(), report["emitted"].as_u64()), (Some("t/r"), Some(1))); + assert_eq!(rig.message_count().await, 3); + + let missing = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/none"}"#.into()) + .await + .unwrap(); + let code = missing.headers.as_ref().and_then(|h| h.get("Nats-Service-Error-Code")); + assert_eq!(code.map(|c| c.as_str().to_string()), Some("404".into())); + + // The micro protocol's built-ins answer too. + let mut pings = rig.client.subscribe("inbox.ping").await.unwrap(); + rig.client + .publish_with_reply("$SRV.PING.walgit-events", "inbox.ping", "".into()) + .await + .unwrap(); + let pong = tokio::time::timeout(Duration::from_secs(5), pings.next()) + .await + .unwrap() + .unwrap(); + let pong: serde_json::Value = serde_json::from_slice(&pong.payload).unwrap(); + assert_eq!(pong["name"], "walgit-events"); + + notify.abort(); + service.abort(); +} + +// --------------------------------------------------------------------------------------------- +// Dedup, races, wipes, gaps, outages and the wake-ups that must be ignored. +// --------------------------------------------------------------------------------------------- + +async fn repo_with_three_events(rig: &Rig, owner: &str, name: &str) -> RepoId { + let id = RepoId::new(owner, name).unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + rig.push(&id, &[("refs/heads/a", ZERO, B), ("refs/heads/b", ZERO, B)]).await; + id +} + +fn msg_id(id: &RepoId, seq: u64, name: &str) -> String { + format!("{id}:{seq}:{name}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn broker_dedup_absorbs_a_republish_inside_the_window() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let first = rig.publisher(NatsStart::Oldest).await; + assert_eq!(first.catch_up(&id).await.unwrap().emitted, 3); + + // A hand-built republish of an existing Nats-Msg-Id is acknowledged as a duplicate. + let event = rig.events_of(&id, 1).await.remove(0); + let msg = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &event).unwrap(); + let subject = msg.subject.to_string(); + let ack = rig.js.send_publish(subject, msg.into_publish()).await.unwrap().await.unwrap(); + assert!(ack.duplicate, "same Nats-Msg-Id inside the window must be a duplicate"); + assert_eq!(rig.message_count().await, 3); + + // The real path: the last message on the subject names a ref the entry does not contain, so + // `after_last_published` cannot find where the stream got to and republishes the whole entry. + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + assert_eq!(rig.message_count().await, 4); + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted, r.conflict), (2, 2, false), "{r:?}"); + let counters = restarted.counters.snapshot(); + assert_eq!(counters["deduplicated"], 2, "both republished events were broker duplicates"); + assert_eq!(counters["published"], 0); + assert_eq!(rig.message_count().await, 4, "the stream did not grow"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn racing_publishers_leave_each_event_exactly_once() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let mut expected = Vec::new(); + let mut ids = Vec::new(); + for round in 0..6 { + for n in 0..3 { + let id = RepoId::new("race", format!("r{round}x{n}").as_str()).unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + for seq in 1..=4u64 { + let a = format!("refs/heads/a{seq}"); + let b = format!("refs/heads/b{seq}"); + rig.push(&id, &[(a.as_str(), ZERO, A), (b.as_str(), ZERO, B)]).await; + expected.push(msg_id(&id, seq, &a)); + expected.push(msg_id(&id, seq, &b)); + } + ids.push(id); + } + } + let one = rig.publisher(NatsStart::Oldest).await; + let two = rig.publisher(NatsStart::Oldest).await; + let mut reports = Vec::new(); + for batch in ids.chunks(3) { + let (a, b) = (batch[0].clone(), batch[1].clone()); + let c = batch[2].clone(); + let runs = tokio::join!( + one.catch_up(&a), + two.catch_up(&a), + one.catch_up(&b), + two.catch_up(&b), + two.catch_up(&c), + one.catch_up(&c), + ); + for r in [runs.0, runs.1, runs.2, runs.3, runs.4, runs.5] { + reports.push(r.expect("a lost race is a conflict or a dedup, never an error")); + } + } + let mut seen = rig.ids().await; + let total = seen.len(); + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), total, "a (repo, seq, ref) landed twice"); + expected.sort(); + assert_eq!(seen, expected, "every event exactly once"); + + let (one, two) = (one.counters.snapshot(), two.counters.snapshot()); + let sum = |key: &str| one[key].as_u64().unwrap() + two[key].as_u64().unwrap(); + assert_eq!(sum("published"), total as u64, "each landed message was counted once"); + let conflicts = reports.iter().filter(|r| r.conflict).count() as u64; + assert_eq!(sum("conflicts"), conflicts); + assert!(conflicts + sum("deduplicated") > 0, "the publishers never actually raced"); + eprintln!( + "race: {conflicts} conflicts, {} deduplicated over {} catch-ups", + sum("deduplicated"), + reports.len() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn after_the_dedup_window_a_republish_lands_twice() { + // At-least-once, documented: JetStream only dedups inside `duplicate_window`, so a consumer + // must dedup on (repo, _walgit.seq, ref_name) itself. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_millis(200))).await; + let id = repo_with_three_events(&rig, "t", "r").await; + assert_eq!(rig.publisher(NatsStart::Oldest).await.catch_up(&id).await.unwrap().emitted, 3); + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 2); + let counters = restarted.counters.snapshot(); + assert_eq!((counters["published"].as_u64(), counters["deduplicated"].as_u64()), (Some(2), Some(0))); + assert_eq!(rig.message_count().await, 6, "both republished events landed a second time"); + let ids = rig.ids().await; + for name in ["refs/heads/a", "refs/heads/b"] { + let id = msg_id(&id, 2, name); + assert_eq!(ids.iter().filter(|i| **i == id).count(), 2, "{id} must be in the stream twice"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_wiped_stream_is_republished_on_the_next_catch_up() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + let original = rig.ids().await; + + // Purged: JetStream keeps its duplicate table across a purge, so inside `duplicate_window` + // the republish is acknowledged as duplicates and lands nothing. PINNED known limitation; the + // repository stays open (not recorded as done) so it heals once the window has passed, see + // `a_purged_stream_heals_once_the_dedup_window_has_passed`. + rig.stream().await.purge().await.unwrap(); + assert_eq!(rig.message_count().await, 0); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "{r:?}"); + assert_eq!(publisher.counters.snapshot()["deduplicated"], 3, "swallowed as duplicates of purged messages"); + assert_eq!(rig.message_count().await, 0, "PINNED: a purge inside the window loses the republish"); + + // Deleted: nothing to publish into (an error, not a silent skip) ... + rig.js.delete_stream(&rig.cfg.events.nats.stream).await.unwrap(); + assert!(publisher.catch_up(&id).await.is_err()); + // ... and recreated, the window is republished again. + walgit_nats::open_stream(&rig.js, &rig.cfg.events.nats, &rig.prefix).await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "a recreated stream is republished: {r:?}"); + assert_eq!(rig.ids().await, original); + + // A steady state still spares the log read and publishes nothing. + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); + assert_eq!(rig.message_count().await, 3); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_purged_stream_heals_once_the_dedup_window_has_passed() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_millis(200))).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + let original = rig.ids().await; + + rig.stream().await.purge().await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "{r:?}"); + assert_eq!(rig.ids().await, original, "the same running publisher republished the window"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0, "and then it is steady"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn entries_folded_before_publishing_are_a_counted_gap() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + rig.push(&id, &[("refs/heads/main", C, A)]).await; + let cp = rig.registry.open(&id).await.unwrap().write_checkpoint().await.unwrap(); + assert_eq!(cp.seq, 4, "seqs 2..=4 are folded away before they were published"); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.gap, Some([2, 4]), "{r:?}"); + assert_eq!((r.head_seq, r.emitted), (5, 1), "the readable entry is still published"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 3); + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 5, "refs/heads/main")]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_gap_is_counted_once_while_nothing_moves_the_subject() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let handle = rig.registry.open(&id).await.unwrap(); + handle.write_checkpoint().await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.gap, r.emitted), (Some([2, 3]), 0), "{r:?}"); + + handle.publish_settings("", "test", "moves the head without a ref event").await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert!(r.head_seq > 3, "{r:?}"); + assert_eq!((r.gap, r.emitted), (None, 0), "the same gap is not reported again: {r:?}"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 2); + + rig.push(&id, &[("refs/heads/main", C, A)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.gap, r.emitted), (None, 1), "{r:?}"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 2); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_outage_mid_catch_up_loses_and_duplicates_nothing() { + const REFS: usize = 1500; + let Some(mut nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "big").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let names: Vec = (0..REFS).map(|i| format!("refs/heads/b{i:04}")).collect(); + let updates: Vec<(&str, &str, &str)> = names.iter().map(|n| (n.as_str(), ZERO, A)).collect(); + // One entry with many refs: a single catch_up has plenty to publish one ack at a time. + rig.push(&id, &updates).await; + + let publisher = rig.publisher(NatsStart::Oldest).await; + let running = tokio::spawn({ + let (publisher, id) = (publisher.clone(), id.clone()); + async move { publisher.catch_up(&id).await } + }); + while rig.message_count().await < 20 { + tokio::time::sleep(Duration::from_millis(2)).await; + } + nats.stop(); + let outcome = running.await.unwrap(); + let error = outcome.expect_err("the server went away mid catch-up"); + eprintln!("outage error: {error:#}"); + + nats.restart().await; + rig.wait_connected().await; + let held = rig.message_count().await; + assert!(held >= 20 && held < REFS as u64, "partial progress survived the restart: {held}"); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted as u64, REFS as u64 - held, "exactly the rest: {r:?}"); + let mut ids = rig.ids().await; + assert_eq!(ids.len(), REFS); + let expected: Vec = names.iter().map(|n| msg_id(&id, 1, n)).collect(); + assert_eq!(ids, expected, "in order, nothing skipped, nothing twice"); + ids.sort(); + ids.dedup(); + assert_eq!(ids.len(), REFS); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn notifications_that_name_nothing_are_ignored_and_the_loop_survives() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let notify = rig.spawn_notify(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + + for body in [ + r#"{"key":"repos/t/r/wal/abc.pack"}"#, + r#"{"key":"somewhere/else/manifest.pb"}"#, + r#"{"repo":"t/ghost"}"#, + r#"{"key":"repos/t/ghost/manifest.pb"}"#, + r#"{"repo":"not a repo id"}"#, + r#"{"Records":[{"s3":{"object":{"key":"repos/t/ghost/manifest.pb"}}}]}"#, + "not json", + "{", + "", + "[]", + "null", + "\"a string\"", + ] { + rig.notify(body).await; + } + rig.client.publish(rig.cfg.events.nats.notify_subject.clone(), vec![0xff, 0xfe, 0x00].into()).await.unwrap(); + rig.client.flush().await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(!notify.is_finished(), "the notify loop must survive garbage"); + assert_eq!(rig.message_count().await, 0, "nothing was published for any of them"); + let repos: Vec = rig.registry.list().await.unwrap().iter().map(ToString::to_string).collect(); + assert_eq!(repos, ["t/r"], "an unknown repository is not created by being notified"); + + // A later valid notification still works. + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + assert!(!notify.is_finished()); + notify.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn known_limitation_head_start_skips_pushes_made_while_the_publisher_was_down() { + // PINNED: docs/EVENTS.md says `start = "head"` publishes only pushes made after the publisher + // first saw the repository. The mark is in memory, so a restart before the repository's first + // event treats everything pushed while the publisher was down as "before". If this test fails + // because the behavior changed, update the `start` paragraph of docs/EVENTS.md with it. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let first = rig.publisher(NatsStart::Head).await; + assert_eq!(first.catch_up(&id).await.unwrap().emitted, 0, "head start: history is not published"); + + // The publisher goes down; these two pushes happen while it is gone. + drop(first); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + + let restarted = rig.publisher(NatsStart::Head).await; + assert_eq!(restarted.catch_up(&id).await.unwrap().emitted, 0, "seqs 2 and 3 are skipped forever"); + rig.push(&id, &[("refs/heads/main", C, A)]).await; + assert_eq!(restarted.catch_up(&id).await.unwrap().emitted, 1); + assert_eq!(rig.ids().await, [msg_id(&id, 4, "refs/heads/main")]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_foreign_message_on_a_repo_subject_fails_every_catch_up_with_a_clear_error() { + // PINNED operational risk: a message without Walgit-Seq / Nats-Msg-Id as the subject's last + // message makes every catch-up for that repository fail until it is removed from the stream. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + rig.js.publish("walgit.ref.t.r", "hello".into()).await.unwrap().await.unwrap(); + for _ in 0..3 { + let error = format!("{:#}", publisher.catch_up(&id).await.unwrap_err()); + assert!( + error.contains("walgit.ref.t.r") && error.contains("lacks Walgit-Seq / Nats-Msg-Id"), + "the error must say what is wrong: {error}" + ); + } + assert_eq!(rig.message_count().await, 1, "nothing was published on top of it"); + + // Other repositories are unaffected. + let other = RepoId::new("t", "other").unwrap(); + rig.registry.create(&other, ObjectFormat::Sha1).await.unwrap(); + rig.push(&other, &[("refs/heads/main", ZERO, A)]).await; + assert_eq!(publisher.catch_up(&other).await.unwrap().emitted, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_nats_server_restart_keeps_the_stream_the_subscriptions_and_the_service() { + let Some(mut nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let notify = rig.spawn_notify(&publisher); + let service = rig.spawn_service(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + + nats.restart().await; + rig.wait_connected().await; + assert_eq!(rig.message_count().await, 1, "the stream survives the restart"); + + // The queue subscription is back: a notification after the restart triggers a catch-up. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(2).await; + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 2, "refs/heads/main")]); + assert!(!notify.is_finished() && !service.is_finished()); + + // The $SRV service is back too. + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let reply = tokio::time::timeout( + Duration::from_secs(5), + rig.client.request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()), + ) + .await + .unwrap() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!(report["emitted"], 1, "{report}"); + assert_eq!(rig.message_count().await, 3, "nothing was republished"); + notify.abort(); + service.abort(); +} + +fn error_code(reply: &async_nats::Message) -> Option { + reply + .headers + .as_ref() + .and_then(|h| h.get("Nats-Service-Error-Code")) + .map(|c| c.as_str().to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_service_answers_bad_requests_and_reports_the_counters() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let service = rig.spawn_service(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + + for (body, code) in [ + ("not json", "400"), + ("{}", "400"), + (r#"{"repo":42}"#, "400"), + (r#"{"repo":"no-slash"}"#, "400"), + (r#"{"repo":"a/b/c"}"#, "400"), + (r#"{"repo":"t/none"}"#, "404"), + ] { + let reply = rig.client.request("walgit.events.catch-up", body.into()).await.unwrap(); + assert_eq!(error_code(&reply).as_deref(), Some(code), "{body}"); + } + + // published: 3 through the endpoint. + let reply = rig.client.request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()).await.unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!((report["emitted"].as_u64(), report["head_seq"].as_u64()), (Some(3), Some(2))); + + // deduplicated: 2, from forcing a republish of the entry inside the window. + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + let restarted = rig.publisher(NatsStart::Oldest).await; + restarted.catch_up(&id).await.unwrap(); + + // gap_entries: 3, on a second repository. + let gap = RepoId::new("t", "gap").unwrap(); + rig.registry.create(&gap, ObjectFormat::Sha1).await.unwrap(); + rig.push(&gap, &[("refs/heads/main", ZERO, A)]).await; + assert_eq!(publisher.catch_up(&gap).await.unwrap().emitted, 1); + for (old, new) in [(A, B), (B, C), (C, A)] { + rig.push(&gap, &[("refs/heads/main", old, new)]).await; + } + rig.registry.open(&gap).await.unwrap().write_checkpoint().await.unwrap(); + rig.push(&gap, &[("refs/heads/main", A, B)]).await; + assert_eq!(publisher.catch_up(&gap).await.unwrap().gap, Some([2, 4])); + + // The service's publisher is `publisher`: its counters are what $SRV.STATS reports. + let stats = rig.client.request("$SRV.STATS.walgit-events", "".into()).await.unwrap(); + let stats: serde_json::Value = serde_json::from_slice(&stats.payload).unwrap(); + eprintln!("stats: {stats}"); + let data = &stats["endpoints"][0]["data"]; + assert_eq!(stats["endpoints"][0]["subject"], "walgit.events.catch-up"); + assert_eq!(data["published"], 5, "3 + 1 + 1"); + assert_eq!(data["deduplicated"], 0, "the duplicates were counted by the other publisher"); + assert_eq!(data["gap_entries"], 3); + assert_eq!(data["conflicts"], 0); + assert_eq!(data["sweep_found"], 0); + assert_eq!(restarted.counters.snapshot()["deduplicated"], 2); + service.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_periodic_sweep_finds_a_push_nobody_notified() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let sweeping = tokio::spawn(walgit_nats::wake::sweep_loop(publisher.clone(), Duration::from_millis(300))); + + // The startup sweep legitimately finds work and does not count it. + rig.wait_for_messages(1).await; + assert_eq!(publisher.counters.snapshot()["sweep_found"], 0); + + // No notification for this push: only the periodic sweep can find it. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.wait_for_messages(2).await; + // The sweep counts what it found once its catch-up returns, after the message is acknowledged. + let t0 = Instant::now(); + while publisher.counters.snapshot()["sweep_found"] == 0 && t0.elapsed() < Duration::from_secs(10) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(publisher.counters.snapshot()["sweep_found"], 1); + sweeping.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_notification_inside_the_freshness_ttl_still_publishes_the_push() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let (publisher, _registry, _cache) = rig + .operator_publisher(|c| c.wal.freshness_ttl = Duration::from_mins(10)) + .await; + let notify = rig.spawn_notify(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + + // Pushed through another registry: the publisher only learns of it from the bucket. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(2).await; + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 2, "refs/heads/main")]); + notify.abort(); +} + +fn pack_files(dir: &std::path::Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|e| e == "pack") { + found.push(path); + } + } + } + found +} + +/// A push carrying a real pack (one commit), so the repository has pack bytes in the bucket. +async fn push_with_pack(rig: &Rig, id: &RepoId) { + let work = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let out = std::process::Command::new("git") + .args(["-c", "user.name=T", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(work.path()) + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).unwrap().trim().to_string() + }; + git(&["init", "-q"]); + std::fs::write(work.path().join("f"), "x").unwrap(); + git(&["add", "."]); + git(&["commit", "-q", "-m", "c"]); + let head = git(&["rev-parse", "HEAD"]); + let pack = { + use std::io::Write as _; + let mut child = std::process::Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(work.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(format!("{head}\n").as_bytes()).unwrap(); + child.wait_with_output().unwrap().stdout + }; + let handle = rig.registry.open(id).await.unwrap(); + let ingested = handle + .local() + .ingest_pack( + std::io::Cursor::new(pack), + walgit_git::IngestOptions { fsck: false, max_bytes: None, thin: false }, + ) + .await + .unwrap(); + let txn = RefTransaction { + updates: vec![RefUpdate { + name: "refs/heads/main".into(), + old_oid: String::new(), + new_oid: head, + ..Default::default() + }], + atomic: true, + ..Default::default() + }; + let r = handle.publish_push(ingested, txn, HashMap::new()).await.unwrap(); + assert!(r.seq > 0, "push rejected: {:?}", r.per_ref); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_publisher_never_prefetches_packs_whatever_the_operator_configured() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + push_with_pack(&rig, &id).await; + + let (publisher, _registry, cache) = rig + .operator_publisher(|c| c.wal.prefetch_packs = true) + .await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(pack_files(cache.path()), Vec::::new(), "no pack bytes pulled"); + + // Control: the same operator config without the adaptation does prefetch. + let control_cache = tempfile::tempdir().unwrap(); + let mut operator = (*rig.cfg).clone(); + operator.cache.dir = control_cache.path().to_path_buf(); + operator.wal.prefetch_packs = true; + let plain = walgit_wal::Registry::new(rig.store.clone(), Arc::new(operator)); + let handle = plain.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let t0 = Instant::now(); + while pack_files(control_cache.path()).is_empty() { + assert!(t0.elapsed() < Duration::from_secs(10), "the control never prefetched"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_missing_stream_with_ensure_stream_off_fails_clearly_and_creates_nothing() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let mut cfg = rig.cfg.events.nats.clone(); + cfg.ensure_stream = false; + let err = walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap_err(); + let text = format!("{err:#}"); + assert!( + text.contains("WALGIT_EVENTS") && text.contains("ensure_stream is off"), + "the error must name the stream and the setting: {text}" + ); + assert!(rig.js.get_stream("WALGIT_EVENTS").await.is_err(), "nothing was created"); + + cfg.ensure_stream = true; + walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap(); + cfg.ensure_stream = false; + walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_existing_stream_with_other_settings_is_reused_not_clobbered() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let existing = jetstream::stream::Config { + name: "WALGIT_EVENTS".into(), + subjects: vec!["walgit.ref.>".into(), "other.>".into()], + max_messages: 1000, + duplicate_window: Duration::from_secs(30), + ..Default::default() + }; + rig.js.create_stream(existing).await.unwrap(); + + let stream = walgit_nats::open_stream(&rig.js, &rig.cfg.events.nats, &rig.prefix) + .await + .unwrap(); + let config = stream.cached_info().config.clone(); + assert_eq!(config.max_messages, 1000); + assert_eq!(config.duplicate_window, Duration::from_secs(30)); + assert_eq!(config.subjects, ["walgit.ref.>", "other.>"]); + + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_unparsable_server_url_fails_to_connect_naming_the_setting() { + let cfg = walgit_config::NatsEventsConfig { + urls: vec!["nats://bad host:not-a-port".into()], + ..Default::default() + }; + let err = walgit_nats::connect(&cfg, &NatsAuth::None).await.unwrap_err(); + assert!(format!("{err:#}").contains("events.nats.urls"), "{err:#}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn tags_and_deletes_carry_their_ref_type_and_zero_oids() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A), ("refs/tags/v1", ZERO, B)]).await; + rig.push(&id, &[("refs/heads/main", A, ZERO), ("refs/tags/v1", B, ZERO)]).await; + rig.push(&id, &[("refs/notes/x", ZERO, C)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 5); + + let stream = rig.stream().await; + let mut seen = Vec::new(); + for n in 1..=5 { + let raw = stream.get_raw_message(n).await.unwrap(); + let b: serde_json::Value = serde_json::from_slice(&raw.payload).unwrap(); + let get = |k: &str| b[k].as_str().unwrap().to_string(); + seen.push((get("action"), get("ref_type"), get("ref_name"), get("old"), get("new"))); + } + let row = |a: &str, t: &str, r: &str, o: &str, n: &str| { + (a.to_string(), t.to_string(), r.to_string(), o.to_string(), n.to_string()) + }; + assert_eq!( + seen, + [ + row("create", "branch", "refs/heads/main", ZERO, A), + row("create", "tag", "refs/tags/v1", ZERO, B), + row("delete", "branch", "refs/heads/main", A, ZERO), + row("delete", "tag", "refs/tags/v1", B, ZERO), + row("create", "", "refs/notes/x", ZERO, C), + ] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn many_sequential_pushes_land_in_seq_order_across_catch_ups() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let publisher = rig.publisher(NatsStart::Oldest).await; + let oid = |i: usize| format!("{i:040x}"); + let mut prev = ZERO.to_string(); + let mut expected = Vec::new(); + for i in 1..=30usize { + let next = oid(i); + let seq = rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + expected.push(msg_id(&id, seq, "refs/heads/main")); + prev = next; + if i % 7 == 0 { + publisher.catch_up(&id).await.unwrap(); + } + } + publisher.catch_up(&id).await.unwrap(); + assert_eq!(rig.ids().await, expected); +} + +async fn published_then_recreated(rig: &Rig, publisher: &Publisher) -> RepoId { + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let mut prev = ZERO.to_string(); + for i in 1..=3usize { + let next = format!("{i:040x}"); + rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + prev = next; + } + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + rig.registry.delete(&id).await.unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + id +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_is_published_as_a_new_one() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let id = published_then_recreated(&rig, &publisher).await; + let seq = rig.push(&id, &[("refs/heads/fresh", ZERO, A)]).await; + assert_eq!(seq, 1, "the new incarnation starts again at seq 1"); + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.head_seq, r.emitted, r.conflict), (1, 1, false), "{r:?}"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); + let ids = rig.ids().await; + assert_eq!(ids.len(), 4); + assert_eq!(ids.last().unwrap(), &msg_id(&id, 1, "refs/heads/fresh")); + let again = publisher.catch_up(&id).await.unwrap(); + assert_eq!(again.emitted, 0, "steady afterwards, not rewound again"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_under_head_start_publishes_pushes_after_its_first_sighting() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let publisher = rig.publisher(NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0, "first sighting at head 0"); + let mut prev = ZERO.to_string(); + for i in 1..=3usize { + let next = format!("{i:040x}"); + rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + prev = next; + } + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + rig.registry.delete(&id).await.unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + + rig.push(&id, &[("refs/heads/before", ZERO, A)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 0, "head start: what the new incarnation held when first seen is skipped"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); + + rig.push(&id, &[("refs/heads/after", ZERO, B)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false), "{r:?}"); + assert_eq!(rig.ids().await.last().unwrap(), &msg_id(&id, 2, "refs/heads/after")); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1, "the same rewind is not counted twice"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window() { + // PINNED: the new incarnation's first event has the old incarnation's Nats-Msg-Id + // (`t/r:1:refs/heads/main`), so inside the window the broker swallows it, the catch-up is not + // confirmed and the repository stays open until the window has passed. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_secs(2))).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let id = published_then_recreated(&rig, &publisher).await; + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false)); + assert_eq!(publisher.counters.snapshot()["deduplicated"], 1); + assert_eq!(rig.message_count().await, 3, "swallowed as a duplicate of the old one"); + + tokio::time::sleep(Duration::from_secs(4)).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false), "{r:?}"); + assert_eq!(rig.message_count().await, 4, "it landed once the window had passed"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); +} diff --git a/crates/walgit-server/Cargo.toml b/crates/walgit-server/Cargo.toml index 9ad63412..5f018659 100644 --- a/crates/walgit-server/Cargo.toml +++ b/crates/walgit-server/Cargo.toml @@ -9,6 +9,7 @@ humantime.workspace = true walgit-proto.workspace = true walgit-store.workspace = true walgit-config.workspace = true +walgit-events.workspace = true walgit-git.workspace = true walgit-wal.workspace = true async-trait.workspace = true @@ -65,6 +66,8 @@ rcgen.workspace = true tokio-rustls.workspace = true tempfile.workspace = true [dev-dependencies] +async-nats.workspace = true +walgit-nats = { workspace = true, features = ["testing"] } reqwest.workspace = true rcgen = { workspace = true, features = ["x509-parser"] } diff --git a/crates/walgit-server/src/bridge.rs b/crates/walgit-server/src/bridge.rs index c51ec2fb..f508682f 100644 --- a/crates/walgit-server/src/bridge.rs +++ b/crates/walgit-server/src/bridge.rs @@ -29,7 +29,9 @@ use chrono::Utc; use walgit_git::RepoId; use walgit_store::{ObjectStoreExt, PutMode, StoreError}; -use crate::events::{self, RefEvent, Sink}; +use walgit_events::{RefEvent, manifest_repo, notified_keys, refs_from_entries}; + +use crate::events::{self, Sink}; const CURSOR_KEY: &str = "events/cursor.json"; @@ -141,7 +143,7 @@ impl Bridge { if head > from { let entries = handle.read_log(&manifest, from + 1, Some(head)).await?; let mut batch = Vec::new(); - events::refs_from_entries(id, &entries, &mut batch); + refs_from_entries(id, &entries, &mut batch); // A failing sink returns here: cursor untouched, retried on the // next wake-up. self.publish(&batch).await?; @@ -205,7 +207,7 @@ impl Bridge { /// A GCS notification: `…/repos///manifest.pb` finalized ⇒ that /// repo committed something. Everything else is ignored. pub async fn object_finalized(&self, object: &str) -> anyhow::Result> { - let Some(id) = self.manifest_repo(object) else { + let Some(id) = manifest_repo(object, &self.store_prefix) else { return Ok(None); }; match self.catch_up(&id).await { @@ -223,19 +225,6 @@ impl Bridge { } } - /// `prefix/repos///manifest.pb` → `o/r`. - fn manifest_repo(&self, object: &str) -> Option { - let rel = object - .strip_prefix(self.store_prefix.as_str())? - .strip_prefix("repos/")? - .strip_suffix("/manifest.pb")?; - let (owner, name) = rel.split_once('/')?; - if name.contains('/') { - return None; - } - RepoId::new(owner, name).ok() - } - async fn publish(&self, batch: &[RefEvent]) -> anyhow::Result<()> { if batch.is_empty() { return Ok(()); @@ -259,55 +248,6 @@ impl Bridge { } } -/// The bucket object keys a notification body names, `store.prefix` included as the -/// notifications carry it. Store-agnostic. -fn notified_keys(v: &serde_json::Value, store_prefix: &str) -> Vec { - let mut keys = Vec::new(); - // GCS → Pub/Sub push envelope. - let attrs = &v["message"]["attributes"]; - if attrs["eventType"] == "OBJECT_FINALIZE" - && let Some(k) = attrs["objectId"].as_str() - { - keys.push(k.to_string()); - } - // S3 event notification (also what MinIO/rustfs/Ceph emit). - if let Some(records) = v["Records"].as_array() { - for r in records { - if r["eventName"] - .as_str() - .is_some_and(|e| e.starts_with("ObjectCreated")) - && let Some(k) = r["s3"]["object"]["key"].as_str() - { - // S3 URL-encodes keys in notifications. - keys.push( - k.replace('+', " ") - .split('%') - .enumerate() - .map(|(i, part)| { - if i == 0 { - return part.to_string(); - } - match u8::from_str_radix(part.get(..2).unwrap_or(""), 16) { - Ok(b) => format!("{}{}", b as char, &part[2..]), - Err(_) => format!("%{part}"), - } - }) - .collect(), - ); - } - } - } - // Plain shapes for your own glue. - if let Some(k) = v["key"].as_str() { - keys.push(k.to_string()); - } - // A repository id names no object: use the key its manifest has in this bucket. - if let Some(r) = v["repo"].as_str() { - keys.push(format!("{store_prefix}repos/{r}/manifest.pb")); - } - keys -} - /// `POST /_events/notify`: a bucket notification naming a finalized `manifest.pb`. /// `200` (ack) when handled or ignored, `503` (redeliver) when a sink failed. pub async fn http_notify( @@ -373,52 +313,3 @@ pub fn spawn_sweeper(state: Arc) { } }); } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn manifest_object_names() { - let b = Bridge { - registry: walgit_wal::Registry::new( - Arc::new(walgit_store::memory::MemoryStore::new()), - Arc::new(walgit_config::Config::default()), - ), - store_prefix: "prefix/".into(), - sinks: Vec::new(), - serial: tokio::sync::Mutex::new(()), - }; - let id = b - .manifest_repo("prefix/repos/acme/monorepo/manifest.pb") - .unwrap(); - assert_eq!(id.to_string(), "acme/monorepo"); - // Every body shape names the same repository under a non-empty prefix. - for body in [ - serde_json::json!({"repo": "acme/monorepo"}), - serde_json::json!({"key": "prefix/repos/acme/monorepo/manifest.pb"}), - serde_json::json!({"message": {"attributes": { - "eventType": "OBJECT_FINALIZE", - "objectId": "prefix/repos/acme/monorepo/manifest.pb"}}}), - serde_json::json!({"Records": [{"eventName": "ObjectCreated:Put", - "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), - ] { - let ids: Vec<_> = notified_keys(&body, &b.store_prefix) - .iter() - .filter_map(|k| b.manifest_repo(k)) - .map(|id| id.to_string()) - .collect(); - assert_eq!(ids, ["acme/monorepo"], "{body}"); - } - for other in [ - "repos/t/r/manifest.pb", - "prefix/repos/t/r/wal/abc.pack", - "prefix/repos/t/r/events/cursor.json", - "walgit-go/repos/t/r/manifest.pb", - "prefix/repos/t/manifest.pb", - "prefix/repos/t/r/x/manifest.pb", - ] { - assert!(b.manifest_repo(other).is_none(), "{other}"); - } - } -} diff --git a/crates/walgit-server/src/events.rs b/crates/walgit-server/src/events.rs index 4d4d77bb..646abf92 100644 --- a/crates/walgit-server/src/events.rs +++ b/crates/walgit-server/src/events.rs @@ -1,160 +1,8 @@ -//! `ref` event shapes and the WAL → event conversion. Contract: -//! `docs/EVENTS.md`. The only producer is the bridge (`crate::bridge`): it -//! tails each repo's WAL from a durable cursor, converts committed PUSH / -//! `REF_UPDATE` entries with [`refs_from_entries`], and delivers to every -//! [`Sink`] before advancing the cursor. Nothing on the push path knows events -//! exist. -//! -//! Wire facts consumers rely on (the doc is normative, this is the code): -//! `old`/`new` are always full zero OIDs on create/delete, never empty; -//! dedup key = `(repo, _walgit.seq, ref_name)`; order by `_walgit.seq`. +//! The webhook sink of the events bridge (`crate::bridge`). The event model +//! (`RefEvent`, `refs_from_entries`, notification parsing) lives in `walgit-events` +//! so the `JetStream` publisher shares it. Contract: `docs/EVENTS.md`. -use serde::Serialize; -use walgit_git::RepoId; -use walgit_proto::v1::{EntryKind, LogEntry}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum Action { - Create, - Update, - Delete, -} - -impl Action { - pub fn as_str(self) -> &'static str { - match self { - Action::Create => "create", - Action::Update => "update", - Action::Delete => "delete", - } - } -} - -fn absent(oid: &str) -> bool { - oid.is_empty() || oid.bytes().all(|b| b == b'0') -} - -/// `create` / `update` / `delete` from the oid pair; `None` = no-op -/// (`0→0`, `X→X`) which must not emit. -pub fn classify(old: &str, new: &str) -> Option { - match (absent(old), absent(new)) { - (true, true) => None, - (true, false) => Some(Action::Create), - (false, true) => Some(Action::Delete), - (false, false) if old == new => None, - (false, false) => Some(Action::Update), - } -} - -/// The zero OID of the same length as `other` (sha1: 40, sha256: 64). The -/// WAL records "absent" as `""` (import-built txns) or the wire's zero OID -/// (pushes); the event always carries the zero OID. -fn zero_like(other: &str) -> String { - "0".repeat(other.len()) -} - -pub fn ref_type(name: &str) -> &'static str { - if name.starts_with("refs/heads/") { - "branch" - } else if name.starts_with("refs/tags/") { - "tag" - } else { - "" - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct WalgitExt { - pub schema_version: u32, - /// WAL seq of the committed entry, as a string (uint64 convention). - pub seq: String, - /// `push` | `ref_update`. - pub entry_kind: String, - #[serde(skip_serializing_if = "String::is_empty")] - pub request_id: String, -} - -/// Top level byte-compatible with the common `RefEvent` shape; walgit adds `repo` -/// and `_walgit`. -#[derive(Debug, Clone, Serialize)] -pub struct RefEvent { - pub action: Action, - pub ref_type: &'static str, - pub ref_name: String, - pub old: String, - pub new: String, - pub pusher: String, - pub correlation_id: String, - pub repo: String, - #[serde(rename = "_walgit")] - pub walgit: WalgitExt, -} - -impl RefEvent { - /// Message key (`/`): per-ref order where a transport keeps it. - pub fn key(&self) -> String { - format!("{}/{}", self.repo, self.ref_name) - } - /// Transport headers (`x_walgit_`): filtering without parsing bodies. - /// Always a strict subset of the body. - pub fn headers(&self) -> [(&'static str, &str); 7] { - [ - ("repo", &self.repo), - ("action", self.action.as_str()), - ("ref_type", self.ref_type), - ("ref_name", &self.ref_name), - ("pusher", &self.pusher), - ("seq", &self.walgit.seq), - ("correlation_id", &self.correlation_id), - ] - } -} - -/// `ref` events for the PUSH / `REF_UPDATE` entries in `entries`, in seq order. -pub(crate) fn refs_from_entries(repo: &RepoId, entries: &[LogEntry], out: &mut Vec) { - let repo = repo.to_string(); - for entry in entries { - let entry_kind = match EntryKind::try_from(entry.kind) { - Ok(EntryKind::Push) => "push", - Ok(EntryKind::RefUpdate) => "ref_update", - // COMPACT / CHECKPOINT never move refs. - _ => continue, - }; - let Some(txn) = &entry.txn else { continue }; - let pusher = entry.meta.get("principal").cloned().unwrap_or_default(); - let request_id = entry.meta.get("request_id").cloned().unwrap_or_default(); - for u in &txn.updates { - if !u.new_symbolic_target.is_empty() { - continue; // HEAD retarget: no oid transition to announce - } - let Some(action) = classify(&u.old_oid, &u.new_oid) else { - continue; - }; - let (old, new) = match action { - Action::Create => (zero_like(&u.new_oid), u.new_oid.clone()), - Action::Delete => (u.old_oid.clone(), zero_like(&u.old_oid)), - Action::Update => (u.old_oid.clone(), u.new_oid.clone()), - }; - out.push(RefEvent { - action, - ref_type: ref_type(&u.name), - ref_name: u.name.clone(), - old, - new, - pusher: pusher.clone(), - correlation_id: request_id.clone(), - repo: repo.clone(), - walgit: WalgitExt { - schema_version: 1, - seq: entry.seq.to_string(), - entry_kind: entry_kind.to_string(), - request_id: request_id.clone(), - }, - }); - } - } -} +use walgit_events::RefEvent; /// Where the bridge publishes: a webhook receives each batch as a JSON array, /// `POST`ed with `Content-Type: application/json`, a `X-Walgit-Delivery` id @@ -220,117 +68,3 @@ impl Sink for WebhookSink { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn classify_shapes() { - assert_eq!(classify("", ""), None); - assert_eq!(classify("0000", "0000"), None); - assert_eq!(classify("", "abc"), Some(Action::Create)); - assert_eq!(classify("abc", ""), Some(Action::Delete)); - assert_eq!(classify("abc", "abc"), None); - assert_eq!(classify("abc", "def"), Some(Action::Update)); - assert_eq!(ref_type("refs/heads/main"), "branch"); - assert_eq!(ref_type("refs/tags/v1"), "tag"); - assert_eq!(ref_type("refs/notes/commits"), ""); - } - - /// The golden body shape (docs/EVENTS.md). Field order and names are the - /// contract; changing this test is changing the wire. - #[test] - fn ref_event_golden() { - let ev = RefEvent { - action: Action::Update, - ref_type: "branch", - ref_name: "refs/heads/main".into(), - old: "48a0637".into(), - new: "cb38da1".into(), - pusher: "alice@example.com".into(), - correlation_id: "d1f916f7".into(), - repo: "acme/monorepo".into(), - walgit: WalgitExt { - schema_version: 1, - seq: "42".into(), - entry_kind: "push".into(), - request_id: "d1f916f7".into(), - }, - }; - let got = serde_json::to_string(&ev).unwrap(); - let want = concat!( - "{\"action\":\"update\",\"ref_type\":\"branch\",\"ref_name\":\"refs/heads/main\",", - "\"old\":\"48a0637\",\"new\":\"cb38da1\",\"pusher\":\"alice@example.com\",", - "\"correlation_id\":\"d1f916f7\",\"repo\":\"acme/monorepo\",", - "\"_walgit\":{\"schema_version\":1,\"seq\":\"42\",\"entry_kind\":\"push\",", - "\"request_id\":\"d1f916f7\"}}" - ); - assert_eq!(got, want); - assert_eq!(ev.key(), "acme/monorepo/refs/heads/main"); - } - - fn log_entry(kind: EntryKind, name: &str, old: &str, new: &str, sym: &str) -> LogEntry { - let mut meta = HashMap::new(); - meta.insert("principal".into(), "alice".into()); - meta.insert("request_id".into(), "cid".into()); - LogEntry { - seq: 9, - kind: kind as i32, - txn: Some(walgit_proto::v1::RefTransaction { - updates: vec![walgit_proto::v1::RefUpdate { - name: name.into(), - old_oid: old.into(), - new_oid: new.into(), - new_symbolic_target: sym.into(), - ..Default::default() - }], - ..Default::default() - }), - meta, - ..Default::default() - } - } - - const SHA: &str = "4a06219d6a5ba9d3e3d1a8e1e2b6f0c9d8e7f6a5"; - - #[test] - fn conversion_emits_ref_skips_compact_noop_symbolic() { - let repo = walgit_git::RepoId::new("t", "r").unwrap(); - let mut out = Vec::new(); - refs_from_entries( - &repo, - &[ - log_entry(EntryKind::Compact, "refs/heads/main", "", SHA, ""), - log_entry(EntryKind::Push, "refs/heads/main", SHA, SHA, ""), - log_entry(EntryKind::Push, "HEAD", "", "", "refs/heads/main"), - log_entry(EntryKind::Push, "refs/heads/main", "", SHA, ""), - log_entry(EntryKind::Push, "refs/heads/main", SHA, &"0".repeat(40), ""), - ], - &mut out, - ); - assert_eq!(out.len(), 2, "compact/noop/symbolic must not emit"); - let r = &out[0]; - assert_eq!(r.action, Action::Create); - assert_eq!(r.ref_name, "refs/heads/main"); - assert_eq!( - r.old, - "0".repeat(40), - "absent old is the zero OID, never empty" - ); - assert_eq!(r.new, SHA); - assert_eq!( - (r.pusher.as_str(), r.correlation_id.as_str()), - ("alice", "cid") - ); - assert_eq!((r.repo.as_str(), r.walgit.seq.as_str()), ("t/r", "9")); - assert_eq!(r.walgit.entry_kind, "push"); - let d = &out[1]; - assert_eq!(d.action, Action::Delete); - assert_eq!( - (d.old.as_str(), d.new.as_str()), - (SHA, "0".repeat(40).as_str()) - ); - } -} diff --git a/crates/walgit-server/tests/events_nats.rs b/crates/walgit-server/tests/events_nats.rs new file mode 100644 index 00000000..bea21b5a --- /dev/null +++ b/crates/walgit-server/tests/events_nats.rs @@ -0,0 +1,479 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used)] + +//! Events (docs/EVENTS.md, `walgit nats`): real `git push`es against the harness server, published +//! into a real `nats-server -js` by a `walgit_nats` publisher over the same registry. Skips (passes) +//! when `nats-server` is not on PATH. +mod harness; + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_nats::jetstream; +use futures::StreamExt; +use harness::{Server, TestRepo, git_in}; +use walgit_config::{NatsEventsConfig, NatsStart}; +use walgit_git::RepoId; +use walgit_nats::publisher::Publisher; +use walgit_nats::subject::SubjectPrefix; +use walgit_nats::testing::{NatsServer, start_nats}; + +type TestResult = anyhow::Result<()>; + +const ZERO_OID: &str = "0000000000000000000000000000000000000000"; + +struct Msg { + subject: String, + seq: String, + repo: String, + dedup_id: String, + body: serde_json::Value, +} + +struct Rig { + server: Server, + nats_cfg: NatsEventsConfig, + client: async_nats::Client, + js: jetstream::Context, + prefix: SubjectPrefix, + _nats: NatsServer, +} + +impl Rig { + async fn new() -> anyhow::Result> { + let Some(nats) = start_nats().await else { + return Ok(None); + }; + Ok(Some(Self::around(nats, Server::start().await?).await?)) + } + + async fn around(nats: NatsServer, server: Server) -> anyhow::Result { + let nats_cfg = NatsEventsConfig { + urls: vec![nats.url.clone()], + ..NatsEventsConfig::default() + }; + let client = walgit_nats::connect(&nats_cfg, &walgit_nats::auth::NatsAuth::None).await?; + let js = jetstream::new(client.clone()); + let prefix = SubjectPrefix::new(&nats_cfg.subject_prefix)?; + Ok(Rig { + server, + nats_cfg, + client, + js, + prefix, + _nats: nats, + }) + } + + /// A publisher with no memory of earlier ones, as after a restart. + async fn publisher(&self) -> anyhow::Result> { + let stream = walgit_nats::open_stream(&self.js, &self.nats_cfg, &self.prefix).await?; + Ok(Publisher::new( + self.server.state.registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + NatsStart::Oldest, + )) + } + + async fn stream(&self) -> jetstream::stream::Stream { + self.js.get_stream(&self.nats_cfg.stream).await.unwrap() + } + + async fn message_count(&self) -> u64 { + self.stream().await.get_info().await.unwrap().state.messages + } + + async fn wait_for_messages(&self, n: u64) { + let t0 = Instant::now(); + loop { + let have = self.message_count().await; + if have >= n { + return; + } + assert!( + t0.elapsed() < Duration::from_secs(10), + "timed out waiting for {n} messages; have {have}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// Every message in the stream, in stream order. + async fn messages(&self) -> Vec { + let stream = self.stream().await; + let count = stream.get_info().await.unwrap().state.messages; + let mut out = Vec::new(); + for stream_seq in 1..=count { + let raw = stream.get_raw_message(stream_seq).await.unwrap(); + let header = |name: &str| raw.headers.get(name).unwrap().as_str().to_string(); + out.push(Msg { + subject: raw.subject.to_string(), + seq: header("Walgit-Seq"), + repo: header("Walgit-Repo"), + dedup_id: header("Nats-Msg-Id"), + body: serde_json::from_slice(&raw.payload).unwrap(), + }); + } + out + } + + fn spawn_notify_loop(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + tokio::spawn(walgit_nats::wake::notify_loop( + self.client.clone(), + self.nats_cfg.notify_subject.clone(), + self.nats_cfg.queue_group.clone(), + self.server.state.cfg.store_prefix(), + publisher.clone(), + )) + } +} + +/// A working clone whose `origin` is `owner/repo` on the server (created first). +async fn clone_of(server: &Server, owner: &str, repo: &str) -> anyhow::Result { + server.put_repo(owner, repo).await?; + let src = TestRepo::synthetic(1, 1)?; + git_in(&src, &["commit", "--allow-empty", "-m", "a"])?; + git_in(&src, &["branch", "-M", "main"])?; + git_in(&src, &["remote", "add", "origin", &server.repo_url(owner, repo)])?; + Ok(src) +} + +/// Let a notified catch-up (a queue subscription, no reply) settle: the loop is subscribed once a +/// published probe reaches it, which a short sleep approximates for a fresh subscription. +async fn subscribed() { + tokio::time::sleep(Duration::from_millis(500)).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pushes_become_events_with_the_right_shape() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + let publisher = rig.publisher().await?; + + // New branch. + git_in(&src, &["push", "origin", "main"])?; + let main_v1 = git_in(&src, &["rev-parse", "HEAD"])?.trim().to_string(); + let r = publisher.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted, r.conflict), (0, 1, 1, false)); + let msgs = rig.messages().await; + assert_eq!(msgs.len(), 1); + let m = &msgs[0]; + assert_eq!(m.subject, "walgit.ref.t.r"); + assert_eq!((m.seq.as_str(), m.repo.as_str()), ("1", "t/r")); + assert_eq!(m.dedup_id, "t/r:1:refs/heads/main"); + assert_eq!(m.body["action"], "create"); + assert_eq!(m.body["ref_type"], "branch"); + assert_eq!(m.body["ref_name"], "refs/heads/main"); + assert_eq!(m.body["old"], ZERO_OID, "create carries the zero OID, never empty"); + assert_eq!(m.body["new"], main_v1); + assert_eq!(m.body["pusher"], "anon"); + assert_eq!(m.body["repo"], "t/r"); + assert_eq!(m.body["_walgit"]["seq"], "1"); + + // A no-op push writes no WAL entry and so no event. + git_in(&src, &["push", "origin", "main"])?; + let r = publisher.catch_up(&id).await?; + assert_eq!((r.head_seq, r.emitted), (1, 0)); + assert_eq!(rig.message_count().await, 1); + + // Fast-forward. + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + let main_v2 = git_in(&src, &["rev-parse", "HEAD"])?.trim().to_string(); + assert_eq!(publisher.catch_up(&id).await?.emitted, 1); + let m = &rig.messages().await[1]; + assert_eq!((m.body["action"].as_str(), m.seq.as_str()), (Some("update"), "2")); + assert_eq!((m.body["old"].as_str(), m.body["new"].as_str()), (Some(main_v1.as_str()), Some(main_v2.as_str()))); + assert_eq!(m.dedup_id, "t/r:2:refs/heads/main"); + + // Lightweight and annotated tags. + git_in(&src, &["tag", "light"])?; + git_in(&src, &["tag", "-a", "-m", "annotated", "v1"])?; + git_in(&src, &["push", "origin", "light"])?; + git_in(&src, &["push", "origin", "v1"])?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 2); + let msgs = rig.messages().await; + for (m, name) in msgs[2..].iter().zip(["refs/tags/light", "refs/tags/v1"]) { + assert_eq!(m.body["action"], "create", "{name}"); + assert_eq!(m.body["ref_type"], "tag", "{name}"); + assert_eq!(m.body["ref_name"], name); + assert_eq!(m.body["old"], ZERO_OID); + } + assert_eq!(msgs[2].body["new"], main_v2, "a lightweight tag points at the commit"); + assert_ne!(msgs[3].body["new"], main_v2, "an annotated tag points at the tag object"); + + // Branch delete. + git_in(&src, &["push", "origin", "main:refs/heads/topic"])?; + git_in(&src, &["push", "origin", ":refs/heads/topic"])?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 2); + let msgs = rig.messages().await; + assert_eq!(msgs[4].body["action"], "create"); + let del = &msgs[5]; + assert_eq!(del.body["action"], "delete"); + assert_eq!(del.body["ref_type"], "branch"); + assert_eq!(del.body["ref_name"], "refs/heads/topic"); + assert_eq!(del.body["old"], main_v2); + assert_eq!(del.body["new"], ZERO_OID); + + // Every message names its own subject, stream order is WAL order. + assert!(msgs.iter().all(|m| m.subject == "walgit.ref.t.r")); + let seqs: Vec = msgs.iter().map(|m| m.seq.parse().unwrap()).collect(); + assert!(seqs.windows(2).all(|w| w[0] < w[1]), "{seqs:?}"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_atomic_multi_ref_push_is_one_message_per_ref_in_order() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["branch", "feature"])?; + git_in(&src, &["tag", "v1"])?; + git_in( + &src, + &["push", "--atomic", "origin", "main", "feature", "refs/tags/v1"], + )?; + + let publisher = rig.publisher().await?; + let r = publisher.catch_up(&id).await?; + assert_eq!((r.head_seq, r.emitted), (1, 3), "one entry, three refs"); + let msgs = rig.messages().await; + assert_eq!(msgs.len(), 3); + assert!(msgs.iter().all(|m| m.seq == "1"), "all share the entry's seq"); + let mut names: Vec<&str> = msgs + .iter() + .map(|m| m.body["ref_name"].as_str().unwrap()) + .collect(); + let ids: Vec<&str> = msgs.iter().map(|m| m.dedup_id.as_str()).collect(); + for (m, name) in msgs.iter().zip(&names) { + assert_eq!(m.dedup_id, format!("t/r:1:{name}")); + } + assert_eq!(ids.len(), ids.iter().collect::>().len()); + names.sort_unstable(); + assert_eq!(names, ["refs/heads/feature", "refs/heads/main", "refs/tags/v1"]); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_dotted_repository_name_is_encoded_in_the_subject() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("acme.io", "mono.repo")?; + let src = clone_of(&rig.server, "acme.io", "mono.repo").await?; + git_in(&src, &["push", "origin", "main"])?; + let publisher = rig.publisher().await?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 1); + let m = &rig.messages().await[0]; + assert_eq!(m.subject, "walgit.ref.acme%2Eio.mono%2Erepo"); + assert_eq!(m.repo, "acme.io/mono.repo"); + assert_eq!(m.dedup_id, "acme.io/mono.repo:1:refs/heads/main"); + assert_eq!(m.body["repo"], "acme.io/mono.repo"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn bucket_notifications_wake_the_publisher_without_a_catch_up_call() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let src = clone_of(&rig.server, "t", "r").await?; + let publisher = rig.publisher().await?; + let notify = rig.spawn_notify_loop(&publisher); + subscribed().await; + + // An S3 bucket notification naming the manifest, as the object store publishes it. + git_in(&src, &["push", "origin", "main"])?; + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "repos/t/r/manifest.pb"}}}]}); + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), s3.to_string().into()) + .await?; + rig.wait_for_messages(1).await; + + // `{"repo": ...}` through the same subject. + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), r#"{"repo":"t/r"}"#.into()) + .await?; + rig.wait_for_messages(2).await; + + // A notification for something that is not a manifest names nothing. + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "origin", "main"])?; + let other = serde_json::json!({"key": "repos/t/r/wal/x.pack"}); + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), other.to_string().into()) + .await?; + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(rig.message_count().await, 2); + + let msgs = rig.messages().await; + assert_eq!( + msgs.iter().map(|m| m.seq.as_str()).collect::>(), + ["1", "2"] + ); + notify.abort(); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restarted_publisher_resumes_from_the_stream() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + assert_eq!(rig.publisher().await?.catch_up(&id).await?.emitted, 2); + assert_eq!(rig.message_count().await, 2); + + let restarted = rig.publisher().await?; + let r = restarted.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 2, 0)); + assert_eq!(rig.message_count().await, 2, "resume must not republish"); + + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "origin", "main"])?; + let r = restarted.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 3, 1)); + assert_eq!(rig.message_count().await, 3, "exactly one new message"); + assert_eq!(rig.messages().await[2].seq, "3"); + + // A third process that never saw any of it publishes nothing more. + assert_eq!(rig.publisher().await?.catch_up(&id).await?.emitted, 0); + assert_eq!(rig.message_count().await, 3); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_catch_up_endpoint_reports_a_real_repository() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["push", "origin", "main"])?; + let publisher = rig.publisher().await?; + let service = tokio::spawn({ + let (client, prefix, publisher) = (rig.client.clone(), rig.prefix.clone(), publisher.clone()); + let queue_group = rig.nats_cfg.queue_group.clone(); + async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } + }); + subscribed().await; + + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await?; + let report: serde_json::Value = serde_json::from_slice(&reply.payload)?; + assert_eq!(report["repo"], "t/r"); + assert_eq!((report["from_seq"].as_u64(), report["head_seq"].as_u64()), (Some(0), Some(1))); + assert_eq!(report["emitted"], 1); + assert!(report.get("conflict").is_none(), "{report}"); + assert_eq!(rig.message_count().await, 1); + + // Asked again, nothing is new. + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await?; + let report: serde_json::Value = serde_json::from_slice(&reply.payload)?; + assert_eq!((report["from_seq"].as_u64(), report["emitted"].as_u64()), (Some(1), Some(0))); + + let missing = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/none"}"#.into()) + .await?; + let code = missing.headers.as_ref().and_then(|h| h.get("Nats-Service-Error-Code")); + assert_eq!(code.map(|c| c.as_str().to_string()), Some("404".into())); + + let mut stats = rig.client.subscribe("inbox.stats").await?; + rig.client + .publish_with_reply("$SRV.STATS.walgit-events", "inbox.stats", "".into()) + .await?; + let stats = tokio::time::timeout(Duration::from_secs(5), stats.next()) + .await? + .expect("stats reply"); + let stats: serde_json::Value = serde_json::from_slice(&stats.payload)?; + assert_eq!(stats["name"], "walgit-events"); + service.abort(); + Ok(()) +} + +type Captured = Arc>>; + +/// The webhook sink's target: records every event the bridge delivers. +async fn webhook() -> (String, Captured) { + let captured: Captured = Arc::default(); + let app = axum::Router::new().route( + "/events", + axum::routing::post({ + let captured = captured.clone(); + move |axum::Json(batch): axum::Json>| { + let captured = captured.clone(); + async move { + captured.lock().unwrap().extend(batch); + axum::http::StatusCode::OK + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.ok(); + }); + (format!("http://{addr}/events"), captured) +} + +/// The identity of an event as a consumer of either sink sees it. +fn identity(event: &serde_json::Value) -> String { + let field = |name: &str| event[name].as_str().unwrap_or_default().to_string(); + format!( + "{} seq={} {} {} {} -> {}", + field("repo"), + event["_walgit"]["seq"].as_str().unwrap_or_default(), + field("ref_name"), + field("action"), + field("old"), + field("new"), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_webhook_bridge_and_the_jetstream_publisher_agree() -> TestResult { + let Some(nats) = start_nats().await else { return Ok(()) }; + let (url, captured) = webhook().await; + let server = Server::start_with_tweak(|c| { + c.events.webhook_url = Some(url.clone()); + c.events.sweep_interval = Duration::ZERO; + }) + .await?; + let rig = Rig::around(nats, server).await?; + let bridge = rig.server.state.bridge.clone().expect("bridge enabled"); + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["tag", "light"])?; + git_in(&src, &["tag", "-a", "-m", "annotated", "v1"])?; + git_in(&src, &["push", "origin", "light", "v1"])?; + git_in(&src, &["branch", "feature"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "--atomic", "origin", "main", "feature"])?; + git_in(&src, &["push", "origin", "--delete", "feature", "light"])?; + + let sent = bridge.catch_up(&id).await?; + let published = rig.publisher().await?.catch_up(&id).await?; + assert_eq!(sent.emitted, published.emitted, "{sent:?} {published:?}"); + assert_eq!(sent.head_seq, published.head_seq); + + let webhook_events: Vec = captured.lock().unwrap().iter().map(identity).collect(); + let stream_events: Vec = rig.messages().await.iter().map(|m| identity(&m.body)).collect(); + assert!(webhook_events.len() >= 8, "the scenario produced events: {webhook_events:#?}"); + assert_eq!(stream_events, webhook_events, "both sinks, same events, same order"); + // The wire body is one and the same, not just the identity fields. + let bodies: Vec = rig.messages().await.into_iter().map(|m| m.body).collect(); + assert_eq!(bodies, *captured.lock().unwrap()); + Ok(()) +} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 8f52b709..311d71a7 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -115,6 +115,112 @@ webhook_secret = "…" # env: WALGIT__EVENTS__WEBHOOK_SECRET sweep_interval = "5m" ``` +## Delivery: NATS JetStream (`walgit nats`) + +`walgit nats` is a second sink for the same `ref` events, a subcommand of the normal CLI +(`walgit nats --config walgit.toml`, `[events.nats]`) built on the `walgit-nats` crate, which depends on +`walgit-events` (the shared event model) and never on `walgit-server`. The webhook bridge is unchanged and the two can run side by side. +Like the bridge it is a reader of the WAL, never a step of a push (D32). + +- **The stream is the cursor (D50).** For each repository the publisher reads the last message on the + repository's subject, takes its `Walgit-Seq` header and resumes from that seq, inclusive. Events already in + the stream are dropped, so a crash in the middle of a multi-ref entry completes only the rest of that entry. A repository with no message yet starts at `events.nats.start`: `oldest` (the oldest + readable seq, as the bridge does) or `head` (only pushes made after the publisher first saw it). +- **Subjects.** `.ref..`. Bytes outside `[A-Za-z0-9_-]` in a name are written + `%XX`, so `acme.io/mono.repo` is `walgit.ref.acme%2Eio.mono%2Erepo` and the mapping is reversible. + Subscribe to `walgit.ref.>` for everything, `walgit.ref.acme.>` for one owner. +- **Message.** The body is one `ref` event JSON object (not an array). Headers: `Walgit-Seq`, `Walgit-Repo` + and `Nats-Msg-Id = ::`, which makes JetStream drop a republished event inside the + stream's duplicate window. The publisher waits for every PubAck before the next message, so order within + a repository is `_walgit.seq` order. +- **Concurrent publishers.** The first publish of a catch-up sets `Nats-Expected-Last-Subject-Sequence`. + If another instance moved the subject first, this one stops, logs and lets the next wake-up retry from the + new last message. Delivery is at least once; consumers still dedup on `(repo, _walgit.seq, ref_name)`. +- **Wake-ups.** A queue-group subscription on `events.nats.notify_subject` accepts the same notification + shapes as `POST /_events/notify` (S3 event JSON, a GCS envelope, `{"key": ...}`, `{"repo": ...}`), and + `events.sweep_interval` is the backstop with the same warning when it finds unpublished entries. + Gap detection matches the bridge: a log gap is logged and the entries that are readable are published. +- **Service API.** `walgit nats` registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in) + with one endpoint, `.events.catch-up`. Request `{"repo": "o/r"}`, reply the catch-up report + (`repo`, `from_seq`, `head_seq`, `emitted`, `conflict`); an unknown repository is a 404 service error. + Gap, conflict and sweep counters are in the stats response. There is no HTTP listener. +- **Losing the stream.** Wiping the stream republishes the WAL window on the next wake-up; history older than + the oldest readable entry is not in the WAL and is backfilled with `walgit wal ls`. Wiping walgit loses nothing. + Deleting and recreating the stream republishes at once. A *purge* does not: JetStream keeps its duplicate + table across a purge, so inside `duplicate_window` the republished events are acknowledged as duplicates and + land nothing; the publisher leaves the repository open and the window's expiry heals it on a later wake-up. + Prefer delete and recreate, or wait out the window. +- **`start = "head"` and restarts.** The first-seen head of a repository without a message is remembered in + memory only. Pushes made while the publisher is down, before the repository's first event, are skipped for good + after a restart (pinned by a test). +- **A foreign message** (no `Walgit-Seq` / `Nats-Msg-Id`) as the last message on a repository's subject makes every + catch-up for that repository fail, naming the subject and stream sequence, until it is removed. +- **Deleting and recreating a repository.** The manifest has no incarnation id, so the only signal is the stream + being ahead of the WAL: when the last message on a subject names a seq above the repository's `head_seq`, the + publisher warns, counts it (`rewinds` in the service stats), ignores that message as a cursor (the first seq + follows `events.nats.start`; with `"head"` the new incarnation's head at that first sighting is kept while the + old message still tops the subject, so later pushes are published) and still guards the first publish with the subject's real last stream sequence, so + it does not conflict. The new incarnation's `Nats-Msg-Id` (`::`) can equal the old one's: inside + `duplicate_window` the broker swallows it, the catch-up is not confirmed, the repository stays open and the + event lands on the first wake-up after the window (pinned by a test). Limitation: a recreated repository whose + head has already reached the old last seq before the publisher wakes is indistinguishable from the old one, and + its events up to that seq are never published. Consumers should treat a delete and recreate as a new stream. + The webhook bridge has the same ambiguity from the other side: its cursor lives under the repository's prefix and + is deleted with it, so the new incarnation is published from seq 1 and the dedup key `(repo, seq, ref_name)` + collides with the old incarnation's events. +- **Config adaptation.** `walgit nats` always revalidates the manifest (`wal.freshness_ttl` is forced to zero) and + never prefetches packs (`wal.prefetch_packs` is forced off), whatever `[wal]` says, because it never applies a + push itself and only reads manifest and log (`walgit_nats::publisher_config`). +- **Cost.** Every wake-up for a repository reads the last message on its subject (one JetStream request) before + deciding there is nothing to do, so a wiped stream is noticed without a restart. + +Each behaviour above has a test against a real `nats-server -js` (`crates/walgit-nats/tests/jetstream.rs`, and +`crates/walgit-server/tests/events_nats.rs` with real git pushes); `tests/e2e.sh` runs `walgit nats` as a process. + +| Behaviour | Test | +|---|---| +| Push becomes one message per ref with the bridge's event shape | `pushes_become_events_with_the_right_shape`, `an_atomic_multi_ref_push_is_one_message_per_ref_in_order` | +| The webhook bridge and `walgit nats` emit the same events | `the_webhook_bridge_and_the_jetstream_publisher_agree` | +| Names are encoded in the subject | `a_dotted_repository_name_is_encoded_in_the_subject` | +| The stream is the cursor; a restart resumes from it | `stream_is_the_cursor`, `a_restarted_publisher_resumes_from_the_stream` | +| A crash mid-entry finishes that entry only | `a_crash_mid_entry_finishes_that_entry_only` | +| A NATS outage mid catch-up loses and duplicates nothing | `an_outage_mid_catch_up_loses_and_duplicates_nothing` | +| Broker dedup inside `duplicate_window`; none after it | `broker_dedup_absorbs_a_republish_inside_the_window`, `after_the_dedup_window_a_republish_lands_twice` | +| Concurrent publishers leave each event once | `a_second_publisher_that_moved_first_stops_this_one`, `racing_publishers_leave_each_event_exactly_once` | +| Losing the stream republishes the WAL window | `a_wiped_stream_is_republished_on_the_next_catch_up`, `a_purged_stream_heals_once_the_dedup_window_has_passed` | +| A log gap is counted, readable entries still published | `entries_folded_before_publishing_are_a_counted_gap` | +| A gap is counted once while nothing moves the subject | `a_gap_is_counted_once_while_nothing_moves_the_subject` | +| Bucket notifications wake it; junk is ignored | `bucket_notifications_wake_the_publisher_without_a_catch_up_call`, `notifications_that_name_nothing_are_ignored_and_the_loop_survives` | +| The sweep finds what nobody notified | `the_periodic_sweep_finds_a_push_nobody_notified` | +| Service API: catch-up, errors, stats | `notifications_and_the_service_api_wake_the_publisher`, `the_catch_up_endpoint_reports_a_real_repository`, `the_service_answers_bad_requests_and_reports_the_counters` | +| A NATS server restart keeps stream, subscriptions and service | `a_nats_server_restart_keeps_the_stream_the_subscriptions_and_the_service` | +| `start = "head"` | `head_start_publishes_nothing_older`, `known_limitation_head_start_skips_pushes_made_while_the_publisher_was_down` | +| A foreign message wedges the repository with a clear error | `a_foreign_message_on_a_repo_subject_fails_every_catch_up_with_a_clear_error` | +| A notification inside `wal.freshness_ttl` still publishes the push; no pack prefetch | `a_notification_inside_the_freshness_ttl_still_publishes_the_push`, `the_publisher_never_prefetches_packs_whatever_the_operator_configured` | +| Subject tokens round-trip odd names and never carry `.`, `*`, `>` | `odd_names_round_trip_and_never_carry_a_metacharacter`, `every_single_byte_string_round_trips` (unit, `subject.rs`) | +| Notification shapes: S3 multi-record, GCS finalize only, `store.prefix` | `an_s3_event_with_several_records_names_each_repo_once`, `a_gcs_envelope_counts_only_finalize_events`, `the_store_prefix_scopes_what_is_notified` (unit, `wake.rs`) | +| `ensure_stream = false` with no stream fails clearly; an existing stream is reused | `a_missing_stream_with_ensure_stream_off_fails_clearly_and_creates_nothing`, `an_existing_stream_with_other_settings_is_reused_not_clobbered` | +| A bad server URL fails to connect naming the setting | `an_unparsable_server_url_fails_to_connect_naming_the_setting` | +| Tags, deletes and non-branch refs carry `ref_type` and zero OIDs | `tags_and_deletes_carry_their_ref_type_and_zero_oids` | +| Many pushes land in seq order across catch-ups | `many_sequential_pushes_land_in_seq_order_across_catch_ups` | +| A deleted and recreated repository is detected and published as new; an id collision heals after the dedup window | `a_recreated_repository_is_published_as_a_new_one`, `a_recreated_repository_under_head_start_publishes_pushes_after_its_first_sighting`, `a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window` | +| SIGTERM exits 0 | `tests/e2e.sh` phase 3 | + +Not covered: Prometheus metrics (the service stats are the only counters), multi-node JetStream clusters, and +the Containerfile / flake packaging. + +```toml +[events] +sweep_interval = "5m" +[events.nats] +urls = ["nats://nats.internal:4222"] +start = "oldest" +``` + +Authentication is never in `walgit.toml`: `walgit nats` reads the standard NATS variables and uses the +first that is set, `NATS_CREDS` (a `.creds` file), `NATS_NKEY` (a seed), `NATS_USER` with `NATS_PASSWORD`, +then `NATS_TOKEN`; with none set it connects without credentials. Only the method is logged. + ## Consumer checklist 1. Verify `X-Walgit-Signature` (if you set a secret), then parse the array. diff --git a/tests/e2e.sh b/tests/e2e.sh index 905d5c2e..60c02e71 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -15,6 +15,9 @@ # WALGIT_TEST_S3_ENDPOINT=http://localhost:9000 tests/e2e.sh # WALGIT=/absolute/path/to/freshly/built/walgit tests/e2e.sh # +# Phase 3 (S3/rustfs plus `nats-server` and the `nats` CLI on PATH): `walgit nats` as a process next to +# `walgit serve`; real pushes become JetStream events; a restart republishes nothing; wake-ups work. +# # Requires: cargo, git, curl, python3 (for random port), ripgrep (S3 maintenance receipt). set -euo pipefail @@ -345,6 +348,184 @@ EOF kill "$SERVER_A_PID" "$SERVER_B_PID" 2>/dev/null || true fi +# --- phase 3: `walgit nats` as processes (optional, local only) -------------- + +# Real processes end to end: nats-server -js, `walgit serve` and `walgit nats` sharing the rustfs +# bucket, real git pushes, assertions through the `nats` CLI. The first walgit nats is woken by the +# sweep (and one notification); after a kill and restart the sweep is off, so a push there can only +# arrive through the notify subject. +if [[ -z "${WALGIT_E2E_BASE_URL:-}" && -n "${WALGIT_TEST_S3_ENDPOINT:-}" ]]; then + if ! command -v nats-server >/dev/null 2>&1 || ! command -v nats >/dev/null 2>&1; then + step "phase 3: walgit nats (skipped: nats-server and the nats CLI must be on PATH)" + else + step "phase 3: walgit nats against nats-server -js ($WALGIT_TEST_S3_ENDPOINT)" + export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-walgit-dev}" AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-walgit-dev-secret}" + + NATS_PORT="$(rand_port)" + NATS_URL="nats://127.0.0.1:${NATS_PORT}" + STREAM="WALGIT_E2E_$$" + REPO3_NAME="nats-$(date +%s)-$$" + REPO3="test/$REPO3_NAME" + SUBJECT3="walgit.ref.test.$REPO3_NAME" + PORT_N="$(rand_port)" + BASE_N="http://127.0.0.1:${PORT_N}" + + mkdir -p "$TMP/nats-store" + nats-server -js -a 127.0.0.1 -p "$NATS_PORT" -sd "$TMP/nats-store" > "$TMP/nats-server.log" 2>&1 & + PIDS+=("$!") + for ((i=0; i<30; i++)); do nats -s "$NATS_URL" rtt >/dev/null 2>&1 && break; sleep 1; done + nats -s "$NATS_URL" rtt >/dev/null 2>&1 || fail "nats-server did not start" + pass "nats-server ready on $NATS_PORT" + + # A fresh store prefix keeps the repositories earlier runs left in the bucket out of the stream. + cat > "$TMP/cfg-nats.toml" </dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["state"]["messages"])' 2>/dev/null || echo 0 + } + wait_nats_count() { + local want="$1" max="${2:-20}" + for ((i=0; i "$log" + RUST_LOG=info env "WALGIT__CACHE__DIR=$TMP/cache-nats-pub" "$@" \ + "$WALGIT" --config "$TMP/cfg-nats.toml" nats > "$log" 2>&1 & + NATS_PUB_PID=$! + PIDS+=("$NATS_PUB_PID") + for ((i=0; i<30; i++)); do + kill -0 "$NATS_PUB_PID" 2>/dev/null || { cat "$log"; fail "walgit nats exited"; } + grep -q "service ready" "$log" && return 0 + sleep 1 + done + cat "$log"; fail "walgit nats did not become ready" + } + + RUST_LOG=info env "WALGIT__CACHE__DIR=$TMP/cache-nats-serve" \ + "$WALGIT" --config "$TMP/cfg-nats.toml" serve > "$TMP/server-nats.log" 2>&1 & + SERVER_N_PID=$! + PIDS+=("$SERVER_N_PID") + wait_http "$BASE_N/healthz" 15 || fail "walgit serve (nats phase) not ready" + start_walgit_nats "$TMP/walgit-nats-1.log" + pass "walgit serve on $PORT_N and walgit nats ready" + + curl_auth -sf -X PUT "$BASE_N/$REPO3.git" -o /dev/null || fail "create $REPO3" + git clone -q "$SYNTH_DIR" "$TMP/work-nats" + (cd "$TMP/work-nats" && git remote set-url origin "$BASE_N/$REPO3.git") + + step "nats: create pushed, woken by a notification on walgit.notify" + (cd "$TMP/work-nats" && git push -q origin main) || fail "push main" + nats -s "$NATS_URL" pub walgit.notify "{\"repo\":\"$REPO3\"}" >/dev/null + wait_nats_count 1 10 || fail "no event for the first push after a notification" + pass "first event arrived" + + step "nats: update, tag and delete, found by the 2s sweep" + (cd "$TMP/work-nats" && git commit -q --allow-empty -m "e2e nats update" && git push -q origin main) || fail "push update" + (cd "$TMP/work-nats" && git tag -a -m "e2e nats tag" v1.0.0 && git push -q origin v1.0.0) || fail "push tag" + (cd "$TMP/work-nats" && git push -q origin :refs/tags/v1.0.0) || fail "delete tag" + wait_nats_count 4 30 || fail "expected 4 events, have $(nats_count)" + sleep 5 + [[ "$(nats_count)" -eq 4 ]] || fail "sweeps republished: $(nats_count) messages, expected 4" + EVENTS="$(nats_events)" + echo "$EVENTS" + [[ "$(echo "$EVENTS" | sed -n 1p)" == "$SUBJECT3 create branch refs/heads/main 1 $REPO3:1:refs/heads/main" ]] || fail "event 1" + [[ "$(echo "$EVENTS" | sed -n 2p)" == "$SUBJECT3 update branch refs/heads/main 2 $REPO3:2:refs/heads/main" ]] || fail "event 2" + [[ "$(echo "$EVENTS" | sed -n 3p)" == "$SUBJECT3 create tag refs/tags/v1.0.0 3 $REPO3:3:refs/tags/v1.0.0" ]] || fail "event 3" + [[ "$(echo "$EVENTS" | sed -n 4p)" == "$SUBJECT3 delete tag refs/tags/v1.0.0 4 $REPO3:4:refs/tags/v1.0.0" ]] || fail "event 4" + pass "create, update, tag create and delete on $SUBJECT3, in WAL order" + + step "nats: kill walgit nats, restart (sweep off): no duplicates, a new push arrives by notification" + kill "$NATS_PUB_PID" 2>/dev/null || true + wait "$NATS_PUB_PID" 2>/dev/null || true + start_walgit_nats "$TMP/walgit-nats-2.log" WALGIT__EVENTS__SWEEP_INTERVAL=0s + sleep 3 + [[ "$(nats_count)" -eq 4 ]] || fail "restart republished: $(nats_count) messages, expected 4" + pass "restart left the stream at 4 messages" + + (cd "$TMP/work-nats" && git commit -q --allow-empty -m "e2e nats after restart" && git push -q origin main) || fail "push after restart" + sleep 3 + [[ "$(nats_count)" -eq 4 ]] || fail "sweep is off but the push was published without a wake-up" + nats -s "$NATS_URL" pub walgit.notify "{\"repo\":\"$REPO3\"}" >/dev/null + wait_nats_count 5 10 || fail "no event after the notification following the restart" + sleep 3 + [[ "$(nats_count)" -eq 5 ]] || fail "expected exactly 5 messages, have $(nats_count)" + LAST="$(nats_events | tail -n 1)" + [[ "$LAST" == "$SUBJECT3 update branch refs/heads/main 5 $REPO3:5:refs/heads/main" ]] || fail "event 5: $LAST" + pass "post-restart push published exactly once" + + step "nats: \$SRV catch-up endpoint" + REPORT="$(nats -s "$NATS_URL" request walgit.events.catch-up "{\"repo\":\"$REPO3\"}" --raw 2>/dev/null || true)" + echo "$REPORT" + echo "$REPORT" | python3 -c ' +import json, sys +r = json.load(sys.stdin) +assert r["repo"] == sys.argv[1] and r["head_seq"] == 5 and r["emitted"] == 0, r' "$REPO3" || fail "catch-up report" + pass "catch-up reports head 5, nothing to emit" + + step "nats: SIGTERM makes walgit nats exit 0 promptly" + kill -TERM "$NATS_PUB_PID" + for ((i=0; i<50; i++)); do + kill -0 "$NATS_PUB_PID" 2>/dev/null || break + sleep 0.1 + done + if kill -0 "$NATS_PUB_PID" 2>/dev/null; then + cat "$TMP/walgit-nats-2.log"; fail "walgit nats still running 5s after SIGTERM" + fi + NATS_RC=0 + wait "$NATS_PUB_PID" 2>/dev/null || NATS_RC=$? + [[ "$NATS_RC" -eq 0 ]] || { cat "$TMP/walgit-nats-2.log"; fail "walgit nats exited $NATS_RC after SIGTERM"; } + grep -q "shutting down" "$TMP/walgit-nats-2.log" || fail "no shutdown line after SIGTERM" + pass "SIGTERM: walgit nats exited 0 within 5s" + + kill "$SERVER_N_PID" 2>/dev/null || true + curl_auth -s -X DELETE "$BASE_N/$REPO3.git" -o /dev/null || true + fi +fi + # --- done -------------------------------------------------------------------- step "e2e: all checks passed" diff --git a/walgit.example.toml b/walgit.example.toml index 79584aa9..7f174580 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -203,6 +203,18 @@ lock_wait_warn = "1s" # WARN "lock wait" (lock, repo, wait_ms, reque # webhook_secret = "" # X-Walgit-Signature: sha256=; unset = unsigned # sweep_interval = "5m" # backstop + health check (warns when a sweep finds unpublished entries); 0 = off +[events.nats] # docs/EVENTS.md: read only by `walgit nats --config walgit.toml` +# urls = ["nats://127.0.0.1:4222"] # the client connects to the first reachable server +# auth comes from the environment, first match wins: NATS_CREDS (.creds file), NATS_NKEY (seed), +# NATS_USER + NATS_PASSWORD, NATS_TOKEN; none set = no auth; secrets are never logged +# stream = "WALGIT_EVENTS" # the JetStream stream holding the events (subjects ".ref.>"); it is the resume cursor +# ensure_stream = true # create the stream when missing (file storage); an existing stream is used as is +# duplicate_window = "1h" # dedup window (Nats-Msg-Id) of a stream `walgit nats` creates +# subject_prefix = "walgit" # events go to ".ref.."; metacharacters in names are %XX encoded +# notify_subject = "walgit.notify" # core subject bucket notifications are published to (S3 event JSON, {"key"} or {"repo"}) +# queue_group = "walgit-events" # notify subscription and service endpoints: instances in a group share the work +# start = "oldest" # first sight of a repo with no events in the stream: "oldest" (oldest readable seq) or "head" (only new pushes) + # Packfile foundations: these policies are validated and stored, but URI delivery and # ref filtering are not enabled by the foundation-only change. [refs]