From f9ca794a77c902e90bc4bbec1ac469b23f68bc7e Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 20:37:33 -0400 Subject: [PATCH 1/8] feat(events): publish ref events into NATS JetStream Signed-off-by: Yordis Prieto --- AGENTS.md | 8 + Cargo.lock | 167 +++++++++++ Cargo.toml | 4 + crates/walgit-config/src/lib.rs | 119 ++++++++ crates/walgit-events/Cargo.toml | 14 + crates/walgit-events/src/lib.rs | 389 +++++++++++++++++++++++++ crates/walgit-nats/Cargo.toml | 36 +++ crates/walgit-nats/src/lib.rs | 58 ++++ crates/walgit-nats/src/main.rs | 118 ++++++++ crates/walgit-nats/src/message.rs | 208 ++++++++++++++ crates/walgit-nats/src/publisher.rs | 284 ++++++++++++++++++ crates/walgit-nats/src/service.rs | 76 +++++ crates/walgit-nats/src/subject.rs | 209 ++++++++++++++ crates/walgit-nats/src/wake.rs | 84 ++++++ crates/walgit-nats/tests/jetstream.rs | 398 ++++++++++++++++++++++++++ crates/walgit-server/Cargo.toml | 1 + crates/walgit-server/src/bridge.rs | 119 +------- crates/walgit-server/src/events.rs | 274 +----------------- docs/EVENTS.md | 41 +++ walgit.example.toml | 11 + 20 files changed, 2234 insertions(+), 384 deletions(-) create mode 100644 crates/walgit-events/Cargo.toml create mode 100644 crates/walgit-events/src/lib.rs create mode 100644 crates/walgit-nats/Cargo.toml create mode 100644 crates/walgit-nats/src/lib.rs create mode 100644 crates/walgit-nats/src/main.rs create mode 100644 crates/walgit-nats/src/message.rs create mode 100644 crates/walgit-nats/src/publisher.rs create mode 100644 crates/walgit-nats/src/service.rs create mode 100644 crates/walgit-nats/src/subject.rs create mode 100644 crates/walgit-nats/src/wake.rs create mode 100644 crates/walgit-nats/tests/jetstream.rs diff --git a/AGENTS.md b/AGENTS.md index 8fc7d20b..18d68492 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -487,6 +487,14 @@ full cold-read/resource acceptance gates listed in `docs/spec/README.md`. and candidate external-boundary proof remain separate obligations; local loose objects and retired download membership cannot justify retirement. See the cost and remaining-evidence rows in the linked docs. +- **D50 (2026-10-26): The JetStream sink's cursor lives in its own stream.** `walgit-nats` publishes the + `ref` events of D32 into NATS JetStream and resumes from the last message on each repository's subject + (`Walgit-Seq` header, inclusive, then drop what the stream already holds). That cursor is the sink's state, + not walgit's: Principle I holds because wiping every walgit instance loses nothing, and wiping the stream + republishes the WAL window with older history backfilled from the WAL. The event model is the `walgit-events` + crate shared with the webhook bridge, whose bucket cursor (`events/cursor.json`) is unchanged. Wake-ups + arrive over NATS and the periodic sweep stays the backstop; the service API is the NATS micro protocol, no HTTP. + ## 5. Working rules - **No backwards compatibility (pre-1.0, banner at top):** change the shape and delete the old one in the same diff --git a/Cargo.lock b/Cargo.lock index 93065712..b7a9c21f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,42 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-nats" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f92759f418774b78c36ef07885849150ce950c67d9bcdeff8a9c253f57d9a1" +dependencies = [ + "base64", + "bytes", + "futures-util", + "memchr", + "nkeys", + "nuid", + "pin-project", + "portable-atomic", + "rand 0.10.2", + "regex", + "ring", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki 0.103.14", + "serde", + "serde_json", + "serde_nanos", + "serde_repr", + "thiserror", + "time", + "tokio", + "tokio-rustls 0.26.4", + "tokio-stream", + "tokio-util", + "tokio-websockets", + "tracing", + "tryhard", + "url", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -1437,6 +1473,7 @@ dependencies = [ "ed25519", "serde", "sha2 0.10.9", + "signature", "subtle", "zeroize", ] @@ -3732,6 +3769,21 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "nkeys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879011babc47a1c7fdf5a935ae3cfe94f34645ca0cac1c7f6424b36fc743d1bf" +dependencies = [ + "data-encoding", + "ed25519", + "ed25519-dalek", + "getrandom 0.2.17", + "log", + "rand 0.8.7", + "signatory", +] + [[package]] name = "nom" version = "7.1.3" @@ -3757,6 +3809,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nuid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83" +dependencies = [ + "rand 0.8.7", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -4908,6 +4969,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_nanos" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93142f0367a4cc53ae0fead1bcda39e85beccfad3dcd717656cacab94b12985" +dependencies = [ + "serde", +] + [[package]] name = "serde_path_to_error" version = "0.1.20" @@ -4919,6 +4989,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -5005,6 +5086,12 @@ dependencies = [ "sha1 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5068,6 +5155,18 @@ dependencies = [ "libc", ] +[[package]] +name = "signatory" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" +dependencies = [ + "pkcs8", + "rand_core 0.6.4", + "signature", + "zeroize", +] + [[package]] name = "signature" version = "2.2.0" @@ -5429,6 +5528,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52efb639344a7c6adb8e62c6f3d2c19c001ff1b79a5041ba1c6ed42e19c6aa5" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-sink", + "http 1.5.0", + "httparse", + "rand 0.10.2", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -5656,6 +5776,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tryhard" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe58ebd5edd976e0fe0f8a14d2a04b7c81ef153ea9a54eebc42e67c2c23b4e5" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "typenum" version = "1.20.1" @@ -5818,6 +5948,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "walgit-events" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "walgit-git", + "walgit-proto", +] + [[package]] name = "walgit-git" version = "0.1.0" @@ -5857,6 +5997,32 @@ dependencies = [ "walgit-proto", ] +[[package]] +name = "walgit-nats" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "bytes", + "clap", + "futures", + "mimalloc", + "rustls 0.23.45", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "walgit-config", + "walgit-events", + "walgit-git", + "walgit-proto", + "walgit-store", + "walgit-wal", +] + [[package]] name = "walgit-proto" version = "0.1.0" @@ -5928,6 +6094,7 @@ dependencies = [ "tracing-subscriber", "uuid", "walgit-config", + "walgit-events", "walgit-git", "walgit-proto", "walgit-store", diff --git a/Cargo.toml b/Cargo.toml index 2a1c3be7..9aca89e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,9 @@ members = [ "crates/walgit-config", "crates/walgit-git", "crates/walgit-wal", + "crates/walgit-events", "crates/walgit-server", + "crates/walgit-nats", "crates/walgit-cli", ] @@ -65,6 +67,7 @@ walgit-store = { path = "crates/walgit-store" } walgit-config = { path = "crates/walgit-config" } walgit-git = { path = "crates/walgit-git" } walgit-wal = { path = "crates/walgit-wal" } +walgit-events = { path = "crates/walgit-events" } walgit-server = { path = "crates/walgit-server" } anyhow = "1" @@ -139,6 +142,7 @@ tempfile = "3" flate2 = "1" async-compression = { version = "0.4", features = ["tokio", "gzip"] } pin-project-lite = "0.2" +async-nats = "0.51" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "json"] } base64 = "0.22" jsonwebtoken = { version = "11.0.0", features = ["rust_crypto"] } diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index 6a156eb3..c120026d 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -637,6 +637,8 @@ pub struct EventsConfig { /// warns when a sweep finds unpublished entries. `0` = off. #[serde(with = "humantime_serde")] pub sweep_interval: Duration, + /// The `JetStream` publisher (`walgit-nats`, `docs/EVENTS.md`). Read only by that binary. + pub nats: NatsEventsConfig, } impl Default for EventsConfig { @@ -645,10 +647,98 @@ impl Default for EventsConfig { webhook_url: None, webhook_secret: None, sweep_interval: Duration::from_mins(5), + nats: NatsEventsConfig::default(), } } } +/// Where a cold repository (no message on its subject yet) starts publishing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum NatsStart { + /// The oldest readable seq (`min_seq - 1`), as the webhook bridge does. + #[default] + Oldest, + /// Nothing older than the head the publisher first sees. + Head, +} + +/// `[events.nats]`: the `walgit-nats` binary publishes the bridge's `ref` events into `JetStream`. +/// The resume cursor is the last message on each repository's subject, so there is no cursor +/// object in the bucket. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct NatsEventsConfig { + /// Server URLs (`nats://host:4222`); the client connects to the first reachable one. + pub urls: Vec, + /// A `.creds` or `NKey` seed file. Never logged. + pub credentials_file: Option, + /// The `JetStream` stream holding the events (subjects `.ref.>`). + pub stream: String, + /// Create the stream when missing (file storage, `duplicate_window`). An existing stream is used as is. + pub ensure_stream: bool, + /// Deduplication window of a created stream (`Nats-Msg-Id`). + #[serde(with = "humantime_serde")] + pub duplicate_window: Duration, + /// Events go to `.ref..`. + pub subject_prefix: String, + /// Core subject bucket notifications are published to (S3 event JSON, `{"key"}` or `{"repo"}`). + pub notify_subject: String, + /// Queue group of the notify subscription and the service endpoints: instances share the work. + pub queue_group: String, + pub start: NatsStart, +} + +impl Default for NatsEventsConfig { + fn default() -> Self { + NatsEventsConfig { + urls: vec!["nats://127.0.0.1:4222".into()], + credentials_file: None, + stream: "WALGIT_EVENTS".into(), + ensure_stream: true, + duplicate_window: Duration::from_hours(1), + subject_prefix: "walgit".into(), + notify_subject: "walgit.notify".into(), + queue_group: "walgit-events".into(), + start: NatsStart::Oldest, + } + } +} + +impl NatsEventsConfig { + fn validate(&self) -> Result<()> { + anyhow::ensure!(!self.urls.is_empty(), "events.nats.urls must not be empty"); + anyhow::ensure!( + self.urls.iter().all(|u| !u.trim().is_empty()), + "events.nats.urls must not contain empty entries" + ); + let plain = |s: &str| { + !s.is_empty() + && !s + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '*' | '>')) + }; + let dotted = |s: &str| plain(s) && s.split('.').all(|t| !t.is_empty()); + anyhow::ensure!( + plain(&self.stream) && !self.stream.contains(['.', '/', '\\']), + "events.nats.stream must be a stream name (no '.', '/', '\\', '*', '>' or whitespace)" + ); + anyhow::ensure!( + dotted(&self.subject_prefix), + "events.nats.subject_prefix must be dot-separated tokens (no '*', '>' or whitespace)" + ); + anyhow::ensure!( + dotted(&self.notify_subject), + "events.nats.notify_subject must be a literal subject (no '*', '>' or whitespace)" + ); + anyhow::ensure!( + plain(&self.queue_group), + "events.nats.queue_group must not be empty or contain '*', '>' or whitespace" + ); + Ok(()) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct TelemetryConfig { @@ -1252,6 +1342,7 @@ impl Config { "events.webhook_url must be an http(s) URL" ); } + self.events.nats.validate()?; Ok(()) } @@ -1740,6 +1831,34 @@ webhook_secret = "s" let err = Config::parse("[events]\nwebhook_url = \"ftp://x\"\n").unwrap_err(); assert!(err.to_string().contains("webhook_url"), "{err}"); } + + #[test] + fn events_nats_section_parses_and_validates() { + let d = Config::parse("").unwrap().events.nats; + assert_eq!((d.stream.as_str(), d.start), ("WALGIT_EVENTS", NatsStart::Oldest)); + let c = Config::parse( + r#" +[events.nats] +urls = ["nats://a:4222", "nats://b:4222"] +start = "head" +subject_prefix = "acme.git" +"#, + ) + .unwrap(); + assert_eq!(c.events.nats.start, NatsStart::Head); + assert_eq!(c.events.nats.urls.len(), 2); + for bad in [ + "subject_prefix = \"a.*\"", + "subject_prefix = \"a..b\"", + "notify_subject = \"x.>\"", + "stream = \"a.b\"", + "queue_group = \"\"", + "urls = []", + ] { + let err = Config::parse(&format!("[events.nats]\n{bad}\n")).unwrap_err(); + assert!(err.to_string().contains("events.nats"), "{bad}: {err}"); + } + } } /// Negotiated static delivery thresholds. Foundation only: no URI emission yet. diff --git a/crates/walgit-events/Cargo.toml b/crates/walgit-events/Cargo.toml new file mode 100644 index 00000000..3b36cd67 --- /dev/null +++ b/crates/walgit-events/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "walgit-events" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +walgit-proto.workspace = true +walgit-git.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/walgit-events/src/lib.rs b/crates/walgit-events/src/lib.rs new file mode 100644 index 00000000..de2be5bb --- /dev/null +++ b/crates/walgit-events/src/lib.rs @@ -0,0 +1,389 @@ +//! `ref` event shapes and the WAL → event conversion. Contract: +//! `docs/EVENTS.md`. Shared by the producers (the webhook bridge in +//! `walgit-server`, the `JetStream` publisher in `walgit-nats`): they tail each +//! repo's WAL from a durable cursor and convert committed PUSH / `REF_UPDATE` +//! entries with [`refs_from_entries`]. Nothing on the push path knows events +//! exist, and neither does the WAL crate (D32). +//! +//! Wire facts consumers rely on (the doc is normative, this is the code): +//! `old`/`new` are always full zero OIDs on create/delete, never empty; +//! dedup key = `(repo, _walgit.seq, ref_name)`; order by `_walgit.seq`. + +#![allow(clippy::indexing_slicing)] + +use serde::Serialize; +use walgit_git::RepoId; +use walgit_proto::v1::{EntryKind, LogEntry}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Action { + Create, + Update, + Delete, +} + +impl Action { + pub fn as_str(self) -> &'static str { + match self { + Action::Create => "create", + Action::Update => "update", + Action::Delete => "delete", + } + } +} + +fn absent(oid: &str) -> bool { + oid.is_empty() || oid.bytes().all(|b| b == b'0') +} + +/// `create` / `update` / `delete` from the oid pair; `None` = no-op +/// (`0→0`, `X→X`) which must not emit. +pub fn classify(old: &str, new: &str) -> Option { + match (absent(old), absent(new)) { + (true, true) => None, + (true, false) => Some(Action::Create), + (false, true) => Some(Action::Delete), + (false, false) if old == new => None, + (false, false) => Some(Action::Update), + } +} + +/// The zero OID of the same length as `other` (sha1: 40, sha256: 64). The +/// WAL records "absent" as `""` (import-built txns) or the wire's zero OID +/// (pushes); the event always carries the zero OID. +fn zero_like(other: &str) -> String { + "0".repeat(other.len()) +} + +pub fn ref_type(name: &str) -> &'static str { + if name.starts_with("refs/heads/") { + "branch" + } else if name.starts_with("refs/tags/") { + "tag" + } else { + "" + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct WalgitExt { + pub schema_version: u32, + /// WAL seq of the committed entry, as a string (uint64 convention). + pub seq: String, + /// `push` | `ref_update`. + pub entry_kind: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub request_id: String, +} + +/// Top level byte-compatible with the common `RefEvent` shape; walgit adds `repo` +/// and `_walgit`. +#[derive(Debug, Clone, Serialize)] +pub struct RefEvent { + pub action: Action, + pub ref_type: &'static str, + pub ref_name: String, + pub old: String, + pub new: String, + pub pusher: String, + pub correlation_id: String, + pub repo: String, + #[serde(rename = "_walgit")] + pub walgit: WalgitExt, +} + +impl RefEvent { + /// Message key (`/`): per-ref order where a transport keeps it. + pub fn key(&self) -> String { + format!("{}/{}", self.repo, self.ref_name) + } + /// Transport headers (`x_walgit_`): filtering without parsing bodies. + /// Always a strict subset of the body. + pub fn headers(&self) -> [(&'static str, &str); 7] { + [ + ("repo", &self.repo), + ("action", self.action.as_str()), + ("ref_type", self.ref_type), + ("ref_name", &self.ref_name), + ("pusher", &self.pusher), + ("seq", &self.walgit.seq), + ("correlation_id", &self.correlation_id), + ] + } +} + +/// `ref` events for the PUSH / `REF_UPDATE` entries in `entries`, in seq order. +pub fn refs_from_entries(repo: &RepoId, entries: &[LogEntry], out: &mut Vec) { + let repo = repo.to_string(); + for entry in entries { + let entry_kind = match EntryKind::try_from(entry.kind) { + Ok(EntryKind::Push) => "push", + Ok(EntryKind::RefUpdate) => "ref_update", + // COMPACT / CHECKPOINT never move refs. + _ => continue, + }; + let Some(txn) = &entry.txn else { continue }; + let pusher = entry.meta.get("principal").cloned().unwrap_or_default(); + let request_id = entry.meta.get("request_id").cloned().unwrap_or_default(); + for u in &txn.updates { + if !u.new_symbolic_target.is_empty() { + continue; // HEAD retarget: no oid transition to announce + } + let Some(action) = classify(&u.old_oid, &u.new_oid) else { + continue; + }; + let (old, new) = match action { + Action::Create => (zero_like(&u.new_oid), u.new_oid.clone()), + Action::Delete => (u.old_oid.clone(), zero_like(&u.old_oid)), + Action::Update => (u.old_oid.clone(), u.new_oid.clone()), + }; + out.push(RefEvent { + action, + ref_type: ref_type(&u.name), + ref_name: u.name.clone(), + old, + new, + pusher: pusher.clone(), + correlation_id: request_id.clone(), + repo: repo.clone(), + walgit: WalgitExt { + schema_version: 1, + seq: entry.seq.to_string(), + entry_kind: entry_kind.to_string(), + request_id: request_id.clone(), + }, + }); + } + } +} + +/// The bucket object keys a notification body names, `store.prefix` included as the +/// notifications carry it. Store-agnostic. +pub fn notified_keys(v: &serde_json::Value, store_prefix: &str) -> Vec { + let mut keys = Vec::new(); + // GCS → Pub/Sub push envelope. + let attrs = &v["message"]["attributes"]; + if attrs["eventType"] == "OBJECT_FINALIZE" + && let Some(k) = attrs["objectId"].as_str() + { + keys.push(k.to_string()); + } + // S3 event notification (also what MinIO/rustfs/Ceph emit; MinIO spells the name `s3:ObjectCreated:*`). + if let Some(records) = v["Records"].as_array() { + for r in records { + if r["eventName"] + .as_str() + .is_some_and(|e| e.strip_prefix("s3:").unwrap_or(e).starts_with("ObjectCreated")) + && let Some(k) = r["s3"]["object"]["key"].as_str() + { + keys.push(percent_decode(k)); + } + } + } + // Plain shapes for your own glue. + if let Some(k) = v["key"].as_str() { + keys.push(k.to_string()); + } + // A repository id names no object: use the key its manifest has in this bucket. + if let Some(r) = v["repo"].as_str() { + keys.push(format!("{store_prefix}repos/{r}/manifest.pb")); + } + keys +} + +/// S3 URL-encodes keys in notifications (`+` is a space, `%XX` a byte). +fn percent_decode(key: &str) -> String { + let bytes = key.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + match b { + b'+' => out.push(b' '), + b'%' => { + let hex = bytes + .get(i + 1..i + 3) + .and_then(|h| std::str::from_utf8(h).ok()) + .and_then(|h| u8::from_str_radix(h, 16).ok()); + if let Some(decoded) = hex { + out.push(decoded); + i += 2; + } else { + out.push(b); + } + } + _ => out.push(b), + } + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// `prefix/repos///manifest.pb` → `o/r`: the repository whose commit point +/// `object` is. `None` for every other key. +pub fn manifest_repo(object: &str, store_prefix: &str) -> Option { + let rel = object + .strip_prefix(store_prefix)? + .strip_prefix("repos/")? + .strip_suffix("/manifest.pb")?; + let (owner, name) = rel.split_once('/')?; + if name.contains('/') { + return None; + } + RepoId::new(owner, name).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn classify_shapes() { + assert_eq!(classify("", ""), None); + assert_eq!(classify("0000", "0000"), None); + assert_eq!(classify("", "abc"), Some(Action::Create)); + assert_eq!(classify("abc", ""), Some(Action::Delete)); + assert_eq!(classify("abc", "abc"), None); + assert_eq!(classify("abc", "def"), Some(Action::Update)); + assert_eq!(ref_type("refs/heads/main"), "branch"); + assert_eq!(ref_type("refs/tags/v1"), "tag"); + assert_eq!(ref_type("refs/notes/commits"), ""); + } + + /// The golden body shape (docs/EVENTS.md). Field order and names are the + /// contract; changing this test is changing the wire. + #[test] + fn ref_event_golden() { + let ev = RefEvent { + action: Action::Update, + ref_type: "branch", + ref_name: "refs/heads/main".into(), + old: "48a0637".into(), + new: "cb38da1".into(), + pusher: "alice@example.com".into(), + correlation_id: "d1f916f7".into(), + repo: "acme/monorepo".into(), + walgit: WalgitExt { + schema_version: 1, + seq: "42".into(), + entry_kind: "push".into(), + request_id: "d1f916f7".into(), + }, + }; + let got = serde_json::to_string(&ev).unwrap(); + let want = concat!( + "{\"action\":\"update\",\"ref_type\":\"branch\",\"ref_name\":\"refs/heads/main\",", + "\"old\":\"48a0637\",\"new\":\"cb38da1\",\"pusher\":\"alice@example.com\",", + "\"correlation_id\":\"d1f916f7\",\"repo\":\"acme/monorepo\",", + "\"_walgit\":{\"schema_version\":1,\"seq\":\"42\",\"entry_kind\":\"push\",", + "\"request_id\":\"d1f916f7\"}}" + ); + assert_eq!(got, want); + assert_eq!(ev.key(), "acme/monorepo/refs/heads/main"); + } + + fn log_entry(kind: EntryKind, name: &str, old: &str, new: &str, sym: &str) -> LogEntry { + let mut meta = HashMap::new(); + meta.insert("principal".into(), "alice".into()); + meta.insert("request_id".into(), "cid".into()); + LogEntry { + seq: 9, + kind: kind as i32, + txn: Some(walgit_proto::v1::RefTransaction { + updates: vec![walgit_proto::v1::RefUpdate { + name: name.into(), + old_oid: old.into(), + new_oid: new.into(), + new_symbolic_target: sym.into(), + ..Default::default() + }], + ..Default::default() + }), + meta, + ..Default::default() + } + } + + #[test] + fn manifest_object_names() { + let prefix = "prefix/"; + let id = manifest_repo("prefix/repos/acme/monorepo/manifest.pb", prefix).unwrap(); + assert_eq!(id.to_string(), "acme/monorepo"); + // Every body shape names the same repository under a non-empty prefix. + for body in [ + serde_json::json!({"repo": "acme/monorepo"}), + serde_json::json!({"key": "prefix/repos/acme/monorepo/manifest.pb"}), + serde_json::json!({"message": {"attributes": { + "eventType": "OBJECT_FINALIZE", + "objectId": "prefix/repos/acme/monorepo/manifest.pb"}}}), + serde_json::json!({"Records": [{"eventName": "ObjectCreated:Put", + "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), + serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), + ] { + let ids: Vec<_> = notified_keys(&body, prefix) + .iter() + .filter_map(|k| manifest_repo(k, prefix)) + .map(|id| id.to_string()) + .collect(); + assert_eq!(ids, ["acme/monorepo"], "{body}"); + } + for other in [ + "repos/t/r/manifest.pb", + "prefix/repos/t/r/wal/abc.pack", + "prefix/repos/t/r/events/cursor.json", + "walgit-go/repos/t/r/manifest.pb", + "prefix/repos/t/manifest.pb", + "prefix/repos/t/r/x/manifest.pb", + ] { + assert!(manifest_repo(other, prefix).is_none(), "{other}"); + } + } + + #[test] + fn s3_keys_are_percent_decoded() { + assert_eq!(percent_decode("a%2Fb+c%zz%4"), "a/b c%zz%4"); + } + + const SHA: &str = "4a06219d6a5ba9d3e3d1a8e1e2b6f0c9d8e7f6a5"; + + #[test] + fn conversion_emits_ref_skips_compact_noop_symbolic() { + let repo = walgit_git::RepoId::new("t", "r").unwrap(); + let mut out = Vec::new(); + refs_from_entries( + &repo, + &[ + log_entry(EntryKind::Compact, "refs/heads/main", "", SHA, ""), + log_entry(EntryKind::Push, "refs/heads/main", SHA, SHA, ""), + log_entry(EntryKind::Push, "HEAD", "", "", "refs/heads/main"), + log_entry(EntryKind::Push, "refs/heads/main", "", SHA, ""), + log_entry(EntryKind::Push, "refs/heads/main", SHA, &"0".repeat(40), ""), + ], + &mut out, + ); + assert_eq!(out.len(), 2, "compact/noop/symbolic must not emit"); + let r = &out[0]; + assert_eq!(r.action, Action::Create); + assert_eq!(r.ref_name, "refs/heads/main"); + assert_eq!( + r.old, + "0".repeat(40), + "absent old is the zero OID, never empty" + ); + assert_eq!(r.new, SHA); + assert_eq!( + (r.pusher.as_str(), r.correlation_id.as_str()), + ("alice", "cid") + ); + assert_eq!((r.repo.as_str(), r.walgit.seq.as_str()), ("t/r", "9")); + assert_eq!(r.walgit.entry_kind, "push"); + let d = &out[1]; + assert_eq!(d.action, Action::Delete); + assert_eq!( + (d.old.as_str(), d.new.as_str()), + (SHA, "0".repeat(40).as_str()) + ); + } +} diff --git a/crates/walgit-nats/Cargo.toml b/crates/walgit-nats/Cargo.toml new file mode 100644 index 00000000..08d035f1 --- /dev/null +++ b/crates/walgit-nats/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "walgit-nats" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "walgit-nats" +path = "src/main.rs" + +[dependencies] +walgit-events.workspace = true +walgit-config.workspace = true +walgit-store.workspace = true +walgit-git.workspace = true +walgit-wal.workspace = true +async-nats.workspace = true +anyhow.workspace = true +thiserror.workspace = true +bytes.workspace = true +futures.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +serde.workspace = true +serde_json.workspace = true +clap.workspace = true +mimalloc.workspace = true +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12", "logging"] } + +[dev-dependencies] +walgit-proto.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs new file mode 100644 index 00000000..b3631dae --- /dev/null +++ b/crates/walgit-nats/src/lib.rs @@ -0,0 +1,58 @@ +//! `walgit-nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with +//! the cursor in the stream, wake-ups over NATS and a `$SRV` service API instead of HTTP. +//! The event model is shared with the webhook bridge through `walgit-events`. + +pub mod message; +pub mod publisher; +pub mod service; +pub mod subject; +pub mod wake; + +use std::str::FromStr; + +use anyhow::Context; +use async_nats::jetstream::{self, stream}; +use walgit_config::NatsEventsConfig; + +use crate::subject::SubjectPrefix; + +/// Connect as `[events.nats]` says. The credentials path is read, never logged. +pub async fn connect(cfg: &NatsEventsConfig) -> anyhow::Result { + let addrs = cfg + .urls + .iter() + .map(|u| async_nats::ServerAddr::from_str(u).with_context(|| format!("events.nats.urls: {u}"))) + .collect::>>()?; + let mut options = async_nats::ConnectOptions::new().name("walgit-nats"); + if let Some(path) = &cfg.credentials_file { + options = options + .credentials_file(path) + .await + .context("events.nats.credentials_file")?; + } + options.connect(addrs).await.context("connecting to NATS") +} + +/// The stream holding the events: created when missing (file storage, `.ref.>`, +/// `duplicate_window`) if `ensure_stream`, otherwise it must exist. +pub async fn open_stream( + js: &jetstream::Context, + cfg: &NatsEventsConfig, + prefix: &SubjectPrefix, +) -> anyhow::Result { + if cfg.ensure_stream { + js.get_or_create_stream(stream::Config { + name: cfg.stream.clone(), + subjects: vec![prefix.stream_filter()], + storage: stream::StorageType::File, + duplicate_window: cfg.duplicate_window, + ..stream::Config::default() + }) + .await + .with_context(|| format!("ensuring stream {}", cfg.stream)) + } else { + js.get_stream(&cfg.stream) + .await + .with_context(|| format!("stream {} (events.nats.ensure_stream is off)", cfg.stream)) + } +} diff --git a/crates/walgit-nats/src/main.rs b/crates/walgit-nats/src/main.rs new file mode 100644 index 00000000..f9178726 --- /dev/null +++ b/crates/walgit-nats/src/main.rs @@ -0,0 +1,118 @@ +//! `walgit-nats --config walgit.toml`: publish the WAL's ref events into `JetStream` (`docs/EVENTS.md`). + +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::Context; +use clap::Parser; +use tracing_subscriber::EnvFilter; +use walgit_config::{Config, LogFormat}; +use walgit_nats::publisher::Publisher; +use walgit_nats::subject::SubjectPrefix; + +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +#[derive(Parser)] +#[command(name = "walgit-nats", version, about = "walgit ref events into NATS JetStream")] +struct Cli { + /// Path to the configuration file. + #[arg(long, env = "WALGIT_CONFIG", default_value = "walgit.toml")] + config: PathBuf, +} + +fn init_tracing(cfg: &Config) { + let filter = EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new(&cfg.telemetry.log_filter)); + let builder = tracing_subscriber::fmt().with_env_filter(filter); + match cfg.telemetry.log_format { + LogFormat::Json => builder.json().init(), + LogFormat::Pretty => builder.pretty().init(), + } +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + match signal(SignalKind::terminate()) { + Ok(mut term) => { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } + return; + } + Err(e) => tracing::warn!(error = %e, "no SIGTERM handler"), + } + } + let _ = tokio::signal::ctrl_c().await; +} + +async fn run(cfg: Arc) -> anyhow::Result<()> { + let nats = &cfg.events.nats; + let prefix = SubjectPrefix::new(&nats.subject_prefix)?; + let store = walgit_store::open_store(&cfg).await?; + let registry = walgit_wal::Registry::new(store, cfg.clone()); + + let client = walgit_nats::connect(nats).await?; + let js = async_nats::jetstream::new(client.clone()); + let stream = walgit_nats::open_stream(&js, nats, &prefix).await?; + let publisher = Publisher::new(registry, js, stream, prefix.clone(), nats.start); + tracing::info!(stream = %nats.stream, prefix = %prefix, start = ?nats.start, "walgit-nats ready"); + + let mut tasks = tokio::task::JoinSet::new(); + tasks.spawn(walgit_nats::wake::notify_loop( + client.clone(), + nats.notify_subject.clone(), + nats.queue_group.clone(), + cfg.store_prefix(), + publisher.clone(), + )); + tasks.spawn({ + let (client, queue_group, publisher) = + (client.clone(), nats.queue_group.clone(), publisher.clone()); + async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } + }); + tasks.spawn({ + let publisher = publisher.clone(); + let every = cfg.events.sweep_interval; + async move { + walgit_nats::wake::sweep_loop(publisher, every).await; + std::future::pending::>().await + } + }); + + tokio::select! { + () = shutdown_signal() => tracing::info!("shutting down"), + Some(done) = tasks.join_next() => { + // A loop ending is a failure: the subscription or the service closed under us. + done.context("worker panicked")?.context("worker stopped")?; + anyhow::bail!("a worker stopped unexpectedly"); + } + } + tasks.shutdown().await; + client.flush().await.ok(); + Ok(()) +} + +fn main() -> anyhow::Result<()> { + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .map_err(|_| anyhow::anyhow!("a rustls crypto provider is already installed"))?; + let cli = Cli::parse(); + let cfg = if cli.config.exists() { + Config::load(&cli.config)? + } else { + eprintln!( + "walgit-nats: config file {} not found (pass --config PATH or WALGIT_CONFIG; `--config /dev/null` for defaults + WALGIT__ env)", + cli.config.display() + ); + std::process::exit(2); + }; + init_tracing(&cfg); + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()? + .block_on(run(Arc::new(cfg))) +} diff --git a/crates/walgit-nats/src/message.rs b/crates/walgit-nats/src/message.rs new file mode 100644 index 00000000..c2fcdbfe --- /dev/null +++ b/crates/walgit-nats/src/message.rs @@ -0,0 +1,208 @@ +//! One `RefEvent` as a `JetStream` message: body = the webhook's event JSON, headers +//! `Walgit-Seq`, `Walgit-Repo` and `Nats-Msg-Id = ::`. + +use std::fmt; + +use async_nats::header::NATS_MESSAGE_ID; +use async_nats::{HeaderMap, jetstream::message::PublishMessage}; +use bytes::Bytes; +use walgit_events::RefEvent; + +use crate::subject::RepoSubject; + +pub const HEADER_SEQ: &str = "Walgit-Seq"; +pub const HEADER_REPO: &str = "Walgit-Repo"; + +/// `::`: `JetStream`'s dedup key and, with the repository, the consumer's. +/// A repository id never contains `:` and a seq is a number, so the first two colons delimit +/// the ref name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MsgId { + repo: String, + seq: u64, + ref_name: String, +} + +impl MsgId { + pub fn new(repo: &str, seq: u64, ref_name: &str) -> Self { + MsgId { + repo: repo.to_string(), + seq, + ref_name: ref_name.to_string(), + } + } + + pub fn of(event: &RefEvent) -> Option { + Some(MsgId::new( + &event.repo, + event.walgit.seq.parse().ok()?, + &event.ref_name, + )) + } + + pub fn parse(raw: &str) -> Option { + let mut parts = raw.splitn(3, ':'); + let repo = parts.next()?; + let seq = parts.next()?.parse().ok()?; + let ref_name = parts.next()?; + Some(MsgId::new(repo, seq, ref_name)) + } + + pub fn seq(&self) -> u64 { + self.seq + } + + pub fn ref_name(&self) -> &str { + &self.ref_name + } +} + +impl fmt::Display for MsgId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}:{}", self.repo, self.seq, self.ref_name) + } +} + +/// The last event on a repository's subject, as the stream remembers it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LastPublished { + /// Stream sequence of the message (the `Nats-Expected-Last-Subject-Sequence` token). + pub stream_seq: u64, + pub id: MsgId, +} + +impl LastPublished { + pub fn from_headers(stream_seq: u64, headers: &HeaderMap) -> Option { + let id = MsgId::parse(headers.get(NATS_MESSAGE_ID)?.as_str())?; + let seq: u64 = headers.get(HEADER_SEQ)?.as_str().parse().ok()?; + (seq == id.seq()).then_some(LastPublished { stream_seq, id }) + } +} + +/// The events of `last`'s entry that were not published yet, and everything after it. One WAL +/// entry can yield several events, so a crash mid-entry leaves a tail to finish; resuming from the +/// entry (inclusive) and dropping what the stream already holds makes it exact. +pub fn after_last_published(events: Vec, last: &LastPublished) -> Vec { + let seq = last.id.seq().to_string(); + let published = events + .iter() + .position(|e| e.walgit.seq == seq && e.ref_name == last.id.ref_name()); + match published { + Some(i) => events.into_iter().skip(i + 1).collect(), + None => events, + } +} + +pub struct EventMessage { + pub subject: RepoSubject, + pub id: MsgId, + publish: PublishMessage, +} + +impl EventMessage { + pub fn new(subject: RepoSubject, event: &RefEvent) -> anyhow::Result { + let id = MsgId::of(event) + .ok_or_else(|| anyhow::anyhow!("event seq {:?} is not a number", event.walgit.seq))?; + let publish = PublishMessage::build() + .payload(Bytes::from(serde_json::to_vec(event)?)) + .message_id(id.to_string()) + .header(HEADER_SEQ, event.walgit.seq.as_str()) + .header(HEADER_REPO, event.repo.as_str()); + Ok(EventMessage { + subject, + id, + publish, + }) + } + + /// Fails the publish unless the subject's last message is `stream_seq` (`0` = none yet). + #[must_use] + pub fn expecting_last_subject_sequence(mut self, stream_seq: u64) -> Self { + self.publish = self.publish.expected_last_subject_sequence(stream_seq); + self + } + + pub fn into_publish(self) -> PublishMessage { + self.publish + } +} + +#[cfg(test)] +mod tests { + use super::*; + use walgit_events::{Action, WalgitExt}; + + fn event(seq: u64, ref_name: &str) -> RefEvent { + RefEvent { + action: Action::Update, + ref_type: "branch", + ref_name: ref_name.into(), + old: "a".into(), + new: "b".into(), + pusher: "p".into(), + correlation_id: "c".into(), + repo: "acme/repo".into(), + walgit: WalgitExt { + schema_version: 1, + seq: seq.to_string(), + entry_kind: "push".into(), + request_id: "c".into(), + }, + } + } + + #[test] + fn msg_id_round_trips_with_colons_in_the_ref() { + let id = MsgId::new("acme/repo", 42, "refs/heads/a:b"); + assert_eq!(id.to_string(), "acme/repo:42:refs/heads/a:b"); + assert_eq!(MsgId::parse(&id.to_string()), Some(id)); + assert_eq!(MsgId::parse("acme/repo:x:refs/heads/a"), None); + assert_eq!(MsgId::parse("acme/repo:1"), None); + } + + #[test] + fn resume_drops_only_what_the_stream_already_holds() { + let events = vec![ + event(5, "refs/heads/a"), + event(5, "refs/heads/b"), + event(5, "refs/heads/c"), + event(6, "refs/heads/a"), + ]; + let last = |seq, name: &str| LastPublished { + stream_seq: 9, + id: MsgId::new("acme/repo", seq, name), + }; + let names = |v: Vec| { + v.into_iter() + .map(|e| format!("{}:{}", e.walgit.seq, e.ref_name)) + .collect::>() + }; + // Crash after b: c and the next entry remain. + assert_eq!( + names(after_last_published(events.clone(), &last(5, "refs/heads/b"))), + ["5:refs/heads/c", "6:refs/heads/a"] + ); + // The whole entry was published. + assert_eq!( + names(after_last_published(events.clone(), &last(5, "refs/heads/c"))), + ["6:refs/heads/a"] + ); + // Nothing new at all. + assert!(after_last_published(events.clone(), &last(6, "refs/heads/a")).is_empty()); + // The message is not among the events (entry rewritten or folded): publish everything. + assert_eq!( + after_last_published(events.clone(), &last(5, "refs/heads/zzz")).len(), + 4 + ); + } + + #[test] + fn message_carries_the_contract_headers() { + let prefix = crate::subject::SubjectPrefix::new("walgit").unwrap(); + let repo = walgit_git::RepoId::new("acme", "repo").unwrap(); + let ev = event(7, "refs/heads/main"); + let msg = EventMessage::new(RepoSubject::new(&prefix, &repo), &ev).unwrap(); + assert_eq!(msg.id.to_string(), "acme/repo:7:refs/heads/main"); + assert_eq!(msg.subject.as_str(), "walgit.ref.acme.repo"); + } +} diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs new file mode 100644 index 00000000..96428059 --- /dev/null +++ b/crates/walgit-nats/src/publisher.rs @@ -0,0 +1,284 @@ +//! The `JetStream` publisher: the WAL → `.ref..`. +//! +//! `catch_up(repo)`: the cursor is the stream. The last message on the repo's subject names the +//! WAL seq (`Walgit-Seq`) and ref it reached; the publisher reads the log from that entry +//! (inclusive: one entry can yield several events and a crash mid-entry must not drop the rest), +//! drops the events the stream already holds, and publishes the remainder one at a time, waiting +//! for each `PubAck`. The first publish of a catch-up carries +//! `Nats-Expected-Last-Subject-Sequence`, so a second publisher that got there first makes this one +//! stop (the next wake-up retries); every message carries `Nats-Msg-Id` (`::`) so +//! `JetStream` dedups within its window. Consumers still dedup on `(repo, _walgit.seq, ref_name)`. +//! +//! The cursor is the sink's state, not walgit's (D50): wiping walgit loses nothing, wiping the +//! stream republishes the WAL window (`events.nats.start` decides where a cold repo begins). + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context; +use async_nats::jetstream::{ + self, + context::PublishErrorKind, + stream::{LastRawMessageErrorKind, Stream}, +}; +use walgit_config::NatsStart; +use walgit_events::refs_from_entries; +use walgit_git::RepoId; + +use crate::message::{EventMessage, LastPublished, after_last_published}; +use crate::subject::{RepoSubject, SubjectPrefix}; + +/// What one `catch_up` did. +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct CatchUp { + pub repo: String, + /// The WAL seq the stream had reached for this repository (or the cold start point). + pub from_seq: u64, + pub head_seq: u64, + /// Messages published now (`JetStream`-deduplicated ones included). + pub emitted: usize, + /// `[first, last]` seqs folded into a checkpoint before they were read: not emitted, counted; + /// consumers backfill from the WAL. + #[serde(skip_serializing_if = "Option::is_none")] + pub gap: Option<[u64; 2]>, + /// Another publisher advanced the subject first; this catch-up stopped and the next retries. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub conflict: bool, +} + +#[derive(Debug, Default)] +pub struct Counters { + pub published: AtomicU64, + pub deduplicated: AtomicU64, + pub gap_entries: AtomicU64, + pub conflicts: AtomicU64, + pub sweep_found: AtomicU64, +} + +impl Counters { + pub fn snapshot(&self) -> serde_json::Value { + let n = |c: &AtomicU64| c.load(Ordering::Relaxed); + serde_json::json!({ + "published": n(&self.published), + "deduplicated": n(&self.deduplicated), + "gap_entries": n(&self.gap_entries), + "conflicts": n(&self.conflicts), + "sweep_found": n(&self.sweep_found), + }) + } +} + +pub struct Publisher { + registry: Arc, + js: jetstream::Context, + stream: Stream, + prefix: SubjectPrefix, + start: NatsStart, + serial: tokio::sync::Mutex<()>, + /// Head each repository was fully published up to by this process: spares the log read when a + /// wake-up finds nothing new. Never a source of truth (the stream is). + done: std::sync::Mutex>, + /// `start = "head"`: the head a cold repository was first seen at. + cold_marks: std::sync::Mutex>, + pub counters: Counters, +} + +fn locked(m: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl Publisher { + pub fn new( + registry: Arc, + js: jetstream::Context, + stream: Stream, + prefix: SubjectPrefix, + start: NatsStart, + ) -> Arc { + Arc::new(Publisher { + registry, + js, + stream, + prefix, + start, + serial: tokio::sync::Mutex::new(()), + done: std::sync::Mutex::default(), + cold_marks: std::sync::Mutex::default(), + counters: Counters::default(), + }) + } + + pub fn registry(&self) -> &Arc { + &self.registry + } + + async fn last_published(&self, subject: &RepoSubject) -> anyhow::Result> { + match self + .stream + .get_last_raw_message_by_subject(subject.as_str()) + .await + { + Ok(m) => { + let last = LastPublished::from_headers(m.sequence, &m.headers); + anyhow::ensure!( + last.is_some(), + "last message on {subject} (stream seq {}) lacks Walgit-Seq / Nats-Msg-Id", + m.sequence + ); + Ok(last) + } + Err(e) if matches!(e.kind(), LastRawMessageErrorKind::NoMessageFound) => Ok(None), + Err(e) => Err(e).with_context(|| format!("reading the last message on {subject}")), + } + } + + /// Publish everything committed after what the stream holds for `id`. + pub async fn catch_up(&self, id: &RepoId) -> anyhow::Result { + let _serial = self.serial.lock().await; + let handle = self.registry.open(id).await?; + let manifest = { + let guard = handle.sync_refs().await?; + let m = handle.manifest(); + drop(guard); + m + }; + let head = manifest.head_seq; + // Entries below `min_seq` are folded into a checkpoint and no longer in the manifest's + // log window; `min_seq - 1` is the newest seq that is lost to us. + let readable_from = manifest.min_seq.saturating_sub(1); + let lowest = readable_from + 1; + + let mut report = CatchUp { + repo: id.to_string(), + from_seq: head, + head_seq: head, + ..CatchUp::default() + }; + if locked(&self.done).get(id).is_some_and(|d| *d >= head) { + return Ok(report); + } + + let subject = RepoSubject::new(&self.prefix, id); + let last = self.last_published(&subject).await?; + + // First seq to read; `resume` is the entry the stream may hold only part of. + let (first, resume) = match &last { + Some(l) if l.id.seq() >= lowest => (l.id.seq(), Some(l)), + Some(l) => { + // The newest published entry is already folded away: what follows it up to the + // log window is lost to us. + let published = l.id.seq(); + if published < readable_from { + report.gap = Some([published + 1, readable_from]); + self.counters + .gap_entries + .fetch_add(readable_from - published, Ordering::Relaxed); + tracing::warn!(repo = %id, first = published + 1, last = readable_from, + "entries folded into a checkpoint before they were published; consumers backfill from the WAL"); + } + (lowest, None) + } + None => match self.start { + NatsStart::Oldest => (lowest, None), + NatsStart::Head => { + let mark = *locked(&self.cold_marks).entry(id.clone()).or_insert(head); + (mark.saturating_add(1).max(lowest), None) + } + }, + }; + report.from_seq = resume.map_or(first.saturating_sub(1), |l| l.id.seq()); + + if first <= head { + let entries = handle.read_log(&manifest, first, Some(head)).await?; + let mut events = Vec::new(); + refs_from_entries(id, &entries, &mut events); + if let Some(l) = resume { + events = after_last_published(events, l); + } + let mut expected = resume.map_or(0, |l| l.stream_seq); + for (i, event) in events.iter().enumerate() { + let mut msg = EventMessage::new(subject.clone(), event)?; + if i == 0 { + msg = msg.expecting_last_subject_sequence(expected); + } + let (msg_id, subj) = (msg.id.clone(), msg.subject.to_string()); + let ack = self + .js + .send_publish(subj, msg.into_publish()) + .await + .with_context(|| format!("publishing {msg_id}"))?; + match ack.await { + Ok(ack) => { + expected = ack.sequence; + report.emitted += 1; + if ack.duplicate { + self.counters.deduplicated.fetch_add(1, Ordering::Relaxed); + } else { + self.counters.published.fetch_add(1, Ordering::Relaxed); + } + // One structured line per published event, as the bridge logs. + if let Ok(body) = serde_json::to_string(event) { + tracing::info!(target: "walgit::event", + event_type = "ref", repo = %event.repo, event = %body, "event"); + } + } + Err(e) if e.kind() == PublishErrorKind::WrongLastSequence => { + self.counters.conflicts.fetch_add(1, Ordering::Relaxed); + report.conflict = true; + tracing::warn!(repo = %id, "another publisher advanced {subject}; stopping this catch-up, the next wake-up retries"); + return Ok(report); + } + Err(e) => return Err(e).with_context(|| format!("PubAck for {msg_id}")), + } + } + } + + locked(&self.done).insert(id.clone(), head); + if report.emitted > 0 { + tracing::info!(repo = %id, from = report.from_seq, head, emitted = report.emitted, "published"); + } + Ok(report) + } + + /// `catch_up` that treats a repository deleted since its notification as nothing to do. + pub async fn catch_up_if_present(&self, id: &RepoId) -> anyhow::Result> { + match self.catch_up(id).await { + Err(e) + if matches!( + e.downcast_ref::(), + Some(walgit_wal::WalError::NotFound) + ) => + { + Ok(None) + } + r => r.map(Some), + } + } + + /// Every repository (a `list` + one conditional manifest GET each): the backstop behind the + /// notifications and the health check. `warn_found`: finding work here means they are not + /// flowing (the startup sweep legitimately finds work and passes `false`). + pub async fn sweep(&self, warn_found: bool) { + let repos = match self.registry.list().await { + Ok(r) => r, + Err(e) => { + tracing::warn!(error = %e, "sweep list failed"); + return; + } + }; + for id in repos { + match self.catch_up_if_present(&id).await { + Ok(Some(c)) if c.emitted > 0 && warn_found => { + self.counters + .sweep_found + .fetch_add(c.emitted as u64, Ordering::Relaxed); + tracing::warn!(repo = %id, emitted = c.emitted, + "sweep found unpublished entries: are the bucket notifications flowing to the notify subject?"); + } + Ok(_) => {} + Err(e) => tracing::warn!(repo = %id, error = %format!("{e:#}"), "sweep catch-up failed"), + } + } + } +} diff --git a/crates/walgit-nats/src/service.rs b/crates/walgit-nats/src/service.rs new file mode 100644 index 00000000..5e311e41 --- /dev/null +++ b/crates/walgit-nats/src/service.rs @@ -0,0 +1,76 @@ +//! The NATS service API (`$SRV` micro protocol) in place of an HTTP listener: +//! `.events.catch-up` takes `{"repo":"owner/name"}` and answers with the catch-up report; +//! `$SRV.PING|INFO|STATS` come with the protocol (stats carry this publisher's counters). + +use std::str::FromStr; +use std::sync::Arc; + +use async_nats::service::ServiceExt; +use async_nats::service::error::Error as ServiceError; +use bytes::Bytes; +use futures::StreamExt; +use walgit_git::RepoId; + +use crate::publisher::Publisher; +use crate::subject::SubjectPrefix; + +pub const SERVICE_NAME: &str = "walgit-events"; + +#[derive(serde::Deserialize)] +struct CatchUpRequest { + repo: String, +} + +fn failure(code: usize, status: &impl ToString) -> ServiceError { + ServiceError { + status: status.to_string(), + code, + } +} + +async fn handle(publisher: &Publisher, payload: &[u8]) -> Result { + let req: CatchUpRequest = + serde_json::from_slice(payload).map_err(|e| failure(400, &format!("request: {e}")))?; + let id = RepoId::from_str(&req.repo).map_err(|e| failure(400, &e))?; + match publisher.catch_up_if_present(&id).await { + Ok(Some(report)) => serde_json::to_vec(&report) + .map(Bytes::from) + .map_err(|e| failure(500, &e)), + Ok(None) => Err(failure(404, &format!("no repository {id}"))), + Err(e) => Err(failure(500, &format!("{e:#}"))), + } +} + +/// Serve until the endpoint closes. Instances share requests through the service queue group. +pub async fn serve( + client: async_nats::Client, + prefix: &SubjectPrefix, + queue_group: &str, + publisher: Arc, +) -> anyhow::Result<()> { + let stats = publisher.clone(); + let service = client + .service_builder() + .description("walgit ref events into JetStream") + .queue_group(queue_group) + .stats_handler(move |_endpoint, _stats| stats.counters.snapshot()) + .start(SERVICE_NAME, env!("CARGO_PKG_VERSION")) + .await + .map_err(|e| anyhow::anyhow!("starting the {SERVICE_NAME} service: {e}"))?; + let mut endpoint = service + .group(prefix.child("events")) + .endpoint("catch-up") + .await + .map_err(|e| anyhow::anyhow!("registering catch-up: {e}"))?; + tracing::info!(subject = %prefix.child("events.catch-up"), "service ready"); + while let Some(request) = endpoint.next().await { + let publisher = publisher.clone(); + tokio::spawn(async move { + let answer = handle(&publisher, &request.message.payload).await; + if let Err(e) = request.respond(answer).await { + tracing::warn!(error = %e, "catch-up reply failed"); + } + }); + } + Ok(()) +} diff --git a/crates/walgit-nats/src/subject.rs b/crates/walgit-nats/src/subject.rs new file mode 100644 index 00000000..1d115b37 --- /dev/null +++ b/crates/walgit-nats/src/subject.rs @@ -0,0 +1,209 @@ +//! Subjects of the `JetStream` events: `.ref..`. +//! +//! A NATS subject token may not contain `.`, `*`, `>` or whitespace, and a repository +//! name may contain `.` (`RepoId` allows `[A-Za-z0-9._-]`). Each of owner and repo is +//! therefore one [`Token`]: every byte outside `[A-Za-z0-9_-]` becomes `%XX` (uppercase hex of +//! the UTF-8 byte, `%` itself included), so `acme.io` is `acme%2Eio` and the encoding is +//! reversible token by token. + +use std::fmt::{self, Write as _}; + +use walgit_git::RepoId; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SubjectError { + #[error("subject prefix {0:?} must be dot-separated tokens without '*', '>' or whitespace")] + InvalidPrefix(String), + #[error("{0:?} is not a repository subject under this prefix")] + NotARepoSubject(String), +} + +/// One encoded subject token (never empty, never contains `.`, `*`, `>` or whitespace). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token(String); + +impl Token { + pub fn encode(raw: &str) -> Self { + let mut out = String::with_capacity(raw.len()); + for b in raw.bytes() { + if b.is_ascii_alphanumeric() || b == b'_' || b == b'-' { + out.push(char::from(b)); + } else { + let _ = write!(out, "%{b:02X}"); + } + } + Token(out) + } + + /// `None` when `encoded` is not what [`Token::encode`] produces. + pub fn decode(encoded: &str) -> Option { + let bytes = encoded.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + match b { + b'%' => { + let hex = bytes.get(i + 1..i + 3)?; + if !hex + .iter() + .all(|h| h.is_ascii_digit() || (b'A'..=b'F').contains(h)) + { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(hex).ok()?, 16).ok()?); + i += 3; + } + _ if b.is_ascii_alphanumeric() || b == b'_' || b == b'-' => { + out.push(b); + i += 1; + } + _ => return None, + } + } + if out.is_empty() { + return None; + } + String::from_utf8(out).ok() + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Token { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// The configured `events.nats.subject_prefix`: dot-separated, literal tokens. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubjectPrefix(String); + +impl SubjectPrefix { + pub fn new(raw: &str) -> Result { + let literal = |t: &str| { + !t.is_empty() + && !t + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '*' | '>')) + }; + if raw.split('.').all(literal) { + Ok(SubjectPrefix(raw.to_string())) + } else { + Err(SubjectError::InvalidPrefix(raw.to_string())) + } + } + + /// `.ref.>`: what the stream captures. + pub fn stream_filter(&self) -> String { + format!("{}.ref.>", self.0) + } + + /// `.` for the service API. + pub fn child(&self, name: &str) -> String { + format!("{}.{name}", self.0) + } + + fn ref_root(&self) -> String { + format!("{}.ref.", self.0) + } +} + +impl fmt::Display for SubjectPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// `.ref..`: one repository's events, in WAL order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoSubject(String); + +impl RepoSubject { + pub fn new(prefix: &SubjectPrefix, repo: &RepoId) -> Self { + RepoSubject(format!( + "{}{}.{}", + prefix.ref_root(), + Token::encode(repo.owner()), + Token::encode(repo.name()) + )) + } + + pub fn parse(prefix: &SubjectPrefix, subject: &str) -> Result { + let err = || SubjectError::NotARepoSubject(subject.to_string()); + let rest = subject.strip_prefix(&prefix.ref_root()).ok_or_else(err)?; + let (owner, name) = rest.split_once('.').ok_or_else(err)?; + let owner = Token::decode(owner).ok_or_else(err)?; + let name = Token::decode(name).ok_or_else(err)?; + RepoId::new(owner, name).map_err(|_| err()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for RepoSubject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn prefix() -> SubjectPrefix { + SubjectPrefix::new("walgit").unwrap() + } + + #[test] + fn dots_are_escaped_and_round_trip() { + let repo = RepoId::new("acme.io", "mono.repo-v2_x").unwrap(); + let subject = RepoSubject::new(&prefix(), &repo); + assert_eq!(subject.as_str(), "walgit.ref.acme%2Eio.mono%2Erepo-v2_x"); + assert_eq!(RepoSubject::parse(&prefix(), subject.as_str()).unwrap(), repo); + } + + #[test] + fn token_never_carries_a_nats_metacharacter() { + let t = Token::encode("a.b*c>d e\t%é/"); + assert!(!t.as_str().contains(['.', '*', '>', ' ', '\t', '/'])); + assert_eq!(t.as_str(), "a%2Eb%2Ac%3Ed%20e%09%25%C3%A9%2F"); + assert_eq!(Token::decode(t.as_str()).unwrap(), "a.b*c>d e\t%é/"); + } + + #[test] + fn decode_refuses_what_encode_never_emits() { + for bad in ["", "a.b", "a%2", "a%zz", "a%2e", "a b", "a*", "%FF"] { + assert_eq!(Token::decode(bad), None, "{bad:?}"); + } + } + + #[test] + fn parse_refuses_foreign_subjects() { + for bad in [ + "walgit.ref.acme", + "walgit.ref.acme.repo.extra", + "other.ref.acme.repo", + "walgit.refs.acme.repo", + "walgit.ref..repo", + "walgit.ref.%2E%2E.repo", + ] { + assert!(RepoSubject::parse(&prefix(), bad).is_err(), "{bad}"); + } + } + + #[test] + fn prefix_validation_and_filter() { + assert_eq!(prefix().stream_filter(), "walgit.ref.>"); + let p = SubjectPrefix::new("acme.git").unwrap(); + assert_eq!(p.stream_filter(), "acme.git.ref.>"); + assert_eq!(p.child("events"), "acme.git.events"); + for bad in ["", ".a", "a.", "a..b", "a.*", "a.>", "a b"] { + assert!(SubjectPrefix::new(bad).is_err(), "{bad:?}"); + } + } +} diff --git a/crates/walgit-nats/src/wake.rs b/crates/walgit-nats/src/wake.rs new file mode 100644 index 00000000..f82f6b54 --- /dev/null +++ b/crates/walgit-nats/src/wake.rs @@ -0,0 +1,84 @@ +//! Wake-ups: a bucket notification on the notify subject, and the periodic sweep. +//! Both only ever call `catch_up`, so both are idempotent. + +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use futures::StreamExt; +use walgit_events::{manifest_repo, notified_keys}; +use walgit_git::RepoId; + +use crate::publisher::Publisher; + +/// The repositories a notification body names: an S3 event (`Records[].s3.object.key`, what +/// `MinIO` and rustfs publish to NATS natively), a GCS envelope, `{"key": "..."}` or +/// `{"repo": "owner/name"}`. Anything else names nothing. +pub fn notified_repos(payload: &[u8], store_prefix: &str) -> BTreeSet { + let Ok(value) = serde_json::from_slice::(payload) else { + return BTreeSet::new(); + }; + notified_keys(&value, store_prefix) + .iter() + .filter_map(|k| manifest_repo(k, store_prefix)) + .collect() +} + +/// Subscribe to `subject` in `queue_group` (instances share the notifications) until the +/// subscription ends. A failed catch-up is logged, not retried: core NATS does not redeliver, and +/// the sweep is the backstop. +pub async fn notify_loop( + client: async_nats::Client, + subject: String, + queue_group: String, + store_prefix: String, + publisher: Arc, +) -> anyhow::Result<()> { + let mut sub = client.queue_subscribe(subject.clone(), queue_group).await?; + tracing::info!(%subject, "listening for bucket notifications"); + while let Some(msg) = sub.next().await { + for id in notified_repos(&msg.payload, &store_prefix) { + if let Err(e) = publisher.catch_up_if_present(&id).await { + tracing::warn!(repo = %id, error = %format!("{e:#}"), "notified catch-up failed; the sweep retries"); + } + } + } + Ok(()) +} + +/// One sweep at startup (quiet: it legitimately finds work), then every `every` (`0` = never again). +pub async fn sweep_loop(publisher: Arc, every: Duration) { + publisher.sweep(false).await; + if every.is_zero() { + return; + } + let mut tick = tokio::time::interval(every); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + tick.tick().await; + loop { + tick.tick().await; + publisher.sweep(true).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_notification_shape_names_the_repo() { + let key = "pre/repos/acme/mono/manifest.pb"; + for body in [ + serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", "s3": {"object": {"key": key}}}]}), + serde_json::json!({"key": key}), + serde_json::json!({"repo": "acme/mono"}), + ] { + let repos = notified_repos(body.to_string().as_bytes(), "pre/"); + let names: Vec<_> = repos.iter().map(ToString::to_string).collect(); + assert_eq!(names, ["acme/mono"], "{body}"); + } + assert!(notified_repos(b"not json", "pre/").is_empty()); + let other = serde_json::json!({"key": "pre/repos/acme/mono/wal/x.pack"}); + assert!(notified_repos(other.to_string().as_bytes(), "pre/").is_empty()); + } +} diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs new file mode 100644 index 00000000..fdc1e73b --- /dev/null +++ b/crates/walgit-nats/tests/jetstream.rs @@ -0,0 +1,398 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used, clippy::expect_used)] + +//! The `JetStream` publisher against a real `nats-server -js` and the in-memory walgit store: +//! the stream is the cursor (resume without republishing), crash-mid-entry completion, the +//! wake-ups and the service API. Skips (passes) when `nats-server` is not on PATH. + +use std::collections::HashMap; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_nats::jetstream; +use futures::StreamExt; +use walgit_config::{Config, NatsStart}; +use walgit_events::refs_from_entries; +use walgit_git::{ObjectFormat, RepoId}; +use walgit_nats::message::EventMessage; +use walgit_nats::publisher::Publisher; +use walgit_nats::subject::{RepoSubject, SubjectPrefix}; +use walgit_proto::v1::{RefTransaction, RefUpdate}; + +const ZERO: &str = "0000000000000000000000000000000000000000"; +const A: &str = "1111111111111111111111111111111111111111"; +const B: &str = "2222222222222222222222222222222222222222"; +const C: &str = "3333333333333333333333333333333333333333"; + +struct NatsServer { + child: Child, + url: String, + _store: tempfile::TempDir, +} + +impl Drop for NatsServer { + fn drop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } +} + +async fn start_nats() -> Option { + if !Command::new("nats-server") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + { + eprintln!("skipping: nats-server is not available on PATH"); + return None; + } + let port = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let store = tempfile::tempdir().unwrap(); + let child = Command::new("nats-server") + .args(["-js", "-a", "127.0.0.1", "-p", &port.to_string(), "-sd"]) + .arg(store.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let url = format!("nats://127.0.0.1:{port}"); + let server = NatsServer { + child, + url, + _store: store, + }; + let t0 = Instant::now(); + while async_nats::connect(&server.url).await.is_err() { + assert!(t0.elapsed() < Duration::from_secs(15), "nats-server did not start"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + Some(server) +} + +struct Rig { + cfg: Arc, + registry: Arc, + client: async_nats::Client, + js: jetstream::Context, + prefix: SubjectPrefix, + _cache: tempfile::TempDir, +} + +impl Rig { + async fn new(nats: &NatsServer, start: NatsStart) -> Self { + let cache = tempfile::tempdir().unwrap(); + let mut cfg = Config::default(); + cfg.store.backend = walgit_config::StoreBackend::Memory; + cfg.cache.dir = cache.path().to_path_buf(); + cfg.events.nats.urls = vec![nats.url.clone()]; + cfg.events.nats.start = start; + let cfg = Arc::new(cfg); + let registry = walgit_wal::Registry::new( + Arc::new(walgit_store::memory::MemoryStore::new()), + cfg.clone(), + ); + let client = walgit_nats::connect(&cfg.events.nats).await.unwrap(); + let js = jetstream::new(client.clone()); + let prefix = SubjectPrefix::new(&cfg.events.nats.subject_prefix).unwrap(); + Rig { + cfg, + registry, + client, + js, + prefix, + _cache: cache, + } + } + + /// A publisher with no memory of earlier ones, as after a restart. + async fn publisher(&self, start: NatsStart) -> Arc { + let stream = walgit_nats::open_stream(&self.js, &self.cfg.events.nats, &self.prefix) + .await + .unwrap(); + Publisher::new( + self.registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + start, + ) + } + + async fn push(&self, id: &RepoId, updates: &[(&str, &str, &str)]) -> u64 { + let handle = self.registry.open(id).await.unwrap(); + let txn = RefTransaction { + updates: updates + .iter() + .map(|(name, old, new)| RefUpdate { + name: (*name).into(), + old_oid: (*old).into(), + new_oid: (*new).into(), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + let meta = HashMap::from([ + ("principal".to_string(), "alice".to_string()), + ("request_id".to_string(), "rid".to_string()), + ]); + let r = handle.publish_ref_update(txn, meta).await.unwrap(); + assert!(r.seq > 0, "push rejected: {:?}", r.per_ref); + r.seq + } + + async fn stream(&self) -> jetstream::stream::Stream { + self.js + .get_stream(&self.cfg.events.nats.stream) + .await + .unwrap() + } + + async fn message_count(&self) -> u64 { + self.stream().await.get_info().await.unwrap().state.messages + } + + async fn wait_for_messages(&self, n: u64) { + let t0 = Instant::now(); + while self.message_count().await < n { + assert!(t0.elapsed() < Duration::from_secs(10), "timed out waiting for {n} messages"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stream_is_the_cursor() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("acme.io", "mono.repo").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + rig.push(&id, &[("refs/heads/main", A, B)]).await; + + let publisher = rig.publisher(NatsStart::Oldest).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.head_seq, r.emitted, r.conflict), (0, 2, 2, false)); + assert_eq!(rig.message_count().await, 2); + + let stream = rig.stream().await; + let first = stream.get_raw_message(1).await.unwrap(); + assert_eq!(first.subject.as_str(), "walgit.ref.acme%2Eio.mono%2Erepo"); + assert_eq!(first.headers.get("Walgit-Seq").unwrap().as_str(), "1"); + assert_eq!(first.headers.get("Walgit-Repo").unwrap().as_str(), "acme.io/mono.repo"); + assert_eq!( + first.headers.get("Nats-Msg-Id").unwrap().as_str(), + "acme.io/mono.repo:1:refs/heads/main" + ); + let body: serde_json::Value = serde_json::from_slice(&first.payload).unwrap(); + assert_eq!(body["action"], "create"); + assert_eq!(body["old"], ZERO); + assert_eq!(body["new"], A); + assert_eq!(body["repo"], "acme.io/mono.repo"); + assert_eq!(body["pusher"], "alice"); + assert_eq!(body["_walgit"]["seq"], "1"); + let second = stream.get_raw_message(2).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&second.payload).unwrap(); + assert_eq!((body["action"].as_str(), body["_walgit"]["seq"].as_str()), (Some("update"), Some("2"))); + + // Restart: a publisher with no memory resumes from the stream and publishes nothing. + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted), (2, 0)); + assert_eq!(rig.message_count().await, 2, "resume must not republish"); + + // New pushes are published, and only those. + rig.push(&id, &[("refs/heads/main", B, ZERO)]).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 3, 1)); + assert_eq!(rig.message_count().await, 3); + let third = stream.get_raw_message(3).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&third.payload).unwrap(); + assert_eq!((body["action"].as_str(), body["new"].as_str()), (Some("delete"), Some(ZERO))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_crash_mid_entry_finishes_that_entry_only() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + // One entry, three refs; the "crash" leaves only the first event in the stream. + let seq = rig + .push( + &id, + &[ + ("refs/heads/a", ZERO, B), + ("refs/heads/b", ZERO, B), + ("refs/tags/c", ZERO, C), + ], + ) + .await; + let handle = rig.registry.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, seq, Some(seq)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(&id, &entries, &mut events); + assert_eq!(events.len(), 3); + let msg = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &events[0]).unwrap(); + let subject = msg.subject.to_string(); + rig.js + .send_publish(subject, msg.into_publish()) + .await + .unwrap() + .await + .unwrap(); + assert_eq!(rig.message_count().await, 2); + + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted), (seq, 2), "the rest of the entry, nothing twice"); + assert_eq!(rig.message_count().await, 4); + let last = rig.stream().await.get_raw_message(4).await.unwrap(); + assert_eq!( + last.headers.get("Nats-Msg-Id").unwrap().as_str(), + format!("t/r:{seq}:refs/tags/c") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn head_start_publishes_nothing_older() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Head).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); + assert_eq!(rig.message_count().await, 0); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 1); + let m = rig.stream().await.get_raw_message(1).await.unwrap(); + assert_eq!(m.headers.get("Walgit-Seq").unwrap().as_str(), "2"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_publisher_that_moved_first_stops_this_one() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + publisher.catch_up(&id).await.unwrap(); + + // The subject moves (another instance published a different entry) between our read of the + // last message and our first publish: emulate by publishing a foreign message afterwards, + // with a stale expectation. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + let handle = rig.registry.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, 2, Some(2)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(&id, &entries, &mut events); + let stale = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &events[0]) + .unwrap() + .expecting_last_subject_sequence(0); + let subject = stale.subject.to_string(); + let err = rig + .js + .send_publish(subject, stale.into_publish()) + .await + .unwrap() + .await + .unwrap_err(); + assert_eq!(err.kind(), jetstream::context::PublishErrorKind::WrongLastSequence); + assert_eq!(rig.message_count().await, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn notifications_and_the_service_api_wake_the_publisher() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let publisher = rig.publisher(NatsStart::Oldest).await; + + let nats_cfg = &rig.cfg.events.nats; + let notify = tokio::spawn(walgit_nats::wake::notify_loop( + rig.client.clone(), + nats_cfg.notify_subject.clone(), + nats_cfg.queue_group.clone(), + rig.cfg.store_prefix(), + publisher.clone(), + )); + let service = tokio::spawn({ + let (client, prefix, publisher) = + (rig.client.clone(), rig.prefix.clone(), publisher.clone()); + let queue_group = nats_cfg.queue_group.clone(); + async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } + }); + tokio::time::sleep(Duration::from_millis(500)).await; + + // An S3 bucket notification, as the object store publishes it. + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "repos/t/r/manifest.pb"}}}]}); + rig.client + .publish(nats_cfg.notify_subject.clone(), s3.to_string().into()) + .await + .unwrap(); + rig.wait_for_messages(1).await; + + // {"repo": ...} through the same subject. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.client + .publish(nats_cfg.notify_subject.clone(), r#"{"repo":"t/r"}"#.into()) + .await + .unwrap(); + rig.wait_for_messages(2).await; + + // The service endpoint. + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!((report["repo"].as_str(), report["emitted"].as_u64()), (Some("t/r"), Some(1))); + assert_eq!(rig.message_count().await, 3); + + let missing = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/none"}"#.into()) + .await + .unwrap(); + let code = missing.headers.as_ref().and_then(|h| h.get("Nats-Service-Error-Code")); + assert_eq!(code.map(|c| c.as_str().to_string()), Some("404".into())); + + // The micro protocol's built-ins answer too. + let mut pings = rig.client.subscribe("inbox.ping").await.unwrap(); + rig.client + .publish_with_reply("$SRV.PING.walgit-events", "inbox.ping", "".into()) + .await + .unwrap(); + let pong = tokio::time::timeout(Duration::from_secs(5), pings.next()) + .await + .unwrap() + .unwrap(); + let pong: serde_json::Value = serde_json::from_slice(&pong.payload).unwrap(); + assert_eq!(pong["name"], "walgit-events"); + + notify.abort(); + service.abort(); +} diff --git a/crates/walgit-server/Cargo.toml b/crates/walgit-server/Cargo.toml index 9ad63412..d922b95f 100644 --- a/crates/walgit-server/Cargo.toml +++ b/crates/walgit-server/Cargo.toml @@ -9,6 +9,7 @@ humantime.workspace = true walgit-proto.workspace = true walgit-store.workspace = true walgit-config.workspace = true +walgit-events.workspace = true walgit-git.workspace = true walgit-wal.workspace = true async-trait.workspace = true diff --git a/crates/walgit-server/src/bridge.rs b/crates/walgit-server/src/bridge.rs index c51ec2fb..f508682f 100644 --- a/crates/walgit-server/src/bridge.rs +++ b/crates/walgit-server/src/bridge.rs @@ -29,7 +29,9 @@ use chrono::Utc; use walgit_git::RepoId; use walgit_store::{ObjectStoreExt, PutMode, StoreError}; -use crate::events::{self, RefEvent, Sink}; +use walgit_events::{RefEvent, manifest_repo, notified_keys, refs_from_entries}; + +use crate::events::{self, Sink}; const CURSOR_KEY: &str = "events/cursor.json"; @@ -141,7 +143,7 @@ impl Bridge { if head > from { let entries = handle.read_log(&manifest, from + 1, Some(head)).await?; let mut batch = Vec::new(); - events::refs_from_entries(id, &entries, &mut batch); + refs_from_entries(id, &entries, &mut batch); // A failing sink returns here: cursor untouched, retried on the // next wake-up. self.publish(&batch).await?; @@ -205,7 +207,7 @@ impl Bridge { /// A GCS notification: `…/repos///manifest.pb` finalized ⇒ that /// repo committed something. Everything else is ignored. pub async fn object_finalized(&self, object: &str) -> anyhow::Result> { - let Some(id) = self.manifest_repo(object) else { + let Some(id) = manifest_repo(object, &self.store_prefix) else { return Ok(None); }; match self.catch_up(&id).await { @@ -223,19 +225,6 @@ impl Bridge { } } - /// `prefix/repos///manifest.pb` → `o/r`. - fn manifest_repo(&self, object: &str) -> Option { - let rel = object - .strip_prefix(self.store_prefix.as_str())? - .strip_prefix("repos/")? - .strip_suffix("/manifest.pb")?; - let (owner, name) = rel.split_once('/')?; - if name.contains('/') { - return None; - } - RepoId::new(owner, name).ok() - } - async fn publish(&self, batch: &[RefEvent]) -> anyhow::Result<()> { if batch.is_empty() { return Ok(()); @@ -259,55 +248,6 @@ impl Bridge { } } -/// The bucket object keys a notification body names, `store.prefix` included as the -/// notifications carry it. Store-agnostic. -fn notified_keys(v: &serde_json::Value, store_prefix: &str) -> Vec { - let mut keys = Vec::new(); - // GCS → Pub/Sub push envelope. - let attrs = &v["message"]["attributes"]; - if attrs["eventType"] == "OBJECT_FINALIZE" - && let Some(k) = attrs["objectId"].as_str() - { - keys.push(k.to_string()); - } - // S3 event notification (also what MinIO/rustfs/Ceph emit). - if let Some(records) = v["Records"].as_array() { - for r in records { - if r["eventName"] - .as_str() - .is_some_and(|e| e.starts_with("ObjectCreated")) - && let Some(k) = r["s3"]["object"]["key"].as_str() - { - // S3 URL-encodes keys in notifications. - keys.push( - k.replace('+', " ") - .split('%') - .enumerate() - .map(|(i, part)| { - if i == 0 { - return part.to_string(); - } - match u8::from_str_radix(part.get(..2).unwrap_or(""), 16) { - Ok(b) => format!("{}{}", b as char, &part[2..]), - Err(_) => format!("%{part}"), - } - }) - .collect(), - ); - } - } - } - // Plain shapes for your own glue. - if let Some(k) = v["key"].as_str() { - keys.push(k.to_string()); - } - // A repository id names no object: use the key its manifest has in this bucket. - if let Some(r) = v["repo"].as_str() { - keys.push(format!("{store_prefix}repos/{r}/manifest.pb")); - } - keys -} - /// `POST /_events/notify`: a bucket notification naming a finalized `manifest.pb`. /// `200` (ack) when handled or ignored, `503` (redeliver) when a sink failed. pub async fn http_notify( @@ -373,52 +313,3 @@ pub fn spawn_sweeper(state: Arc) { } }); } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn manifest_object_names() { - let b = Bridge { - registry: walgit_wal::Registry::new( - Arc::new(walgit_store::memory::MemoryStore::new()), - Arc::new(walgit_config::Config::default()), - ), - store_prefix: "prefix/".into(), - sinks: Vec::new(), - serial: tokio::sync::Mutex::new(()), - }; - let id = b - .manifest_repo("prefix/repos/acme/monorepo/manifest.pb") - .unwrap(); - assert_eq!(id.to_string(), "acme/monorepo"); - // Every body shape names the same repository under a non-empty prefix. - for body in [ - serde_json::json!({"repo": "acme/monorepo"}), - serde_json::json!({"key": "prefix/repos/acme/monorepo/manifest.pb"}), - serde_json::json!({"message": {"attributes": { - "eventType": "OBJECT_FINALIZE", - "objectId": "prefix/repos/acme/monorepo/manifest.pb"}}}), - serde_json::json!({"Records": [{"eventName": "ObjectCreated:Put", - "s3": {"object": {"key": "prefix/repos/acme/monorepo/manifest.pb"}}}]}), - ] { - let ids: Vec<_> = notified_keys(&body, &b.store_prefix) - .iter() - .filter_map(|k| b.manifest_repo(k)) - .map(|id| id.to_string()) - .collect(); - assert_eq!(ids, ["acme/monorepo"], "{body}"); - } - for other in [ - "repos/t/r/manifest.pb", - "prefix/repos/t/r/wal/abc.pack", - "prefix/repos/t/r/events/cursor.json", - "walgit-go/repos/t/r/manifest.pb", - "prefix/repos/t/manifest.pb", - "prefix/repos/t/r/x/manifest.pb", - ] { - assert!(b.manifest_repo(other).is_none(), "{other}"); - } - } -} diff --git a/crates/walgit-server/src/events.rs b/crates/walgit-server/src/events.rs index 4d4d77bb..646abf92 100644 --- a/crates/walgit-server/src/events.rs +++ b/crates/walgit-server/src/events.rs @@ -1,160 +1,8 @@ -//! `ref` event shapes and the WAL → event conversion. Contract: -//! `docs/EVENTS.md`. The only producer is the bridge (`crate::bridge`): it -//! tails each repo's WAL from a durable cursor, converts committed PUSH / -//! `REF_UPDATE` entries with [`refs_from_entries`], and delivers to every -//! [`Sink`] before advancing the cursor. Nothing on the push path knows events -//! exist. -//! -//! Wire facts consumers rely on (the doc is normative, this is the code): -//! `old`/`new` are always full zero OIDs on create/delete, never empty; -//! dedup key = `(repo, _walgit.seq, ref_name)`; order by `_walgit.seq`. +//! The webhook sink of the events bridge (`crate::bridge`). The event model +//! (`RefEvent`, `refs_from_entries`, notification parsing) lives in `walgit-events` +//! so the `JetStream` publisher shares it. Contract: `docs/EVENTS.md`. -use serde::Serialize; -use walgit_git::RepoId; -use walgit_proto::v1::{EntryKind, LogEntry}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum Action { - Create, - Update, - Delete, -} - -impl Action { - pub fn as_str(self) -> &'static str { - match self { - Action::Create => "create", - Action::Update => "update", - Action::Delete => "delete", - } - } -} - -fn absent(oid: &str) -> bool { - oid.is_empty() || oid.bytes().all(|b| b == b'0') -} - -/// `create` / `update` / `delete` from the oid pair; `None` = no-op -/// (`0→0`, `X→X`) which must not emit. -pub fn classify(old: &str, new: &str) -> Option { - match (absent(old), absent(new)) { - (true, true) => None, - (true, false) => Some(Action::Create), - (false, true) => Some(Action::Delete), - (false, false) if old == new => None, - (false, false) => Some(Action::Update), - } -} - -/// The zero OID of the same length as `other` (sha1: 40, sha256: 64). The -/// WAL records "absent" as `""` (import-built txns) or the wire's zero OID -/// (pushes); the event always carries the zero OID. -fn zero_like(other: &str) -> String { - "0".repeat(other.len()) -} - -pub fn ref_type(name: &str) -> &'static str { - if name.starts_with("refs/heads/") { - "branch" - } else if name.starts_with("refs/tags/") { - "tag" - } else { - "" - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct WalgitExt { - pub schema_version: u32, - /// WAL seq of the committed entry, as a string (uint64 convention). - pub seq: String, - /// `push` | `ref_update`. - pub entry_kind: String, - #[serde(skip_serializing_if = "String::is_empty")] - pub request_id: String, -} - -/// Top level byte-compatible with the common `RefEvent` shape; walgit adds `repo` -/// and `_walgit`. -#[derive(Debug, Clone, Serialize)] -pub struct RefEvent { - pub action: Action, - pub ref_type: &'static str, - pub ref_name: String, - pub old: String, - pub new: String, - pub pusher: String, - pub correlation_id: String, - pub repo: String, - #[serde(rename = "_walgit")] - pub walgit: WalgitExt, -} - -impl RefEvent { - /// Message key (`/`): per-ref order where a transport keeps it. - pub fn key(&self) -> String { - format!("{}/{}", self.repo, self.ref_name) - } - /// Transport headers (`x_walgit_`): filtering without parsing bodies. - /// Always a strict subset of the body. - pub fn headers(&self) -> [(&'static str, &str); 7] { - [ - ("repo", &self.repo), - ("action", self.action.as_str()), - ("ref_type", self.ref_type), - ("ref_name", &self.ref_name), - ("pusher", &self.pusher), - ("seq", &self.walgit.seq), - ("correlation_id", &self.correlation_id), - ] - } -} - -/// `ref` events for the PUSH / `REF_UPDATE` entries in `entries`, in seq order. -pub(crate) fn refs_from_entries(repo: &RepoId, entries: &[LogEntry], out: &mut Vec) { - let repo = repo.to_string(); - for entry in entries { - let entry_kind = match EntryKind::try_from(entry.kind) { - Ok(EntryKind::Push) => "push", - Ok(EntryKind::RefUpdate) => "ref_update", - // COMPACT / CHECKPOINT never move refs. - _ => continue, - }; - let Some(txn) = &entry.txn else { continue }; - let pusher = entry.meta.get("principal").cloned().unwrap_or_default(); - let request_id = entry.meta.get("request_id").cloned().unwrap_or_default(); - for u in &txn.updates { - if !u.new_symbolic_target.is_empty() { - continue; // HEAD retarget: no oid transition to announce - } - let Some(action) = classify(&u.old_oid, &u.new_oid) else { - continue; - }; - let (old, new) = match action { - Action::Create => (zero_like(&u.new_oid), u.new_oid.clone()), - Action::Delete => (u.old_oid.clone(), zero_like(&u.old_oid)), - Action::Update => (u.old_oid.clone(), u.new_oid.clone()), - }; - out.push(RefEvent { - action, - ref_type: ref_type(&u.name), - ref_name: u.name.clone(), - old, - new, - pusher: pusher.clone(), - correlation_id: request_id.clone(), - repo: repo.clone(), - walgit: WalgitExt { - schema_version: 1, - seq: entry.seq.to_string(), - entry_kind: entry_kind.to_string(), - request_id: request_id.clone(), - }, - }); - } - } -} +use walgit_events::RefEvent; /// Where the bridge publishes: a webhook receives each batch as a JSON array, /// `POST`ed with `Content-Type: application/json`, a `X-Walgit-Delivery` id @@ -220,117 +68,3 @@ impl Sink for WebhookSink { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn classify_shapes() { - assert_eq!(classify("", ""), None); - assert_eq!(classify("0000", "0000"), None); - assert_eq!(classify("", "abc"), Some(Action::Create)); - assert_eq!(classify("abc", ""), Some(Action::Delete)); - assert_eq!(classify("abc", "abc"), None); - assert_eq!(classify("abc", "def"), Some(Action::Update)); - assert_eq!(ref_type("refs/heads/main"), "branch"); - assert_eq!(ref_type("refs/tags/v1"), "tag"); - assert_eq!(ref_type("refs/notes/commits"), ""); - } - - /// The golden body shape (docs/EVENTS.md). Field order and names are the - /// contract; changing this test is changing the wire. - #[test] - fn ref_event_golden() { - let ev = RefEvent { - action: Action::Update, - ref_type: "branch", - ref_name: "refs/heads/main".into(), - old: "48a0637".into(), - new: "cb38da1".into(), - pusher: "alice@example.com".into(), - correlation_id: "d1f916f7".into(), - repo: "acme/monorepo".into(), - walgit: WalgitExt { - schema_version: 1, - seq: "42".into(), - entry_kind: "push".into(), - request_id: "d1f916f7".into(), - }, - }; - let got = serde_json::to_string(&ev).unwrap(); - let want = concat!( - "{\"action\":\"update\",\"ref_type\":\"branch\",\"ref_name\":\"refs/heads/main\",", - "\"old\":\"48a0637\",\"new\":\"cb38da1\",\"pusher\":\"alice@example.com\",", - "\"correlation_id\":\"d1f916f7\",\"repo\":\"acme/monorepo\",", - "\"_walgit\":{\"schema_version\":1,\"seq\":\"42\",\"entry_kind\":\"push\",", - "\"request_id\":\"d1f916f7\"}}" - ); - assert_eq!(got, want); - assert_eq!(ev.key(), "acme/monorepo/refs/heads/main"); - } - - fn log_entry(kind: EntryKind, name: &str, old: &str, new: &str, sym: &str) -> LogEntry { - let mut meta = HashMap::new(); - meta.insert("principal".into(), "alice".into()); - meta.insert("request_id".into(), "cid".into()); - LogEntry { - seq: 9, - kind: kind as i32, - txn: Some(walgit_proto::v1::RefTransaction { - updates: vec![walgit_proto::v1::RefUpdate { - name: name.into(), - old_oid: old.into(), - new_oid: new.into(), - new_symbolic_target: sym.into(), - ..Default::default() - }], - ..Default::default() - }), - meta, - ..Default::default() - } - } - - const SHA: &str = "4a06219d6a5ba9d3e3d1a8e1e2b6f0c9d8e7f6a5"; - - #[test] - fn conversion_emits_ref_skips_compact_noop_symbolic() { - let repo = walgit_git::RepoId::new("t", "r").unwrap(); - let mut out = Vec::new(); - refs_from_entries( - &repo, - &[ - log_entry(EntryKind::Compact, "refs/heads/main", "", SHA, ""), - log_entry(EntryKind::Push, "refs/heads/main", SHA, SHA, ""), - log_entry(EntryKind::Push, "HEAD", "", "", "refs/heads/main"), - log_entry(EntryKind::Push, "refs/heads/main", "", SHA, ""), - log_entry(EntryKind::Push, "refs/heads/main", SHA, &"0".repeat(40), ""), - ], - &mut out, - ); - assert_eq!(out.len(), 2, "compact/noop/symbolic must not emit"); - let r = &out[0]; - assert_eq!(r.action, Action::Create); - assert_eq!(r.ref_name, "refs/heads/main"); - assert_eq!( - r.old, - "0".repeat(40), - "absent old is the zero OID, never empty" - ); - assert_eq!(r.new, SHA); - assert_eq!( - (r.pusher.as_str(), r.correlation_id.as_str()), - ("alice", "cid") - ); - assert_eq!((r.repo.as_str(), r.walgit.seq.as_str()), ("t/r", "9")); - assert_eq!(r.walgit.entry_kind, "push"); - let d = &out[1]; - assert_eq!(d.action, Action::Delete); - assert_eq!( - (d.old.as_str(), d.new.as_str()), - (SHA, "0".repeat(40).as_str()) - ); - } -} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 8f52b709..bf447475 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -115,6 +115,47 @@ webhook_secret = "…" # env: WALGIT__EVENTS__WEBHOOK_SECRET sweep_interval = "5m" ``` +## Delivery: NATS JetStream (`walgit-nats`) + +`walgit-nats` is a second sink for the same `ref` events, a separate binary +(`walgit-nats --config walgit.toml`, `[events.nats]`) that depends on `walgit-events` (the shared event +model) and never on `walgit-server`. The webhook bridge is unchanged and the two can run side by side. +Like the bridge it is a reader of the WAL, never a step of a push (D32). + +- **The stream is the cursor (D50).** For each repository the publisher reads the last message on the + repository's subject, takes its `Walgit-Seq` header and resumes from that seq, inclusive. Events already in + the stream are dropped, so a crash in the middle of a multi-ref entry completes only the rest of that entry. A repository with no message yet starts at `events.nats.start`: `oldest` (the oldest + readable seq, as the bridge does) or `head` (only pushes made after the publisher first saw it). +- **Subjects.** `.ref..`. Bytes outside `[A-Za-z0-9_-]` in a name are written + `%XX`, so `acme.io/mono.repo` is `walgit.ref.acme%2Eio.mono%2Erepo` and the mapping is reversible. + Subscribe to `walgit.ref.>` for everything, `walgit.ref.acme.>` for one owner. +- **Message.** The body is one `ref` event JSON object (not an array). Headers: `Walgit-Seq`, `Walgit-Repo` + and `Nats-Msg-Id = ::`, which makes JetStream drop a republished event inside the + stream's duplicate window. The publisher waits for every PubAck before the next message, so order within + a repository is `_walgit.seq` order. +- **Concurrent publishers.** The first publish of a catch-up sets `Nats-Expected-Last-Subject-Sequence`. + If another instance moved the subject first, this one stops, logs and lets the next wake-up retry from the + new last message. Delivery is at least once; consumers still dedup on `(repo, _walgit.seq, ref_name)`. +- **Wake-ups.** A queue-group subscription on `events.nats.notify_subject` accepts the same notification + shapes as `POST /_events/notify` (S3 event JSON, a GCS envelope, `{"key": ...}`, `{"repo": ...}`), and + `events.sweep_interval` is the backstop with the same warning when it finds unpublished entries. + Gap detection matches the bridge: a log gap is logged and the entries that are readable are published. +- **Service API.** The binary registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in) + with one endpoint, `.events.catch-up`. Request `{"repo": "o/r"}`, reply the catch-up report + (`repo`, `from_seq`, `head_seq`, `emitted`, `conflict`); an unknown repository is a 404 service error. + Gap, conflict and sweep counters are in the stats response. There is no HTTP listener. +- **Losing the stream.** Wiping the stream republishes the WAL window on the next wake-up; history older than + the oldest readable entry is not in the WAL and is backfilled with `walgit wal ls`. Wiping walgit loses nothing. + +```toml +[events] +sweep_interval = "5m" +[events.nats] +urls = ["nats://nats.internal:4222"] +credentials_file = "/etc/walgit/nats.creds" +start = "oldest" +``` + ## Consumer checklist 1. Verify `X-Walgit-Signature` (if you set a secret), then parse the array. diff --git a/walgit.example.toml b/walgit.example.toml index 79584aa9..54be482e 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -203,6 +203,17 @@ lock_wait_warn = "1s" # WARN "lock wait" (lock, repo, wait_ms, reque # webhook_secret = "" # X-Walgit-Signature: sha256=; unset = unsigned # sweep_interval = "5m" # backstop + health check (warns when a sweep finds unpublished entries); 0 = off +[events.nats] # docs/EVENTS.md: read only by the walgit-nats binary (`walgit-nats --config walgit.toml`) +# urls = ["nats://127.0.0.1:4222"] # the client connects to the first reachable server +# credentials_file = "/etc/walgit/nats.creds" # .creds or NKey seed file; unset = no credentials; never logged +# stream = "WALGIT_EVENTS" # the JetStream stream holding the events (subjects ".ref.>"); it is the resume cursor +# ensure_stream = true # create the stream when missing (file storage); an existing stream is used as is +# duplicate_window = "1h" # dedup window (Nats-Msg-Id) of a stream walgit-nats creates +# subject_prefix = "walgit" # events go to ".ref.."; metacharacters in names are %XX encoded +# notify_subject = "walgit.notify" # core subject bucket notifications are published to (S3 event JSON, {"key"} or {"repo"}) +# queue_group = "walgit-events" # notify subscription and service endpoints: instances in a group share the work +# start = "oldest" # first sight of a repo with no events in the stream: "oldest" (oldest readable seq) or "head" (only new pushes) + # Packfile foundations: these policies are validated and stored, but URI delivery and # ref filtering are not enabled by the foundation-only change. [refs] From 4a17b9f0f7b320ae6e95efddc491b307a66b8699 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 20:46:22 -0400 Subject: [PATCH 2/8] refactor(events): run the JetStream publisher as walgit nats Signed-off-by: Yordis Prieto --- AGENTS.md | 2 +- Cargo.lock | 5 +- Cargo.toml | 1 + crates/walgit-cli/Cargo.toml | 1 + crates/walgit-cli/src/bin/walgit.rs | 2 +- crates/walgit-cli/src/lib.rs | 3 + crates/walgit-config/src/lib.rs | 4 +- crates/walgit-nats/Cargo.toml | 8 -- crates/walgit-nats/src/lib.rs | 74 ++++++++++++++++- crates/walgit-nats/src/main.rs | 118 ---------------------------- docs/EVENTS.md | 10 +-- walgit.example.toml | 4 +- 12 files changed, 88 insertions(+), 144 deletions(-) delete mode 100644 crates/walgit-nats/src/main.rs diff --git a/AGENTS.md b/AGENTS.md index 18d68492..1c6fc398 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -487,7 +487,7 @@ full cold-read/resource acceptance gates listed in `docs/spec/README.md`. and candidate external-boundary proof remain separate obligations; local loose objects and retired download membership cannot justify retirement. See the cost and remaining-evidence rows in the linked docs. -- **D50 (2026-10-26): The JetStream sink's cursor lives in its own stream.** `walgit-nats` publishes the +- **D50 (2026-10-26): The JetStream sink's cursor lives in its own stream.** `walgit nats` publishes the `ref` events of D32 into NATS JetStream and resumes from the last message on each repository's subject (`Walgit-Seq` header, inclusive, then drop what the stream already holds). That cursor is the sink's state, not walgit's: Principle I holds because wiping every walgit instance loses nothing, and wiping the stream diff --git a/Cargo.lock b/Cargo.lock index b7a9c21f..4c0ab618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5928,6 +5928,7 @@ dependencies = [ "tracing-subscriber", "walgit-config", "walgit-git", + "walgit-nats", "walgit-proto", "walgit-server", "walgit-store", @@ -6004,17 +6005,13 @@ dependencies = [ "anyhow", "async-nats", "bytes", - "clap", "futures", - "mimalloc", - "rustls 0.23.45", "serde", "serde_json", "tempfile", "thiserror", "tokio", "tracing", - "tracing-subscriber", "walgit-config", "walgit-events", "walgit-git", diff --git a/Cargo.toml b/Cargo.toml index 9aca89e2..b3543f3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ walgit-git = { path = "crates/walgit-git" } walgit-wal = { path = "crates/walgit-wal" } walgit-events = { path = "crates/walgit-events" } walgit-server = { path = "crates/walgit-server" } +walgit-nats = { path = "crates/walgit-nats" } anyhow = "1" thiserror = "2" diff --git a/crates/walgit-cli/Cargo.toml b/crates/walgit-cli/Cargo.toml index f7231266..a65684d4 100644 --- a/crates/walgit-cli/Cargo.toml +++ b/crates/walgit-cli/Cargo.toml @@ -24,6 +24,7 @@ walgit-config.workspace = true walgit-git.workspace = true walgit-wal.workspace = true walgit-server.workspace = true +walgit-nats.workspace = true gix-hash.workspace = true anyhow.workspace = true tokio.workspace = true diff --git a/crates/walgit-cli/src/bin/walgit.rs b/crates/walgit-cli/src/bin/walgit.rs index e4acf99b..08e900ff 100644 --- a/crates/walgit-cli/src/bin/walgit.rs +++ b/crates/walgit-cli/src/bin/walgit.rs @@ -1,4 +1,4 @@ -//! `walgit` — the full CLI (serve | compact | repo | wal | synth | import | mirror | config). +//! `walgit` — the full CLI (serve | compact | repo | wal | nats | synth | import | mirror | config). fn main() -> anyhow::Result<()> { walgit_cli::main() } diff --git a/crates/walgit-cli/src/lib.rs b/crates/walgit-cli/src/lib.rs index c7b49b35..d6f55c24 100644 --- a/crates/walgit-cli/src/lib.rs +++ b/crates/walgit-cli/src/lib.rs @@ -109,6 +109,8 @@ enum Command { #[command(subcommand)] action: RepoAction, }, + /// Publish the WAL's ref events into NATS `JetStream` (`[events.nats]`, `docs/EVENTS.md`). + Nats, /// WAL inspection and rewind. Wal { #[command(subcommand)] @@ -471,6 +473,7 @@ async fn dispatch(command: Command, cfg: Config) -> Result<()> { } => compact::run(repo, all, once, base, &cfg).await, Command::Repo { action } => repo::run(action, &cfg).await, Command::Wal { action } => wal_cmd::run(action, &cfg).await, + Command::Nats => walgit_nats::run(cfg.clone()).await, Command::Mirror { from, to, diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index c120026d..b1f609e6 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -637,7 +637,7 @@ pub struct EventsConfig { /// warns when a sweep finds unpublished entries. `0` = off. #[serde(with = "humantime_serde")] pub sweep_interval: Duration, - /// The `JetStream` publisher (`walgit-nats`, `docs/EVENTS.md`). Read only by that binary. + /// The `JetStream` publisher (`walgit nats`, `docs/EVENTS.md`). Read only by that subcommand. pub nats: NatsEventsConfig, } @@ -663,7 +663,7 @@ pub enum NatsStart { Head, } -/// `[events.nats]`: the `walgit-nats` binary publishes the bridge's `ref` events into `JetStream`. +/// `[events.nats]`: `walgit nats` publishes the bridge's `ref` events into `JetStream`. /// The resume cursor is the last message on each repository's subject, so there is no cursor /// object in the bucket. #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/walgit-nats/Cargo.toml b/crates/walgit-nats/Cargo.toml index 08d035f1..5c6347c0 100644 --- a/crates/walgit-nats/Cargo.toml +++ b/crates/walgit-nats/Cargo.toml @@ -4,10 +4,6 @@ version.workspace = true edition.workspace = true license.workspace = true -[[bin]] -name = "walgit-nats" -path = "src/main.rs" - [dependencies] walgit-events.workspace = true walgit-config.workspace = true @@ -21,12 +17,8 @@ bytes.workspace = true futures.workspace = true tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true serde.workspace = true serde_json.workspace = true -clap.workspace = true -mimalloc.workspace = true -rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12", "logging"] } [dev-dependencies] walgit-proto.workspace = true diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs index b3631dae..848a2600 100644 --- a/crates/walgit-nats/src/lib.rs +++ b/crates/walgit-nats/src/lib.rs @@ -1,4 +1,4 @@ -//! `walgit-nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with +//! `walgit nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with //! the cursor in the stream, wake-ups over NATS and a `$SRV` service API instead of HTTP. //! The event model is shared with the webhook bridge through `walgit-events`. @@ -9,11 +9,13 @@ pub mod subject; pub mod wake; use std::str::FromStr; +use std::sync::Arc; use anyhow::Context; use async_nats::jetstream::{self, stream}; -use walgit_config::NatsEventsConfig; +use walgit_config::{Config, NatsEventsConfig}; +use crate::publisher::Publisher; use crate::subject::SubjectPrefix; /// Connect as `[events.nats]` says. The credentials path is read, never logged. @@ -23,7 +25,7 @@ pub async fn connect(cfg: &NatsEventsConfig) -> anyhow::Result>>()?; - let mut options = async_nats::ConnectOptions::new().name("walgit-nats"); + let mut options = async_nats::ConnectOptions::new().name("walgit"); if let Some(path) = &cfg.credentials_file { options = options .credentials_file(path) @@ -56,3 +58,69 @@ pub async fn open_stream( .with_context(|| format!("stream {} (events.nats.ensure_stream is off)", cfg.stream)) } } + +async fn shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + match signal(SignalKind::terminate()) { + Ok(mut term) => { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } + return; + } + Err(e) => tracing::warn!(error = %e, "no SIGTERM handler"), + } + } + let _ = tokio::signal::ctrl_c().await; +} + +/// `walgit nats`: publish until SIGTERM/ctrl-c, or fail when a worker stops. +pub async fn run(cfg: Arc) -> anyhow::Result<()> { + let nats = &cfg.events.nats; + let prefix = SubjectPrefix::new(&nats.subject_prefix)?; + let store = walgit_store::open_store(&cfg).await?; + let registry = walgit_wal::Registry::new(store, cfg.clone()); + + let client = crate::connect(nats).await?; + let js = async_nats::jetstream::new(client.clone()); + let stream = crate::open_stream(&js, nats, &prefix).await?; + let publisher = Publisher::new(registry, js, stream, prefix.clone(), nats.start); + tracing::info!(stream = %nats.stream, prefix = %prefix, start = ?nats.start, "walgit-nats ready"); + + let mut tasks = tokio::task::JoinSet::new(); + tasks.spawn(crate::wake::notify_loop( + client.clone(), + nats.notify_subject.clone(), + nats.queue_group.clone(), + cfg.store_prefix(), + publisher.clone(), + )); + tasks.spawn({ + let (client, queue_group, publisher) = + (client.clone(), nats.queue_group.clone(), publisher.clone()); + async move { crate::service::serve(client, &prefix, &queue_group, publisher).await } + }); + tasks.spawn({ + let publisher = publisher.clone(); + let every = cfg.events.sweep_interval; + async move { + crate::wake::sweep_loop(publisher, every).await; + std::future::pending::>().await + } + }); + + tokio::select! { + () = shutdown_signal() => tracing::info!("shutting down"), + Some(done) = tasks.join_next() => { + // A loop ending is a failure: the subscription or the service closed under us. + done.context("worker panicked")?.context("worker stopped")?; + anyhow::bail!("a worker stopped unexpectedly"); + } + } + tasks.shutdown().await; + client.flush().await.ok(); + Ok(()) +} diff --git a/crates/walgit-nats/src/main.rs b/crates/walgit-nats/src/main.rs deleted file mode 100644 index f9178726..00000000 --- a/crates/walgit-nats/src/main.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! `walgit-nats --config walgit.toml`: publish the WAL's ref events into `JetStream` (`docs/EVENTS.md`). - -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::Context; -use clap::Parser; -use tracing_subscriber::EnvFilter; -use walgit_config::{Config, LogFormat}; -use walgit_nats::publisher::Publisher; -use walgit_nats::subject::SubjectPrefix; - -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -#[derive(Parser)] -#[command(name = "walgit-nats", version, about = "walgit ref events into NATS JetStream")] -struct Cli { - /// Path to the configuration file. - #[arg(long, env = "WALGIT_CONFIG", default_value = "walgit.toml")] - config: PathBuf, -} - -fn init_tracing(cfg: &Config) { - let filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(&cfg.telemetry.log_filter)); - let builder = tracing_subscriber::fmt().with_env_filter(filter); - match cfg.telemetry.log_format { - LogFormat::Json => builder.json().init(), - LogFormat::Pretty => builder.pretty().init(), - } -} - -async fn shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - match signal(SignalKind::terminate()) { - Ok(mut term) => { - tokio::select! { - _ = tokio::signal::ctrl_c() => {} - _ = term.recv() => {} - } - return; - } - Err(e) => tracing::warn!(error = %e, "no SIGTERM handler"), - } - } - let _ = tokio::signal::ctrl_c().await; -} - -async fn run(cfg: Arc) -> anyhow::Result<()> { - let nats = &cfg.events.nats; - let prefix = SubjectPrefix::new(&nats.subject_prefix)?; - let store = walgit_store::open_store(&cfg).await?; - let registry = walgit_wal::Registry::new(store, cfg.clone()); - - let client = walgit_nats::connect(nats).await?; - let js = async_nats::jetstream::new(client.clone()); - let stream = walgit_nats::open_stream(&js, nats, &prefix).await?; - let publisher = Publisher::new(registry, js, stream, prefix.clone(), nats.start); - tracing::info!(stream = %nats.stream, prefix = %prefix, start = ?nats.start, "walgit-nats ready"); - - let mut tasks = tokio::task::JoinSet::new(); - tasks.spawn(walgit_nats::wake::notify_loop( - client.clone(), - nats.notify_subject.clone(), - nats.queue_group.clone(), - cfg.store_prefix(), - publisher.clone(), - )); - tasks.spawn({ - let (client, queue_group, publisher) = - (client.clone(), nats.queue_group.clone(), publisher.clone()); - async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } - }); - tasks.spawn({ - let publisher = publisher.clone(); - let every = cfg.events.sweep_interval; - async move { - walgit_nats::wake::sweep_loop(publisher, every).await; - std::future::pending::>().await - } - }); - - tokio::select! { - () = shutdown_signal() => tracing::info!("shutting down"), - Some(done) = tasks.join_next() => { - // A loop ending is a failure: the subscription or the service closed under us. - done.context("worker panicked")?.context("worker stopped")?; - anyhow::bail!("a worker stopped unexpectedly"); - } - } - tasks.shutdown().await; - client.flush().await.ok(); - Ok(()) -} - -fn main() -> anyhow::Result<()> { - rustls::crypto::aws_lc_rs::default_provider() - .install_default() - .map_err(|_| anyhow::anyhow!("a rustls crypto provider is already installed"))?; - let cli = Cli::parse(); - let cfg = if cli.config.exists() { - Config::load(&cli.config)? - } else { - eprintln!( - "walgit-nats: config file {} not found (pass --config PATH or WALGIT_CONFIG; `--config /dev/null` for defaults + WALGIT__ env)", - cli.config.display() - ); - std::process::exit(2); - }; - init_tracing(&cfg); - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()? - .block_on(run(Arc::new(cfg))) -} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index bf447475..8972ede6 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -115,11 +115,11 @@ webhook_secret = "…" # env: WALGIT__EVENTS__WEBHOOK_SECRET sweep_interval = "5m" ``` -## Delivery: NATS JetStream (`walgit-nats`) +## Delivery: NATS JetStream (`walgit nats`) -`walgit-nats` is a second sink for the same `ref` events, a separate binary -(`walgit-nats --config walgit.toml`, `[events.nats]`) that depends on `walgit-events` (the shared event -model) and never on `walgit-server`. The webhook bridge is unchanged and the two can run side by side. +`walgit nats` is a second sink for the same `ref` events, a subcommand of the normal CLI +(`walgit nats --config walgit.toml`, `[events.nats]`) built on the `walgit-nats` crate, which depends on +`walgit-events` (the shared event model) and never on `walgit-server`. The webhook bridge is unchanged and the two can run side by side. Like the bridge it is a reader of the WAL, never a step of a push (D32). - **The stream is the cursor (D50).** For each repository the publisher reads the last message on the @@ -140,7 +140,7 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). shapes as `POST /_events/notify` (S3 event JSON, a GCS envelope, `{"key": ...}`, `{"repo": ...}`), and `events.sweep_interval` is the backstop with the same warning when it finds unpublished entries. Gap detection matches the bridge: a log gap is logged and the entries that are readable are published. -- **Service API.** The binary registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in) +- **Service API.** `walgit nats` registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in) with one endpoint, `.events.catch-up`. Request `{"repo": "o/r"}`, reply the catch-up report (`repo`, `from_seq`, `head_seq`, `emitted`, `conflict`); an unknown repository is a 404 service error. Gap, conflict and sweep counters are in the stats response. There is no HTTP listener. diff --git a/walgit.example.toml b/walgit.example.toml index 54be482e..471b8633 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -203,12 +203,12 @@ lock_wait_warn = "1s" # WARN "lock wait" (lock, repo, wait_ms, reque # webhook_secret = "" # X-Walgit-Signature: sha256=; unset = unsigned # sweep_interval = "5m" # backstop + health check (warns when a sweep finds unpublished entries); 0 = off -[events.nats] # docs/EVENTS.md: read only by the walgit-nats binary (`walgit-nats --config walgit.toml`) +[events.nats] # docs/EVENTS.md: read only by `walgit nats --config walgit.toml` # urls = ["nats://127.0.0.1:4222"] # the client connects to the first reachable server # credentials_file = "/etc/walgit/nats.creds" # .creds or NKey seed file; unset = no credentials; never logged # stream = "WALGIT_EVENTS" # the JetStream stream holding the events (subjects ".ref.>"); it is the resume cursor # ensure_stream = true # create the stream when missing (file storage); an existing stream is used as is -# duplicate_window = "1h" # dedup window (Nats-Msg-Id) of a stream walgit-nats creates +# duplicate_window = "1h" # dedup window (Nats-Msg-Id) of a stream `walgit nats` creates # subject_prefix = "walgit" # events go to ".ref.."; metacharacters in names are %XX encoded # notify_subject = "walgit.notify" # core subject bucket notifications are published to (S3 event JSON, {"key"} or {"repo"}) # queue_group = "walgit-events" # notify subscription and service endpoints: instances in a group share the work From b6c86c19db6bbd682b8d072e34791d75bdacc0f2 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 21:08:38 -0400 Subject: [PATCH 3/8] test(events): prove the JetStream publisher's delivery guarantees Signed-off-by: Yordis Prieto --- Cargo.lock | 3 + crates/walgit-nats/Cargo.toml | 7 +- crates/walgit-nats/src/lib.rs | 2 + crates/walgit-nats/src/publisher.rs | 39 +- crates/walgit-nats/src/testing.rs | 93 ++++ crates/walgit-nats/tests/jetstream.rs | 617 ++++++++++++++++++++-- crates/walgit-server/Cargo.toml | 2 + crates/walgit-server/tests/events_nats.rs | 479 +++++++++++++++++ docs/EVENTS.md | 37 ++ tests/e2e.sh | 181 +++++++ 10 files changed, 1397 insertions(+), 63 deletions(-) create mode 100644 crates/walgit-nats/src/testing.rs create mode 100644 crates/walgit-server/tests/events_nats.rs diff --git a/Cargo.lock b/Cargo.lock index 4c0ab618..6da16f94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6015,6 +6015,7 @@ dependencies = [ "walgit-config", "walgit-events", "walgit-git", + "walgit-nats", "walgit-proto", "walgit-store", "walgit-wal", @@ -6039,6 +6040,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "async-nats", "async-trait", "axum", "base64", @@ -6093,6 +6095,7 @@ dependencies = [ "walgit-config", "walgit-events", "walgit-git", + "walgit-nats", "walgit-proto", "walgit-store", "walgit-wal", diff --git a/crates/walgit-nats/Cargo.toml b/crates/walgit-nats/Cargo.toml index 5c6347c0..19e24170 100644 --- a/crates/walgit-nats/Cargo.toml +++ b/crates/walgit-nats/Cargo.toml @@ -19,10 +19,15 @@ tokio.workspace = true tracing.workspace = true serde.workspace = true serde_json.workspace = true +tempfile = { workspace = true, optional = true } + +[features] +# `testing::start_nats`: a throwaway `nats-server -js` for integration tests. +testing = ["dep:tempfile"] [dev-dependencies] +walgit-nats = { path = ".", features = ["testing"] } walgit-proto.workspace = true -tempfile.workspace = true [lints] workspace = true diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs index 848a2600..e7286620 100644 --- a/crates/walgit-nats/src/lib.rs +++ b/crates/walgit-nats/src/lib.rs @@ -6,6 +6,8 @@ pub mod message; pub mod publisher; pub mod service; pub mod subject; +#[cfg(feature = "testing")] +pub mod testing; pub mod wake; use std::str::FromStr; diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs index 96428059..9dcc1781 100644 --- a/crates/walgit-nats/src/publisher.rs +++ b/crates/walgit-nats/src/publisher.rs @@ -26,7 +26,7 @@ use walgit_config::NatsStart; use walgit_events::refs_from_entries; use walgit_git::RepoId; -use crate::message::{EventMessage, LastPublished, after_last_published}; +use crate::message::{EventMessage, LastPublished, MsgId, after_last_published}; use crate::subject::{RepoSubject, SubjectPrefix}; /// What one `catch_up` did. @@ -76,9 +76,10 @@ pub struct Publisher { prefix: SubjectPrefix, start: NatsStart, serial: tokio::sync::Mutex<()>, - /// Head each repository was fully published up to by this process: spares the log read when a - /// wake-up finds nothing new. Never a source of truth (the stream is). - done: std::sync::Mutex>, + /// Where each repository was fully published up to by this process: the head and the stream's + /// last message for its subject. A wake-up that finds both unchanged spares the log read. The + /// stream half is what notices a purged or recreated stream. Never a source of truth. + done: std::sync::Mutex)>>, /// `start = "head"`: the head a cold repository was first seen at. cold_marks: std::sync::Mutex>, pub counters: Counters, @@ -155,12 +156,16 @@ impl Publisher { head_seq: head, ..CatchUp::default() }; - if locked(&self.done).get(id).is_some_and(|d| *d >= head) { - return Ok(report); - } + // One cheap request on every wake-up: the stream, not this process, says what is published. let subject = RepoSubject::new(&self.prefix, id); let last = self.last_published(&subject).await?; + if locked(&self.done) + .get(id) + .is_some_and(|(done_head, done_last)| *done_head >= head && *done_last == last) + { + return Ok(report); + } // First seq to read; `resume` is the entry the stream may hold only part of. let (first, resume) = match &last { @@ -189,6 +194,7 @@ impl Publisher { }; report.from_seq = resume.map_or(first.saturating_sub(1), |l| l.id.seq()); + let mut last_sent: Option = None; if first <= head { let entries = handle.read_log(&manifest, first, Some(head)).await?; let mut events = Vec::new(); @@ -196,8 +202,10 @@ impl Publisher { if let Some(l) = resume { events = after_last_published(events, l); } - let mut expected = resume.map_or(0, |l| l.stream_seq); + // Whatever is on the subject now (a gap leaves older messages there too). + let mut expected = last.as_ref().map_or(0, |l| l.stream_seq); for (i, event) in events.iter().enumerate() { + last_sent = MsgId::of(event); let mut msg = EventMessage::new(subject.clone(), event)?; if i == 0 { msg = msg.expecting_last_subject_sequence(expected); @@ -234,7 +242,20 @@ impl Publisher { } } - locked(&self.done).insert(id.clone(), head); + // Done only when the stream confirms its last message is the one just sent: a broker + // that swallowed it as a duplicate of a purged message (the dedup table outlives a purge) + // leaves the repository open, and the retry lands once the dedup window has passed. + let settled = if report.emitted > 0 { + self.last_published(&subject).await? + } else { + last + }; + let confirmed = last_sent + .as_ref() + .is_none_or(|sent| settled.as_ref().is_some_and(|s| s.id == *sent)); + if confirmed { + locked(&self.done).insert(id.clone(), (head, settled)); + } if report.emitted > 0 { tracing::info!(repo = %id, from = report.from_seq, head, emitted = report.emitted, "published"); } diff --git a/crates/walgit-nats/src/testing.rs b/crates/walgit-nats/src/testing.rs new file mode 100644 index 00000000..7d66fe56 --- /dev/null +++ b/crates/walgit-nats/src/testing.rs @@ -0,0 +1,93 @@ +//! A throwaway `nats-server -js` for tests (feature `testing`). Shared by this crate's tests and +//! `walgit-server`'s, so neither copies the launcher. + +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub struct NatsServer { + child: Child, + port: u16, + pub url: String, + store: tempfile::TempDir, +} + +impl NatsServer { + fn spawn(port: u16, store: &std::path::Path) -> Child { + #[allow(clippy::expect_used)] + Command::new("nats-server") + .args(["-js", "-a", "127.0.0.1", "-p", &port.to_string(), "-sd"]) + .arg(store) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn nats-server") + } + + /// Kill the server at once (no drain), leaving its store dir for [`NatsServer::restart`]. + pub fn stop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } + + /// Start the server again on the same port and store dir: streams survive, clients reconnect. + /// + /// # Panics + /// When the server does not accept connections within 15 s. + pub async fn restart(&mut self) { + self.stop(); + self.child = Self::spawn(self.port, self.store.path()); + wait_ready(&self.url).await; + } +} + +async fn wait_ready(url: &str) { + let t0 = Instant::now(); + while async_nats::connect(url).await.is_err() { + assert!( + t0.elapsed() < Duration::from_secs(15), + "nats-server did not start" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +impl Drop for NatsServer { + fn drop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } +} + +/// Start `nats-server -js` on a free loopback port with a temp store dir. `None` (after saying so +/// on stderr) when the binary is not on PATH, so tests skip instead of failing. +/// +/// # Panics +/// When the server cannot be spawned or does not accept connections within 15 s. +#[allow(clippy::expect_used)] +pub async fn start_nats() -> Option { + if !Command::new("nats-server") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + { + eprintln!("skipping: nats-server is not available on PATH"); + return None; + } + let port = std::net::TcpListener::bind("127.0.0.1:0") + .expect("free port") + .local_addr() + .expect("local addr") + .port(); + let store = tempfile::tempdir().expect("tempdir"); + let child = NatsServer::spawn(port, store.path()); + let server = NatsServer { + child, + port, + url: format!("nats://127.0.0.1:{port}"), + store, + }; + wait_ready(&server.url).await; + Some(server) +} diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs index fdc1e73b..d618bc60 100644 --- a/crates/walgit-nats/tests/jetstream.rs +++ b/crates/walgit-nats/tests/jetstream.rs @@ -6,7 +6,6 @@ //! wake-ups and the service API. Skips (passes) when `nats-server` is not on PATH. use std::collections::HashMap; -use std::process::{Child, Command, Stdio}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -18,64 +17,15 @@ use walgit_git::{ObjectFormat, RepoId}; use walgit_nats::message::EventMessage; use walgit_nats::publisher::Publisher; use walgit_nats::subject::{RepoSubject, SubjectPrefix}; +use walgit_nats::testing::{NatsServer, start_nats}; use walgit_proto::v1::{RefTransaction, RefUpdate}; +use async_nats::jetstream::message::PublishMessage; const ZERO: &str = "0000000000000000000000000000000000000000"; const A: &str = "1111111111111111111111111111111111111111"; const B: &str = "2222222222222222222222222222222222222222"; const C: &str = "3333333333333333333333333333333333333333"; -struct NatsServer { - child: Child, - url: String, - _store: tempfile::TempDir, -} - -impl Drop for NatsServer { - fn drop(&mut self) { - self.child.kill().ok(); - self.child.wait().ok(); - } -} - -async fn start_nats() -> Option { - if !Command::new("nats-server") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .is_ok_and(|s| s.success()) - { - eprintln!("skipping: nats-server is not available on PATH"); - return None; - } - let port = std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); - let store = tempfile::tempdir().unwrap(); - let child = Command::new("nats-server") - .args(["-js", "-a", "127.0.0.1", "-p", &port.to_string(), "-sd"]) - .arg(store.path()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - let url = format!("nats://127.0.0.1:{port}"); - let server = NatsServer { - child, - url, - _store: store, - }; - let t0 = Instant::now(); - while async_nats::connect(&server.url).await.is_err() { - assert!(t0.elapsed() < Duration::from_secs(15), "nats-server did not start"); - tokio::time::sleep(Duration::from_millis(100)).await; - } - Some(server) -} - struct Rig { cfg: Arc, registry: Arc, @@ -87,19 +37,29 @@ struct Rig { impl Rig { async fn new(nats: &NatsServer, start: NatsStart) -> Self { + Self::with_window(nats, start, None).await + } + + /// `window`: the stream's `duplicate_window` (default: the config's one hour). + async fn with_window(nats: &NatsServer, start: NatsStart, window: Option) -> Self { let cache = tempfile::tempdir().unwrap(); let mut cfg = Config::default(); cfg.store.backend = walgit_config::StoreBackend::Memory; cfg.cache.dir = cache.path().to_path_buf(); cfg.events.nats.urls = vec![nats.url.clone()]; cfg.events.nats.start = start; + if let Some(window) = window { + cfg.events.nats.duplicate_window = window; + } let cfg = Arc::new(cfg); let registry = walgit_wal::Registry::new( Arc::new(walgit_store::memory::MemoryStore::new()), cfg.clone(), ); let client = walgit_nats::connect(&cfg.events.nats).await.unwrap(); - let js = jetstream::new(client.clone()); + let js = jetstream::ContextBuilder::new() + .timeout(Duration::from_secs(3)) + .build(client.clone()); let prefix = SubjectPrefix::new(&cfg.events.nats.subject_prefix).unwrap(); Rig { cfg, @@ -148,6 +108,91 @@ impl Rig { r.seq } + /// The events (one per ref) that WAL entry `seq` yields. + async fn events_of(&self, id: &RepoId, seq: u64) -> Vec { + let handle = self.registry.open(id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let manifest = handle.manifest(); + let entries = handle.read_log(&manifest, seq, Some(seq)).await.unwrap(); + let mut events = Vec::new(); + refs_from_entries(id, &entries, &mut events); + events + } + + /// Publish a message on the repo's subject as walgit would, but for a ref the entry does not + /// contain: the stream's last message for the subject then names something `catch_up` cannot + /// find in the entry, which forces it to republish the entry. Returns its stream sequence. + async fn inject(&self, id: &RepoId, seq: u64, ref_name: &str) -> u64 { + let msg = PublishMessage::build() + .payload("{}".into()) + .message_id(format!("{id}:{seq}:{ref_name}")) + .header("Walgit-Seq", seq.to_string().as_str()) + .header("Walgit-Repo", id.to_string().as_str()); + let subject = RepoSubject::new(&self.prefix, id).to_string(); + self.js + .send_publish(subject, msg) + .await + .unwrap() + .await + .unwrap() + .sequence + } + + /// `Nats-Msg-Id` of every message in the stream, in stream order. + async fn ids(&self) -> Vec { + let stream = self.stream().await; + let state = stream.get_info().await.unwrap().state; + let mut out = Vec::new(); + if state.messages == 0 { + return out; + } + for seq in state.first_sequence..=state.last_sequence { + if let Ok(raw) = stream.get_raw_message(seq).await { + out.push( + raw.headers + .get("Nats-Msg-Id") + .map(|v| v.as_str().to_string()) + .unwrap_or_default(), + ); + } + } + out + } + + async fn wait_connected(&self) { + let t0 = Instant::now(); + while self.client.connection_state() != async_nats::connection::State::Connected { + assert!(t0.elapsed() < Duration::from_secs(20), "client did not reconnect"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + fn spawn_notify(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + let nats_cfg = &self.cfg.events.nats; + tokio::spawn(walgit_nats::wake::notify_loop( + self.client.clone(), + nats_cfg.notify_subject.clone(), + nats_cfg.queue_group.clone(), + self.cfg.store_prefix(), + publisher.clone(), + )) + } + + fn spawn_service(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + let (client, prefix, publisher) = (self.client.clone(), self.prefix.clone(), publisher.clone()); + let queue_group = self.cfg.events.nats.queue_group.clone(); + tokio::spawn(async move { + walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await + }) + } + + async fn notify(&self, body: &str) { + self.client + .publish(self.cfg.events.nats.notify_subject.clone(), body.to_string().into()) + .await + .unwrap(); + } + async fn stream(&self) -> jetstream::stream::Stream { self.js .get_stream(&self.cfg.events.nats.stream) @@ -396,3 +441,469 @@ async fn notifications_and_the_service_api_wake_the_publisher() { notify.abort(); service.abort(); } + +// --------------------------------------------------------------------------------------------- +// Dedup, races, wipes, gaps, outages and the wake-ups that must be ignored. +// --------------------------------------------------------------------------------------------- + +async fn repo_with_three_events(rig: &Rig, owner: &str, name: &str) -> RepoId { + let id = RepoId::new(owner, name).unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + rig.push(&id, &[("refs/heads/a", ZERO, B), ("refs/heads/b", ZERO, B)]).await; + id +} + +fn msg_id(id: &RepoId, seq: u64, name: &str) -> String { + format!("{id}:{seq}:{name}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn broker_dedup_absorbs_a_republish_inside_the_window() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let first = rig.publisher(NatsStart::Oldest).await; + assert_eq!(first.catch_up(&id).await.unwrap().emitted, 3); + + // A hand-built republish of an existing Nats-Msg-Id is acknowledged as a duplicate. + let event = rig.events_of(&id, 1).await.remove(0); + let msg = EventMessage::new(RepoSubject::new(&rig.prefix, &id), &event).unwrap(); + let subject = msg.subject.to_string(); + let ack = rig.js.send_publish(subject, msg.into_publish()).await.unwrap().await.unwrap(); + assert!(ack.duplicate, "same Nats-Msg-Id inside the window must be a duplicate"); + assert_eq!(rig.message_count().await, 3); + + // The real path: the last message on the subject names a ref the entry does not contain, so + // `after_last_published` cannot find where the stream got to and republishes the whole entry. + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + assert_eq!(rig.message_count().await, 4); + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!((r.from_seq, r.emitted, r.conflict), (2, 2, false), "{r:?}"); + let counters = restarted.counters.snapshot(); + assert_eq!(counters["deduplicated"], 2, "both republished events were broker duplicates"); + assert_eq!(counters["published"], 0); + assert_eq!(rig.message_count().await, 4, "the stream did not grow"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn racing_publishers_leave_each_event_exactly_once() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let mut expected = Vec::new(); + let mut ids = Vec::new(); + for round in 0..6 { + for n in 0..3 { + let id = RepoId::new("race", format!("r{round}x{n}").as_str()).unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + for seq in 1..=4u64 { + let a = format!("refs/heads/a{seq}"); + let b = format!("refs/heads/b{seq}"); + rig.push(&id, &[(a.as_str(), ZERO, A), (b.as_str(), ZERO, B)]).await; + expected.push(msg_id(&id, seq, &a)); + expected.push(msg_id(&id, seq, &b)); + } + ids.push(id); + } + } + let one = rig.publisher(NatsStart::Oldest).await; + let two = rig.publisher(NatsStart::Oldest).await; + let mut reports = Vec::new(); + for batch in ids.chunks(3) { + let (a, b) = (batch[0].clone(), batch[1].clone()); + let c = batch[2].clone(); + let runs = tokio::join!( + one.catch_up(&a), + two.catch_up(&a), + one.catch_up(&b), + two.catch_up(&b), + two.catch_up(&c), + one.catch_up(&c), + ); + for r in [runs.0, runs.1, runs.2, runs.3, runs.4, runs.5] { + reports.push(r.expect("a lost race is a conflict or a dedup, never an error")); + } + } + let mut seen = rig.ids().await; + let total = seen.len(); + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), total, "a (repo, seq, ref) landed twice"); + expected.sort(); + assert_eq!(seen, expected, "every event exactly once"); + + let (one, two) = (one.counters.snapshot(), two.counters.snapshot()); + let sum = |key: &str| one[key].as_u64().unwrap() + two[key].as_u64().unwrap(); + assert_eq!(sum("published"), total as u64, "each landed message was counted once"); + let conflicts = reports.iter().filter(|r| r.conflict).count() as u64; + assert_eq!(sum("conflicts"), conflicts); + assert!(conflicts + sum("deduplicated") > 0, "the publishers never actually raced"); + eprintln!( + "race: {conflicts} conflicts, {} deduplicated over {} catch-ups", + sum("deduplicated"), + reports.len() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn after_the_dedup_window_a_republish_lands_twice() { + // At-least-once, documented: JetStream only dedups inside `duplicate_window`, so a consumer + // must dedup on (repo, _walgit.seq, ref_name) itself. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_millis(200))).await; + let id = repo_with_three_events(&rig, "t", "r").await; + assert_eq!(rig.publisher(NatsStart::Oldest).await.catch_up(&id).await.unwrap().emitted, 3); + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + tokio::time::sleep(Duration::from_secs(2)).await; + + let restarted = rig.publisher(NatsStart::Oldest).await; + let r = restarted.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 2); + let counters = restarted.counters.snapshot(); + assert_eq!((counters["published"].as_u64(), counters["deduplicated"].as_u64()), (Some(2), Some(0))); + assert_eq!(rig.message_count().await, 6, "both republished events landed a second time"); + let ids = rig.ids().await; + for name in ["refs/heads/a", "refs/heads/b"] { + let id = msg_id(&id, 2, name); + assert_eq!(ids.iter().filter(|i| **i == id).count(), 2, "{id} must be in the stream twice"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_wiped_stream_is_republished_on_the_next_catch_up() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + let original = rig.ids().await; + + // Purged: JetStream keeps its duplicate table across a purge, so inside `duplicate_window` + // the republish is acknowledged as duplicates and lands nothing. PINNED known limitation; the + // repository stays open (not recorded as done) so it heals once the window has passed, see + // `a_purged_stream_heals_once_the_dedup_window_has_passed`. + rig.stream().await.purge().await.unwrap(); + assert_eq!(rig.message_count().await, 0); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "{r:?}"); + assert_eq!(publisher.counters.snapshot()["deduplicated"], 3, "swallowed as duplicates of purged messages"); + assert_eq!(rig.message_count().await, 0, "PINNED: a purge inside the window loses the republish"); + + // Deleted: nothing to publish into (an error, not a silent skip) ... + rig.js.delete_stream(&rig.cfg.events.nats.stream).await.unwrap(); + assert!(publisher.catch_up(&id).await.is_err()); + // ... and recreated, the window is republished again. + walgit_nats::open_stream(&rig.js, &rig.cfg.events.nats, &rig.prefix).await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "a recreated stream is republished: {r:?}"); + assert_eq!(rig.ids().await, original); + + // A steady state still spares the log read and publishes nothing. + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); + assert_eq!(rig.message_count().await, 3); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_purged_stream_heals_once_the_dedup_window_has_passed() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_millis(200))).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + let original = rig.ids().await; + + rig.stream().await.purge().await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 3, "{r:?}"); + assert_eq!(rig.ids().await, original, "the same running publisher republished the window"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0, "and then it is steady"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn entries_folded_before_publishing_are_a_counted_gap() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + rig.push(&id, &[("refs/heads/main", C, A)]).await; + let cp = rig.registry.open(&id).await.unwrap().write_checkpoint().await.unwrap(); + assert_eq!(cp.seq, 4, "seqs 2..=4 are folded away before they were published"); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.gap, Some([2, 4]), "{r:?}"); + assert_eq!((r.head_seq, r.emitted), (5, 1), "the readable entry is still published"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 3); + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 5, "refs/heads/main")]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_outage_mid_catch_up_loses_and_duplicates_nothing() { + const REFS: usize = 1500; + let Some(mut nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "big").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let names: Vec = (0..REFS).map(|i| format!("refs/heads/b{i:04}")).collect(); + let updates: Vec<(&str, &str, &str)> = names.iter().map(|n| (n.as_str(), ZERO, A)).collect(); + // One entry with many refs: a single catch_up has plenty to publish one ack at a time. + rig.push(&id, &updates).await; + + let publisher = rig.publisher(NatsStart::Oldest).await; + let running = tokio::spawn({ + let (publisher, id) = (publisher.clone(), id.clone()); + async move { publisher.catch_up(&id).await } + }); + while rig.message_count().await < 20 { + tokio::time::sleep(Duration::from_millis(2)).await; + } + nats.stop(); + let outcome = running.await.unwrap(); + let error = outcome.expect_err("the server went away mid catch-up"); + eprintln!("outage error: {error:#}"); + + nats.restart().await; + rig.wait_connected().await; + let held = rig.message_count().await; + assert!(held >= 20 && held < REFS as u64, "partial progress survived the restart: {held}"); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted as u64, REFS as u64 - held, "exactly the rest: {r:?}"); + let mut ids = rig.ids().await; + assert_eq!(ids.len(), REFS); + let expected: Vec = names.iter().map(|n| msg_id(&id, 1, n)).collect(); + assert_eq!(ids, expected, "in order, nothing skipped, nothing twice"); + ids.sort(); + ids.dedup(); + assert_eq!(ids.len(), REFS); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn notifications_that_name_nothing_are_ignored_and_the_loop_survives() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let notify = rig.spawn_notify(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + + for body in [ + r#"{"key":"repos/t/r/wal/abc.pack"}"#, + r#"{"key":"somewhere/else/manifest.pb"}"#, + r#"{"repo":"t/ghost"}"#, + r#"{"key":"repos/t/ghost/manifest.pb"}"#, + r#"{"repo":"not a repo id"}"#, + r#"{"Records":[{"s3":{"object":{"key":"repos/t/ghost/manifest.pb"}}}]}"#, + "not json", + "{", + "", + "[]", + "null", + "\"a string\"", + ] { + rig.notify(body).await; + } + rig.client.publish(rig.cfg.events.nats.notify_subject.clone(), vec![0xff, 0xfe, 0x00].into()).await.unwrap(); + rig.client.flush().await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(!notify.is_finished(), "the notify loop must survive garbage"); + assert_eq!(rig.message_count().await, 0, "nothing was published for any of them"); + let repos: Vec = rig.registry.list().await.unwrap().iter().map(ToString::to_string).collect(); + assert_eq!(repos, ["t/r"], "an unknown repository is not created by being notified"); + + // A later valid notification still works. + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + assert!(!notify.is_finished()); + notify.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn known_limitation_head_start_skips_pushes_made_while_the_publisher_was_down() { + // PINNED: docs/EVENTS.md says `start = "head"` publishes only pushes made after the publisher + // first saw the repository. The mark is in memory, so a restart before the repository's first + // event treats everything pushed while the publisher was down as "before". If this test fails + // because the behavior changed, update the `start` paragraph of docs/EVENTS.md with it. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let first = rig.publisher(NatsStart::Head).await; + assert_eq!(first.catch_up(&id).await.unwrap().emitted, 0, "head start: history is not published"); + + // The publisher goes down; these two pushes happen while it is gone. + drop(first); + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + + let restarted = rig.publisher(NatsStart::Head).await; + assert_eq!(restarted.catch_up(&id).await.unwrap().emitted, 0, "seqs 2 and 3 are skipped forever"); + rig.push(&id, &[("refs/heads/main", C, A)]).await; + assert_eq!(restarted.catch_up(&id).await.unwrap().emitted, 1); + assert_eq!(rig.ids().await, [msg_id(&id, 4, "refs/heads/main")]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_foreign_message_on_a_repo_subject_fails_every_catch_up_with_a_clear_error() { + // PINNED operational risk: a message without Walgit-Seq / Nats-Msg-Id as the subject's last + // message makes every catch-up for that repository fail until it is removed from the stream. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + rig.js.publish("walgit.ref.t.r", "hello".into()).await.unwrap().await.unwrap(); + for _ in 0..3 { + let error = format!("{:#}", publisher.catch_up(&id).await.unwrap_err()); + assert!( + error.contains("walgit.ref.t.r") && error.contains("lacks Walgit-Seq / Nats-Msg-Id"), + "the error must say what is wrong: {error}" + ); + } + assert_eq!(rig.message_count().await, 1, "nothing was published on top of it"); + + // Other repositories are unaffected. + let other = RepoId::new("t", "other").unwrap(); + rig.registry.create(&other, ObjectFormat::Sha1).await.unwrap(); + rig.push(&other, &[("refs/heads/main", ZERO, A)]).await; + assert_eq!(publisher.catch_up(&other).await.unwrap().emitted, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_nats_server_restart_keeps_the_stream_the_subscriptions_and_the_service() { + let Some(mut nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let notify = rig.spawn_notify(&publisher); + let service = rig.spawn_service(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + + nats.restart().await; + rig.wait_connected().await; + assert_eq!(rig.message_count().await, 1, "the stream survives the restart"); + + // The queue subscription is back: a notification after the restart triggers a catch-up. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(2).await; + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 2, "refs/heads/main")]); + assert!(!notify.is_finished() && !service.is_finished()); + + // The $SRV service is back too. + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let reply = tokio::time::timeout( + Duration::from_secs(5), + rig.client.request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()), + ) + .await + .unwrap() + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!(report["emitted"], 1, "{report}"); + assert_eq!(rig.message_count().await, 3, "nothing was republished"); + notify.abort(); + service.abort(); +} + +fn error_code(reply: &async_nats::Message) -> Option { + reply + .headers + .as_ref() + .and_then(|h| h.get("Nats-Service-Error-Code")) + .map(|c| c.as_str().to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_service_answers_bad_requests_and_reports_the_counters() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = repo_with_three_events(&rig, "t", "r").await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let service = rig.spawn_service(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + + for (body, code) in [ + ("not json", "400"), + ("{}", "400"), + (r#"{"repo":42}"#, "400"), + (r#"{"repo":"no-slash"}"#, "400"), + (r#"{"repo":"a/b/c"}"#, "400"), + (r#"{"repo":"t/none"}"#, "404"), + ] { + let reply = rig.client.request("walgit.events.catch-up", body.into()).await.unwrap(); + assert_eq!(error_code(&reply).as_deref(), Some(code), "{body}"); + } + + // published: 3 through the endpoint. + let reply = rig.client.request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()).await.unwrap(); + let report: serde_json::Value = serde_json::from_slice(&reply.payload).unwrap(); + assert_eq!((report["emitted"].as_u64(), report["head_seq"].as_u64()), (Some(3), Some(2))); + + // deduplicated: 2, from forcing a republish of the entry inside the window. + rig.inject(&id, 2, "refs/heads/not-in-the-entry").await; + let restarted = rig.publisher(NatsStart::Oldest).await; + restarted.catch_up(&id).await.unwrap(); + + // gap_entries: 3, on a second repository. + let gap = RepoId::new("t", "gap").unwrap(); + rig.registry.create(&gap, ObjectFormat::Sha1).await.unwrap(); + rig.push(&gap, &[("refs/heads/main", ZERO, A)]).await; + assert_eq!(publisher.catch_up(&gap).await.unwrap().emitted, 1); + for (old, new) in [(A, B), (B, C), (C, A)] { + rig.push(&gap, &[("refs/heads/main", old, new)]).await; + } + rig.registry.open(&gap).await.unwrap().write_checkpoint().await.unwrap(); + rig.push(&gap, &[("refs/heads/main", A, B)]).await; + assert_eq!(publisher.catch_up(&gap).await.unwrap().gap, Some([2, 4])); + + // The service's publisher is `publisher`: its counters are what $SRV.STATS reports. + let stats = rig.client.request("$SRV.STATS.walgit-events", "".into()).await.unwrap(); + let stats: serde_json::Value = serde_json::from_slice(&stats.payload).unwrap(); + eprintln!("stats: {stats}"); + let data = &stats["endpoints"][0]["data"]; + assert_eq!(stats["endpoints"][0]["subject"], "walgit.events.catch-up"); + assert_eq!(data["published"], 5, "3 + 1 + 1"); + assert_eq!(data["deduplicated"], 0, "the duplicates were counted by the other publisher"); + assert_eq!(data["gap_entries"], 3); + assert_eq!(data["conflicts"], 0); + assert_eq!(data["sweep_found"], 0); + assert_eq!(restarted.counters.snapshot()["deduplicated"], 2); + service.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_periodic_sweep_finds_a_push_nobody_notified() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let sweeping = tokio::spawn(walgit_nats::wake::sweep_loop(publisher.clone(), Duration::from_millis(300))); + + // The startup sweep legitimately finds work and does not count it. + rig.wait_for_messages(1).await; + assert_eq!(publisher.counters.snapshot()["sweep_found"], 0); + + // No notification for this push: only the periodic sweep can find it. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.wait_for_messages(2).await; + assert_eq!(publisher.counters.snapshot()["sweep_found"], 1); + sweeping.abort(); +} diff --git a/crates/walgit-server/Cargo.toml b/crates/walgit-server/Cargo.toml index d922b95f..5f018659 100644 --- a/crates/walgit-server/Cargo.toml +++ b/crates/walgit-server/Cargo.toml @@ -66,6 +66,8 @@ rcgen.workspace = true tokio-rustls.workspace = true tempfile.workspace = true [dev-dependencies] +async-nats.workspace = true +walgit-nats = { workspace = true, features = ["testing"] } reqwest.workspace = true rcgen = { workspace = true, features = ["x509-parser"] } diff --git a/crates/walgit-server/tests/events_nats.rs b/crates/walgit-server/tests/events_nats.rs new file mode 100644 index 00000000..b6b6ede1 --- /dev/null +++ b/crates/walgit-server/tests/events_nats.rs @@ -0,0 +1,479 @@ +// Test fixtures use panics to fail the test, including shared helper functions. +#![allow(clippy::indexing_slicing, clippy::unwrap_used)] + +//! Events (docs/EVENTS.md, `walgit nats`): real `git push`es against the harness server, published +//! into a real `nats-server -js` by a `walgit_nats` publisher over the same registry. Skips (passes) +//! when `nats-server` is not on PATH. +mod harness; + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_nats::jetstream; +use futures::StreamExt; +use harness::{Server, TestRepo, git_in}; +use walgit_config::{NatsEventsConfig, NatsStart}; +use walgit_git::RepoId; +use walgit_nats::publisher::Publisher; +use walgit_nats::subject::SubjectPrefix; +use walgit_nats::testing::{NatsServer, start_nats}; + +type TestResult = anyhow::Result<()>; + +const ZERO_OID: &str = "0000000000000000000000000000000000000000"; + +struct Msg { + subject: String, + seq: String, + repo: String, + dedup_id: String, + body: serde_json::Value, +} + +struct Rig { + server: Server, + nats_cfg: NatsEventsConfig, + client: async_nats::Client, + js: jetstream::Context, + prefix: SubjectPrefix, + _nats: NatsServer, +} + +impl Rig { + async fn new() -> anyhow::Result> { + let Some(nats) = start_nats().await else { + return Ok(None); + }; + Ok(Some(Self::around(nats, Server::start().await?).await?)) + } + + async fn around(nats: NatsServer, server: Server) -> anyhow::Result { + let nats_cfg = NatsEventsConfig { + urls: vec![nats.url.clone()], + ..NatsEventsConfig::default() + }; + let client = walgit_nats::connect(&nats_cfg).await?; + let js = jetstream::new(client.clone()); + let prefix = SubjectPrefix::new(&nats_cfg.subject_prefix)?; + Ok(Rig { + server, + nats_cfg, + client, + js, + prefix, + _nats: nats, + }) + } + + /// A publisher with no memory of earlier ones, as after a restart. + async fn publisher(&self) -> anyhow::Result> { + let stream = walgit_nats::open_stream(&self.js, &self.nats_cfg, &self.prefix).await?; + Ok(Publisher::new( + self.server.state.registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + NatsStart::Oldest, + )) + } + + async fn stream(&self) -> jetstream::stream::Stream { + self.js.get_stream(&self.nats_cfg.stream).await.unwrap() + } + + async fn message_count(&self) -> u64 { + self.stream().await.get_info().await.unwrap().state.messages + } + + async fn wait_for_messages(&self, n: u64) { + let t0 = Instant::now(); + loop { + let have = self.message_count().await; + if have >= n { + return; + } + assert!( + t0.elapsed() < Duration::from_secs(10), + "timed out waiting for {n} messages; have {have}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// Every message in the stream, in stream order. + async fn messages(&self) -> Vec { + let stream = self.stream().await; + let count = stream.get_info().await.unwrap().state.messages; + let mut out = Vec::new(); + for stream_seq in 1..=count { + let raw = stream.get_raw_message(stream_seq).await.unwrap(); + let header = |name: &str| raw.headers.get(name).unwrap().as_str().to_string(); + out.push(Msg { + subject: raw.subject.to_string(), + seq: header("Walgit-Seq"), + repo: header("Walgit-Repo"), + dedup_id: header("Nats-Msg-Id"), + body: serde_json::from_slice(&raw.payload).unwrap(), + }); + } + out + } + + fn spawn_notify_loop(&self, publisher: &Arc) -> tokio::task::JoinHandle> { + tokio::spawn(walgit_nats::wake::notify_loop( + self.client.clone(), + self.nats_cfg.notify_subject.clone(), + self.nats_cfg.queue_group.clone(), + self.server.state.cfg.store_prefix(), + publisher.clone(), + )) + } +} + +/// A working clone whose `origin` is `owner/repo` on the server (created first). +async fn clone_of(server: &Server, owner: &str, repo: &str) -> anyhow::Result { + server.put_repo(owner, repo).await?; + let src = TestRepo::synthetic(1, 1)?; + git_in(&src, &["commit", "--allow-empty", "-m", "a"])?; + git_in(&src, &["branch", "-M", "main"])?; + git_in(&src, &["remote", "add", "origin", &server.repo_url(owner, repo)])?; + Ok(src) +} + +/// Let a notified catch-up (a queue subscription, no reply) settle: the loop is subscribed once a +/// published probe reaches it, which a short sleep approximates for a fresh subscription. +async fn subscribed() { + tokio::time::sleep(Duration::from_millis(500)).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pushes_become_events_with_the_right_shape() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + let publisher = rig.publisher().await?; + + // New branch. + git_in(&src, &["push", "origin", "main"])?; + let main_v1 = git_in(&src, &["rev-parse", "HEAD"])?.trim().to_string(); + let r = publisher.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted, r.conflict), (0, 1, 1, false)); + let msgs = rig.messages().await; + assert_eq!(msgs.len(), 1); + let m = &msgs[0]; + assert_eq!(m.subject, "walgit.ref.t.r"); + assert_eq!((m.seq.as_str(), m.repo.as_str()), ("1", "t/r")); + assert_eq!(m.dedup_id, "t/r:1:refs/heads/main"); + assert_eq!(m.body["action"], "create"); + assert_eq!(m.body["ref_type"], "branch"); + assert_eq!(m.body["ref_name"], "refs/heads/main"); + assert_eq!(m.body["old"], ZERO_OID, "create carries the zero OID, never empty"); + assert_eq!(m.body["new"], main_v1); + assert_eq!(m.body["pusher"], "anon"); + assert_eq!(m.body["repo"], "t/r"); + assert_eq!(m.body["_walgit"]["seq"], "1"); + + // A no-op push writes no WAL entry and so no event. + git_in(&src, &["push", "origin", "main"])?; + let r = publisher.catch_up(&id).await?; + assert_eq!((r.head_seq, r.emitted), (1, 0)); + assert_eq!(rig.message_count().await, 1); + + // Fast-forward. + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + let main_v2 = git_in(&src, &["rev-parse", "HEAD"])?.trim().to_string(); + assert_eq!(publisher.catch_up(&id).await?.emitted, 1); + let m = &rig.messages().await[1]; + assert_eq!((m.body["action"].as_str(), m.seq.as_str()), (Some("update"), "2")); + assert_eq!((m.body["old"].as_str(), m.body["new"].as_str()), (Some(main_v1.as_str()), Some(main_v2.as_str()))); + assert_eq!(m.dedup_id, "t/r:2:refs/heads/main"); + + // Lightweight and annotated tags. + git_in(&src, &["tag", "light"])?; + git_in(&src, &["tag", "-a", "-m", "annotated", "v1"])?; + git_in(&src, &["push", "origin", "light"])?; + git_in(&src, &["push", "origin", "v1"])?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 2); + let msgs = rig.messages().await; + for (m, name) in msgs[2..].iter().zip(["refs/tags/light", "refs/tags/v1"]) { + assert_eq!(m.body["action"], "create", "{name}"); + assert_eq!(m.body["ref_type"], "tag", "{name}"); + assert_eq!(m.body["ref_name"], name); + assert_eq!(m.body["old"], ZERO_OID); + } + assert_eq!(msgs[2].body["new"], main_v2, "a lightweight tag points at the commit"); + assert_ne!(msgs[3].body["new"], main_v2, "an annotated tag points at the tag object"); + + // Branch delete. + git_in(&src, &["push", "origin", "main:refs/heads/topic"])?; + git_in(&src, &["push", "origin", ":refs/heads/topic"])?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 2); + let msgs = rig.messages().await; + assert_eq!(msgs[4].body["action"], "create"); + let del = &msgs[5]; + assert_eq!(del.body["action"], "delete"); + assert_eq!(del.body["ref_type"], "branch"); + assert_eq!(del.body["ref_name"], "refs/heads/topic"); + assert_eq!(del.body["old"], main_v2); + assert_eq!(del.body["new"], ZERO_OID); + + // Every message names its own subject, stream order is WAL order. + assert!(msgs.iter().all(|m| m.subject == "walgit.ref.t.r")); + let seqs: Vec = msgs.iter().map(|m| m.seq.parse().unwrap()).collect(); + assert!(seqs.windows(2).all(|w| w[0] < w[1]), "{seqs:?}"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_atomic_multi_ref_push_is_one_message_per_ref_in_order() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["branch", "feature"])?; + git_in(&src, &["tag", "v1"])?; + git_in( + &src, + &["push", "--atomic", "origin", "main", "feature", "refs/tags/v1"], + )?; + + let publisher = rig.publisher().await?; + let r = publisher.catch_up(&id).await?; + assert_eq!((r.head_seq, r.emitted), (1, 3), "one entry, three refs"); + let msgs = rig.messages().await; + assert_eq!(msgs.len(), 3); + assert!(msgs.iter().all(|m| m.seq == "1"), "all share the entry's seq"); + let mut names: Vec<&str> = msgs + .iter() + .map(|m| m.body["ref_name"].as_str().unwrap()) + .collect(); + let ids: Vec<&str> = msgs.iter().map(|m| m.dedup_id.as_str()).collect(); + for (m, name) in msgs.iter().zip(&names) { + assert_eq!(m.dedup_id, format!("t/r:1:{name}")); + } + assert_eq!(ids.len(), ids.iter().collect::>().len()); + names.sort_unstable(); + assert_eq!(names, ["refs/heads/feature", "refs/heads/main", "refs/tags/v1"]); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_dotted_repository_name_is_encoded_in_the_subject() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("acme.io", "mono.repo")?; + let src = clone_of(&rig.server, "acme.io", "mono.repo").await?; + git_in(&src, &["push", "origin", "main"])?; + let publisher = rig.publisher().await?; + assert_eq!(publisher.catch_up(&id).await?.emitted, 1); + let m = &rig.messages().await[0]; + assert_eq!(m.subject, "walgit.ref.acme%2Eio.mono%2Erepo"); + assert_eq!(m.repo, "acme.io/mono.repo"); + assert_eq!(m.dedup_id, "acme.io/mono.repo:1:refs/heads/main"); + assert_eq!(m.body["repo"], "acme.io/mono.repo"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn bucket_notifications_wake_the_publisher_without_a_catch_up_call() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let src = clone_of(&rig.server, "t", "r").await?; + let publisher = rig.publisher().await?; + let notify = rig.spawn_notify_loop(&publisher); + subscribed().await; + + // An S3 bucket notification naming the manifest, as the object store publishes it. + git_in(&src, &["push", "origin", "main"])?; + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "repos/t/r/manifest.pb"}}}]}); + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), s3.to_string().into()) + .await?; + rig.wait_for_messages(1).await; + + // `{"repo": ...}` through the same subject. + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), r#"{"repo":"t/r"}"#.into()) + .await?; + rig.wait_for_messages(2).await; + + // A notification for something that is not a manifest names nothing. + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "origin", "main"])?; + let other = serde_json::json!({"key": "repos/t/r/wal/x.pack"}); + rig.client + .publish(rig.nats_cfg.notify_subject.clone(), other.to_string().into()) + .await?; + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(rig.message_count().await, 2); + + let msgs = rig.messages().await; + assert_eq!( + msgs.iter().map(|m| m.seq.as_str()).collect::>(), + ["1", "2"] + ); + notify.abort(); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restarted_publisher_resumes_from_the_stream() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + assert_eq!(rig.publisher().await?.catch_up(&id).await?.emitted, 2); + assert_eq!(rig.message_count().await, 2); + + let restarted = rig.publisher().await?; + let r = restarted.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 2, 0)); + assert_eq!(rig.message_count().await, 2, "resume must not republish"); + + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "origin", "main"])?; + let r = restarted.catch_up(&id).await?; + assert_eq!((r.from_seq, r.head_seq, r.emitted), (2, 3, 1)); + assert_eq!(rig.message_count().await, 3, "exactly one new message"); + assert_eq!(rig.messages().await[2].seq, "3"); + + // A third process that never saw any of it publishes nothing more. + assert_eq!(rig.publisher().await?.catch_up(&id).await?.emitted, 0); + assert_eq!(rig.message_count().await, 3); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_catch_up_endpoint_reports_a_real_repository() -> TestResult { + let Some(rig) = Rig::new().await? else { return Ok(()) }; + let src = clone_of(&rig.server, "t", "r").await?; + git_in(&src, &["push", "origin", "main"])?; + let publisher = rig.publisher().await?; + let service = tokio::spawn({ + let (client, prefix, publisher) = (rig.client.clone(), rig.prefix.clone(), publisher.clone()); + let queue_group = rig.nats_cfg.queue_group.clone(); + async move { walgit_nats::service::serve(client, &prefix, &queue_group, publisher).await } + }); + subscribed().await; + + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await?; + let report: serde_json::Value = serde_json::from_slice(&reply.payload)?; + assert_eq!(report["repo"], "t/r"); + assert_eq!((report["from_seq"].as_u64(), report["head_seq"].as_u64()), (Some(0), Some(1))); + assert_eq!(report["emitted"], 1); + assert!(report.get("conflict").is_none(), "{report}"); + assert_eq!(rig.message_count().await, 1); + + // Asked again, nothing is new. + let reply = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/r"}"#.into()) + .await?; + let report: serde_json::Value = serde_json::from_slice(&reply.payload)?; + assert_eq!((report["from_seq"].as_u64(), report["emitted"].as_u64()), (Some(1), Some(0))); + + let missing = rig + .client + .request("walgit.events.catch-up", r#"{"repo":"t/none"}"#.into()) + .await?; + let code = missing.headers.as_ref().and_then(|h| h.get("Nats-Service-Error-Code")); + assert_eq!(code.map(|c| c.as_str().to_string()), Some("404".into())); + + let mut stats = rig.client.subscribe("inbox.stats").await?; + rig.client + .publish_with_reply("$SRV.STATS.walgit-events", "inbox.stats", "".into()) + .await?; + let stats = tokio::time::timeout(Duration::from_secs(5), stats.next()) + .await? + .expect("stats reply"); + let stats: serde_json::Value = serde_json::from_slice(&stats.payload)?; + assert_eq!(stats["name"], "walgit-events"); + service.abort(); + Ok(()) +} + +type Captured = Arc>>; + +/// The webhook sink's target: records every event the bridge delivers. +async fn webhook() -> (String, Captured) { + let captured: Captured = Arc::default(); + let app = axum::Router::new().route( + "/events", + axum::routing::post({ + let captured = captured.clone(); + move |axum::Json(batch): axum::Json>| { + let captured = captured.clone(); + async move { + captured.lock().unwrap().extend(batch); + axum::http::StatusCode::OK + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.ok(); + }); + (format!("http://{addr}/events"), captured) +} + +/// The identity of an event as a consumer of either sink sees it. +fn identity(event: &serde_json::Value) -> String { + let field = |name: &str| event[name].as_str().unwrap_or_default().to_string(); + format!( + "{} seq={} {} {} {} -> {}", + field("repo"), + event["_walgit"]["seq"].as_str().unwrap_or_default(), + field("ref_name"), + field("action"), + field("old"), + field("new"), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_webhook_bridge_and_the_jetstream_publisher_agree() -> TestResult { + let Some(nats) = start_nats().await else { return Ok(()) }; + let (url, captured) = webhook().await; + let server = Server::start_with_tweak(|c| { + c.events.webhook_url = Some(url.clone()); + c.events.sweep_interval = Duration::ZERO; + }) + .await?; + let rig = Rig::around(nats, server).await?; + let bridge = rig.server.state.bridge.clone().expect("bridge enabled"); + let id = RepoId::new("t", "r")?; + let src = clone_of(&rig.server, "t", "r").await?; + + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "b"])?; + git_in(&src, &["push", "origin", "main"])?; + git_in(&src, &["tag", "light"])?; + git_in(&src, &["tag", "-a", "-m", "annotated", "v1"])?; + git_in(&src, &["push", "origin", "light", "v1"])?; + git_in(&src, &["branch", "feature"])?; + git_in(&src, &["commit", "--allow-empty", "-m", "c"])?; + git_in(&src, &["push", "--atomic", "origin", "main", "feature"])?; + git_in(&src, &["push", "origin", "--delete", "feature", "light"])?; + + let sent = bridge.catch_up(&id).await?; + let published = rig.publisher().await?.catch_up(&id).await?; + assert_eq!(sent.emitted, published.emitted, "{sent:?} {published:?}"); + assert_eq!(sent.head_seq, published.head_seq); + + let webhook_events: Vec = captured.lock().unwrap().iter().map(identity).collect(); + let stream_events: Vec = rig.messages().await.iter().map(|m| identity(&m.body)).collect(); + assert!(webhook_events.len() >= 8, "the scenario produced events: {webhook_events:#?}"); + assert_eq!(stream_events, webhook_events, "both sinks, same events, same order"); + // The wire body is one and the same, not just the identity fields. + let bodies: Vec = rig.messages().await.into_iter().map(|m| m.body).collect(); + assert_eq!(bodies, *captured.lock().unwrap()); + Ok(()) +} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 8972ede6..f3e3c809 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -146,6 +146,43 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). Gap, conflict and sweep counters are in the stats response. There is no HTTP listener. - **Losing the stream.** Wiping the stream republishes the WAL window on the next wake-up; history older than the oldest readable entry is not in the WAL and is backfilled with `walgit wal ls`. Wiping walgit loses nothing. + Deleting and recreating the stream republishes at once. A *purge* does not: JetStream keeps its duplicate + table across a purge, so inside `duplicate_window` the republished events are acknowledged as duplicates and + land nothing; the publisher leaves the repository open and the window's expiry heals it on a later wake-up. + Prefer delete and recreate, or wait out the window. +- **`start = "head"` and restarts.** The first-seen head of a repository without a message is remembered in + memory only. Pushes made while the publisher is down, before the repository's first event, are skipped for good + after a restart (pinned by a test). +- **A foreign message** (no `Walgit-Seq` / `Nats-Msg-Id`) as the last message on a repository's subject makes every + catch-up for that repository fail, naming the subject and stream sequence, until it is removed. +- **Cost.** Every wake-up for a repository reads the last message on its subject (one JetStream request) before + deciding there is nothing to do, so a wiped stream is noticed without a restart. + +Each behaviour above has a test against a real `nats-server -js` (`crates/walgit-nats/tests/jetstream.rs`, and +`crates/walgit-server/tests/events_nats.rs` with real git pushes); `tests/e2e.sh` runs `walgit nats` as a process. + +| Behaviour | Test | +|---|---| +| Push becomes one message per ref with the bridge's event shape | `pushes_become_events_with_the_right_shape`, `an_atomic_multi_ref_push_is_one_message_per_ref_in_order` | +| The webhook bridge and `walgit nats` emit the same events | `the_webhook_bridge_and_the_jetstream_publisher_agree` | +| Names are encoded in the subject | `a_dotted_repository_name_is_encoded_in_the_subject` | +| The stream is the cursor; a restart resumes from it | `stream_is_the_cursor`, `a_restarted_publisher_resumes_from_the_stream` | +| A crash mid-entry finishes that entry only | `a_crash_mid_entry_finishes_that_entry_only` | +| A NATS outage mid catch-up loses and duplicates nothing | `an_outage_mid_catch_up_loses_and_duplicates_nothing` | +| Broker dedup inside `duplicate_window`; none after it | `broker_dedup_absorbs_a_republish_inside_the_window`, `after_the_dedup_window_a_republish_lands_twice` | +| Concurrent publishers leave each event once | `a_second_publisher_that_moved_first_stops_this_one`, `racing_publishers_leave_each_event_exactly_once` | +| Losing the stream republishes the WAL window | `a_wiped_stream_is_republished_on_the_next_catch_up`, `a_purged_stream_heals_once_the_dedup_window_has_passed` | +| A log gap is counted, readable entries still published | `entries_folded_before_publishing_are_a_counted_gap` | +| Bucket notifications wake it; junk is ignored | `bucket_notifications_wake_the_publisher_without_a_catch_up_call`, `notifications_that_name_nothing_are_ignored_and_the_loop_survives` | +| The sweep finds what nobody notified | `the_periodic_sweep_finds_a_push_nobody_notified` | +| Service API: catch-up, errors, stats | `notifications_and_the_service_api_wake_the_publisher`, `the_catch_up_endpoint_reports_a_real_repository`, `the_service_answers_bad_requests_and_reports_the_counters` | +| A NATS server restart keeps stream, subscriptions and service | `a_nats_server_restart_keeps_the_stream_the_subscriptions_and_the_service` | +| `start = "head"` | `head_start_publishes_nothing_older`, `known_limitation_head_start_skips_pushes_made_while_the_publisher_was_down` | +| A foreign message wedges the repository with a clear error | `a_foreign_message_on_a_repo_subject_fails_every_catch_up_with_a_clear_error` | +| SIGTERM exits 0 | `tests/e2e.sh` phase 3 | + +Not covered: Prometheus metrics (the service stats are the only counters), multi-node JetStream clusters, and +the Containerfile / flake packaging. ```toml [events] diff --git a/tests/e2e.sh b/tests/e2e.sh index 905d5c2e..60c02e71 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -15,6 +15,9 @@ # WALGIT_TEST_S3_ENDPOINT=http://localhost:9000 tests/e2e.sh # WALGIT=/absolute/path/to/freshly/built/walgit tests/e2e.sh # +# Phase 3 (S3/rustfs plus `nats-server` and the `nats` CLI on PATH): `walgit nats` as a process next to +# `walgit serve`; real pushes become JetStream events; a restart republishes nothing; wake-ups work. +# # Requires: cargo, git, curl, python3 (for random port), ripgrep (S3 maintenance receipt). set -euo pipefail @@ -345,6 +348,184 @@ EOF kill "$SERVER_A_PID" "$SERVER_B_PID" 2>/dev/null || true fi +# --- phase 3: `walgit nats` as processes (optional, local only) -------------- + +# Real processes end to end: nats-server -js, `walgit serve` and `walgit nats` sharing the rustfs +# bucket, real git pushes, assertions through the `nats` CLI. The first walgit nats is woken by the +# sweep (and one notification); after a kill and restart the sweep is off, so a push there can only +# arrive through the notify subject. +if [[ -z "${WALGIT_E2E_BASE_URL:-}" && -n "${WALGIT_TEST_S3_ENDPOINT:-}" ]]; then + if ! command -v nats-server >/dev/null 2>&1 || ! command -v nats >/dev/null 2>&1; then + step "phase 3: walgit nats (skipped: nats-server and the nats CLI must be on PATH)" + else + step "phase 3: walgit nats against nats-server -js ($WALGIT_TEST_S3_ENDPOINT)" + export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-walgit-dev}" AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-walgit-dev-secret}" + + NATS_PORT="$(rand_port)" + NATS_URL="nats://127.0.0.1:${NATS_PORT}" + STREAM="WALGIT_E2E_$$" + REPO3_NAME="nats-$(date +%s)-$$" + REPO3="test/$REPO3_NAME" + SUBJECT3="walgit.ref.test.$REPO3_NAME" + PORT_N="$(rand_port)" + BASE_N="http://127.0.0.1:${PORT_N}" + + mkdir -p "$TMP/nats-store" + nats-server -js -a 127.0.0.1 -p "$NATS_PORT" -sd "$TMP/nats-store" > "$TMP/nats-server.log" 2>&1 & + PIDS+=("$!") + for ((i=0; i<30; i++)); do nats -s "$NATS_URL" rtt >/dev/null 2>&1 && break; sleep 1; done + nats -s "$NATS_URL" rtt >/dev/null 2>&1 || fail "nats-server did not start" + pass "nats-server ready on $NATS_PORT" + + # A fresh store prefix keeps the repositories earlier runs left in the bucket out of the stream. + cat > "$TMP/cfg-nats.toml" </dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["state"]["messages"])' 2>/dev/null || echo 0 + } + wait_nats_count() { + local want="$1" max="${2:-20}" + for ((i=0; i "$log" + RUST_LOG=info env "WALGIT__CACHE__DIR=$TMP/cache-nats-pub" "$@" \ + "$WALGIT" --config "$TMP/cfg-nats.toml" nats > "$log" 2>&1 & + NATS_PUB_PID=$! + PIDS+=("$NATS_PUB_PID") + for ((i=0; i<30; i++)); do + kill -0 "$NATS_PUB_PID" 2>/dev/null || { cat "$log"; fail "walgit nats exited"; } + grep -q "service ready" "$log" && return 0 + sleep 1 + done + cat "$log"; fail "walgit nats did not become ready" + } + + RUST_LOG=info env "WALGIT__CACHE__DIR=$TMP/cache-nats-serve" \ + "$WALGIT" --config "$TMP/cfg-nats.toml" serve > "$TMP/server-nats.log" 2>&1 & + SERVER_N_PID=$! + PIDS+=("$SERVER_N_PID") + wait_http "$BASE_N/healthz" 15 || fail "walgit serve (nats phase) not ready" + start_walgit_nats "$TMP/walgit-nats-1.log" + pass "walgit serve on $PORT_N and walgit nats ready" + + curl_auth -sf -X PUT "$BASE_N/$REPO3.git" -o /dev/null || fail "create $REPO3" + git clone -q "$SYNTH_DIR" "$TMP/work-nats" + (cd "$TMP/work-nats" && git remote set-url origin "$BASE_N/$REPO3.git") + + step "nats: create pushed, woken by a notification on walgit.notify" + (cd "$TMP/work-nats" && git push -q origin main) || fail "push main" + nats -s "$NATS_URL" pub walgit.notify "{\"repo\":\"$REPO3\"}" >/dev/null + wait_nats_count 1 10 || fail "no event for the first push after a notification" + pass "first event arrived" + + step "nats: update, tag and delete, found by the 2s sweep" + (cd "$TMP/work-nats" && git commit -q --allow-empty -m "e2e nats update" && git push -q origin main) || fail "push update" + (cd "$TMP/work-nats" && git tag -a -m "e2e nats tag" v1.0.0 && git push -q origin v1.0.0) || fail "push tag" + (cd "$TMP/work-nats" && git push -q origin :refs/tags/v1.0.0) || fail "delete tag" + wait_nats_count 4 30 || fail "expected 4 events, have $(nats_count)" + sleep 5 + [[ "$(nats_count)" -eq 4 ]] || fail "sweeps republished: $(nats_count) messages, expected 4" + EVENTS="$(nats_events)" + echo "$EVENTS" + [[ "$(echo "$EVENTS" | sed -n 1p)" == "$SUBJECT3 create branch refs/heads/main 1 $REPO3:1:refs/heads/main" ]] || fail "event 1" + [[ "$(echo "$EVENTS" | sed -n 2p)" == "$SUBJECT3 update branch refs/heads/main 2 $REPO3:2:refs/heads/main" ]] || fail "event 2" + [[ "$(echo "$EVENTS" | sed -n 3p)" == "$SUBJECT3 create tag refs/tags/v1.0.0 3 $REPO3:3:refs/tags/v1.0.0" ]] || fail "event 3" + [[ "$(echo "$EVENTS" | sed -n 4p)" == "$SUBJECT3 delete tag refs/tags/v1.0.0 4 $REPO3:4:refs/tags/v1.0.0" ]] || fail "event 4" + pass "create, update, tag create and delete on $SUBJECT3, in WAL order" + + step "nats: kill walgit nats, restart (sweep off): no duplicates, a new push arrives by notification" + kill "$NATS_PUB_PID" 2>/dev/null || true + wait "$NATS_PUB_PID" 2>/dev/null || true + start_walgit_nats "$TMP/walgit-nats-2.log" WALGIT__EVENTS__SWEEP_INTERVAL=0s + sleep 3 + [[ "$(nats_count)" -eq 4 ]] || fail "restart republished: $(nats_count) messages, expected 4" + pass "restart left the stream at 4 messages" + + (cd "$TMP/work-nats" && git commit -q --allow-empty -m "e2e nats after restart" && git push -q origin main) || fail "push after restart" + sleep 3 + [[ "$(nats_count)" -eq 4 ]] || fail "sweep is off but the push was published without a wake-up" + nats -s "$NATS_URL" pub walgit.notify "{\"repo\":\"$REPO3\"}" >/dev/null + wait_nats_count 5 10 || fail "no event after the notification following the restart" + sleep 3 + [[ "$(nats_count)" -eq 5 ]] || fail "expected exactly 5 messages, have $(nats_count)" + LAST="$(nats_events | tail -n 1)" + [[ "$LAST" == "$SUBJECT3 update branch refs/heads/main 5 $REPO3:5:refs/heads/main" ]] || fail "event 5: $LAST" + pass "post-restart push published exactly once" + + step "nats: \$SRV catch-up endpoint" + REPORT="$(nats -s "$NATS_URL" request walgit.events.catch-up "{\"repo\":\"$REPO3\"}" --raw 2>/dev/null || true)" + echo "$REPORT" + echo "$REPORT" | python3 -c ' +import json, sys +r = json.load(sys.stdin) +assert r["repo"] == sys.argv[1] and r["head_seq"] == 5 and r["emitted"] == 0, r' "$REPO3" || fail "catch-up report" + pass "catch-up reports head 5, nothing to emit" + + step "nats: SIGTERM makes walgit nats exit 0 promptly" + kill -TERM "$NATS_PUB_PID" + for ((i=0; i<50; i++)); do + kill -0 "$NATS_PUB_PID" 2>/dev/null || break + sleep 0.1 + done + if kill -0 "$NATS_PUB_PID" 2>/dev/null; then + cat "$TMP/walgit-nats-2.log"; fail "walgit nats still running 5s after SIGTERM" + fi + NATS_RC=0 + wait "$NATS_PUB_PID" 2>/dev/null || NATS_RC=$? + [[ "$NATS_RC" -eq 0 ]] || { cat "$TMP/walgit-nats-2.log"; fail "walgit nats exited $NATS_RC after SIGTERM"; } + grep -q "shutting down" "$TMP/walgit-nats-2.log" || fail "no shutdown line after SIGTERM" + pass "SIGTERM: walgit nats exited 0 within 5s" + + kill "$SERVER_N_PID" 2>/dev/null || true + curl_auth -s -X DELETE "$BASE_N/$REPO3.git" -o /dev/null || true + fi +fi + # --- done -------------------------------------------------------------------- step "e2e: all checks passed" From 6520d2bf08900b201fdf91b3497c6b7d79010e1b Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 21:20:45 -0400 Subject: [PATCH 4/8] fix(events): keep walgit nats from serving a stale manifest or prefetching packs Signed-off-by: Yordis Prieto --- crates/walgit-nats/src/lib.rs | 13 +- crates/walgit-nats/src/subject.rs | 31 +++ crates/walgit-nats/src/wake.rs | 51 +++++ crates/walgit-nats/tests/jetstream.rs | 274 +++++++++++++++++++++++++- docs/EVENTS.md | 10 + 5 files changed, 374 insertions(+), 5 deletions(-) diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs index e7286620..f6cb6f13 100644 --- a/crates/walgit-nats/src/lib.rs +++ b/crates/walgit-nats/src/lib.rs @@ -79,12 +79,23 @@ async fn shutdown_signal() { let _ = tokio::signal::ctrl_c().await; } +/// The config the publisher's registry runs on. This process never applies a push itself, so a +/// wake-up must always see the manifest as it is in the bucket (`freshness_ttl` would let it +/// skip the GET and miss the push it was woken for), and it only reads manifest and log, so +/// pack prefetch would pull bytes nobody reads. Both hold whatever `[wal]` says. +pub fn publisher_config(cfg: &Config) -> Config { + let mut cfg = cfg.clone(); + cfg.wal.freshness_ttl = std::time::Duration::ZERO; + cfg.wal.prefetch_packs = false; + cfg +} + /// `walgit nats`: publish until SIGTERM/ctrl-c, or fail when a worker stops. pub async fn run(cfg: Arc) -> anyhow::Result<()> { let nats = &cfg.events.nats; let prefix = SubjectPrefix::new(&nats.subject_prefix)?; let store = walgit_store::open_store(&cfg).await?; - let registry = walgit_wal::Registry::new(store, cfg.clone()); + let registry = walgit_wal::Registry::new(store, Arc::new(publisher_config(&cfg))); let client = crate::connect(nats).await?; let js = async_nats::jetstream::new(client.clone()); diff --git a/crates/walgit-nats/src/subject.rs b/crates/walgit-nats/src/subject.rs index 1d115b37..7fd5ea9c 100644 --- a/crates/walgit-nats/src/subject.rs +++ b/crates/walgit-nats/src/subject.rs @@ -206,4 +206,35 @@ mod tests { assert!(SubjectPrefix::new(bad).is_err(), "{bad:?}"); } } + + #[test] + fn odd_names_round_trip_and_never_carry_a_metacharacter() { + let names = [ + "a", "-", "_", "a.b.c", "*", ">", "%", "%2E", "%%", "a b", "tab\tnl\n", "é", "日本語", + "emoji\u{1F600}", "a/b\\c", "\u{0}", "..", "*.>", "Z%zz", + ]; + for raw in names { + let t = Token::encode(raw); + assert!(!t.as_str().is_empty(), "{raw:?}"); + assert!( + t.as_str() + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'%')), + "{raw:?} -> {t}" + ); + assert_eq!(Token::decode(t.as_str()).as_deref(), Some(raw), "{raw:?}"); + } + let mut seen = std::collections::HashSet::new(); + assert!(names.iter().all(|n| seen.insert(Token::encode(n).to_string())), "injective"); + } + + #[test] + fn every_single_byte_string_round_trips() { + for c in (1u32..0x250).filter_map(char::from_u32) { + let raw = c.to_string(); + let t = Token::encode(&raw); + assert_eq!(Token::decode(t.as_str()).as_deref(), Some(raw.as_str()), "{c:?}"); + assert!(!t.as_str().contains(['.', '*', '>']), "{c:?}"); + } + } } diff --git a/crates/walgit-nats/src/wake.rs b/crates/walgit-nats/src/wake.rs index f82f6b54..d10dc643 100644 --- a/crates/walgit-nats/src/wake.rs +++ b/crates/walgit-nats/src/wake.rs @@ -81,4 +81,55 @@ mod tests { let other = serde_json::json!({"key": "pre/repos/acme/mono/wal/x.pack"}); assert!(notified_repos(other.to_string().as_bytes(), "pre/").is_empty()); } + + fn names(body: &serde_json::Value, prefix: &str) -> Vec { + notified_repos(body.to_string().as_bytes(), prefix) + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn an_s3_event_with_several_records_names_each_repo_once() { + let rec = |name: &str, key: &str| { + serde_json::json!({"eventName": name, "s3": {"object": {"key": key}}}) + }; + let body = serde_json::json!({"Records": [ + rec("s3:ObjectCreated:Put", "repos/a/one/manifest.pb"), + rec("s3:ObjectCreated:CompleteMultipartUpload", "repos/b/two/manifest.pb"), + rec("ObjectCreated:Put", "repos/c/three/manifest.pb"), + rec("s3:ObjectCreated:Put", "repos/a/one/manifest.pb"), + rec("s3:ObjectRemoved:Delete", "repos/d/gone/manifest.pb"), + rec("s3:ObjectCreated:Put", "repos/e/four/log/1.pb"), + rec("s3:ObjectCreated:Put", "repos/a%2Dx/five/manifest.pb"), + ]}); + assert_eq!(names(&body, ""), ["a/one", "a-x/five", "b/two", "c/three"]); + } + + #[test] + fn a_gcs_envelope_counts_only_finalize_events() { + let env = |event: &str, id: &str| { + serde_json::json!({"message": {"attributes": {"eventType": event, "objectId": id}}}) + }; + assert_eq!( + names(&env("OBJECT_FINALIZE", "repos/a/one/manifest.pb"), ""), + ["a/one"] + ); + for event in ["OBJECT_DELETE", "OBJECT_ARCHIVE", "OBJECT_METADATA_UPDATE", ""] { + assert!(names(&env(event, "repos/a/one/manifest.pb"), "").is_empty(), "{event}"); + } + } + + #[test] + fn the_store_prefix_scopes_what_is_notified() { + let key = |k: &str| serde_json::json!({"key": k}); + assert_eq!(names(&key("pre/fix/repos/a/one/manifest.pb"), "pre/fix/"), ["a/one"]); + assert!(names(&key("repos/a/one/manifest.pb"), "pre/fix/").is_empty()); + assert!(names(&key("other/repos/a/one/manifest.pb"), "pre/fix/").is_empty()); + assert_eq!(names(&serde_json::json!({"repo": "a/one"}), "pre/fix/"), ["a/one"]); + let s3 = serde_json::json!({"Records": [{"eventName": "s3:ObjectCreated:Put", + "s3": {"object": {"key": "pre/fix/repos/a/one/manifest.pb"}}}]}); + assert_eq!(names(&s3, "pre/fix/"), ["a/one"]); + assert!(names(&s3, "").is_empty()); + } } diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs index d618bc60..31993f35 100644 --- a/crates/walgit-nats/tests/jetstream.rs +++ b/crates/walgit-nats/tests/jetstream.rs @@ -28,6 +28,7 @@ const C: &str = "3333333333333333333333333333333333333333"; struct Rig { cfg: Arc, + store: Arc, registry: Arc, client: async_nats::Client, js: jetstream::Context, @@ -52,10 +53,8 @@ impl Rig { cfg.events.nats.duplicate_window = window; } let cfg = Arc::new(cfg); - let registry = walgit_wal::Registry::new( - Arc::new(walgit_store::memory::MemoryStore::new()), - cfg.clone(), - ); + let store = Arc::new(walgit_store::memory::MemoryStore::new()); + let registry = walgit_wal::Registry::new(store.clone(), cfg.clone()); let client = walgit_nats::connect(&cfg.events.nats).await.unwrap(); let js = jetstream::ContextBuilder::new() .timeout(Duration::from_secs(3)) @@ -63,6 +62,7 @@ impl Rig { let prefix = SubjectPrefix::new(&cfg.events.nats.subject_prefix).unwrap(); Rig { cfg, + store, registry, client, js, @@ -85,6 +85,34 @@ impl Rig { ) } + /// What `walgit nats` runs: its own registry over the same store (it never sees the + /// writer's pushes except through the bucket) on the operator's `[wal]` settings, as + /// `run` adapts them. Returns the registry's cache dir guard too. + async fn operator_publisher( + &self, + operator: impl FnOnce(&mut Config), + ) -> (Arc, Arc, tempfile::TempDir) { + let cache = tempfile::tempdir().unwrap(); + let mut cfg = (*self.cfg).clone(); + cfg.cache.dir = cache.path().to_path_buf(); + operator(&mut cfg); + let registry = walgit_wal::Registry::new( + self.store.clone(), + Arc::new(walgit_nats::publisher_config(&cfg)), + ); + let stream = walgit_nats::open_stream(&self.js, &self.cfg.events.nats, &self.prefix) + .await + .unwrap(); + let publisher = Publisher::new( + registry.clone(), + self.js.clone(), + stream, + self.prefix.clone(), + NatsStart::Oldest, + ); + (publisher, registry, cache) + } + async fn push(&self, id: &RepoId, updates: &[(&str, &str, &str)]) -> u64 { let handle = self.registry.open(id).await.unwrap(); let txn = RefTransaction { @@ -907,3 +935,241 @@ async fn the_periodic_sweep_finds_a_push_nobody_notified() { assert_eq!(publisher.counters.snapshot()["sweep_found"], 1); sweeping.abort(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_notification_inside_the_freshness_ttl_still_publishes_the_push() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let (publisher, _registry, _cache) = rig + .operator_publisher(|c| c.wal.freshness_ttl = Duration::from_mins(10)) + .await; + let notify = rig.spawn_notify(&publisher); + tokio::time::sleep(Duration::from_millis(500)).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(1).await; + + // Pushed through another registry: the publisher only learns of it from the bucket. + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.notify(r#"{"repo":"t/r"}"#).await; + rig.wait_for_messages(2).await; + assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 2, "refs/heads/main")]); + notify.abort(); +} + +fn pack_files(dir: &std::path::Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for entry in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|e| e == "pack") { + found.push(path); + } + } + } + found +} + +/// A push carrying a real pack (one commit), so the repository has pack bytes in the bucket. +async fn push_with_pack(rig: &Rig, id: &RepoId) { + let work = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let out = std::process::Command::new("git") + .args(["-c", "user.name=T", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(work.path()) + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).unwrap().trim().to_string() + }; + git(&["init", "-q"]); + std::fs::write(work.path().join("f"), "x").unwrap(); + git(&["add", "."]); + git(&["commit", "-q", "-m", "c"]); + let head = git(&["rev-parse", "HEAD"]); + let pack = { + use std::io::Write as _; + let mut child = std::process::Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(work.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(format!("{head}\n").as_bytes()).unwrap(); + child.wait_with_output().unwrap().stdout + }; + let handle = rig.registry.open(id).await.unwrap(); + let ingested = handle + .local() + .ingest_pack( + std::io::Cursor::new(pack), + walgit_git::IngestOptions { fsck: false, max_bytes: None, thin: false }, + ) + .await + .unwrap(); + let txn = RefTransaction { + updates: vec![RefUpdate { + name: "refs/heads/main".into(), + old_oid: String::new(), + new_oid: head, + ..Default::default() + }], + atomic: true, + ..Default::default() + }; + let r = handle.publish_push(ingested, txn, HashMap::new()).await.unwrap(); + assert!(r.seq > 0, "push rejected: {:?}", r.per_ref); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_publisher_never_prefetches_packs_whatever_the_operator_configured() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + push_with_pack(&rig, &id).await; + + let (publisher, _registry, cache) = rig + .operator_publisher(|c| c.wal.prefetch_packs = true) + .await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(pack_files(cache.path()), Vec::::new(), "no pack bytes pulled"); + + // Control: the same operator config without the adaptation does prefetch. + let control_cache = tempfile::tempdir().unwrap(); + let mut operator = (*rig.cfg).clone(); + operator.cache.dir = control_cache.path().to_path_buf(); + operator.wal.prefetch_packs = true; + let plain = walgit_wal::Registry::new(rig.store.clone(), Arc::new(operator)); + let handle = plain.open(&id).await.unwrap(); + drop(handle.sync_refs().await.unwrap()); + let t0 = Instant::now(); + while pack_files(control_cache.path()).is_empty() { + assert!(t0.elapsed() < Duration::from_secs(10), "the control never prefetched"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_missing_stream_with_ensure_stream_off_fails_clearly_and_creates_nothing() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let mut cfg = rig.cfg.events.nats.clone(); + cfg.ensure_stream = false; + let err = walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap_err(); + let text = format!("{err:#}"); + assert!( + text.contains("WALGIT_EVENTS") && text.contains("ensure_stream is off"), + "the error must name the stream and the setting: {text}" + ); + assert!(rig.js.get_stream("WALGIT_EVENTS").await.is_err(), "nothing was created"); + + cfg.ensure_stream = true; + walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap(); + cfg.ensure_stream = false; + walgit_nats::open_stream(&rig.js, &cfg, &rig.prefix).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_existing_stream_with_other_settings_is_reused_not_clobbered() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let existing = jetstream::stream::Config { + name: "WALGIT_EVENTS".into(), + subjects: vec!["walgit.ref.>".into(), "other.>".into()], + max_messages: 1000, + duplicate_window: Duration::from_secs(30), + ..Default::default() + }; + rig.js.create_stream(existing).await.unwrap(); + + let stream = walgit_nats::open_stream(&rig.js, &rig.cfg.events.nats, &rig.prefix) + .await + .unwrap(); + let config = stream.cached_info().config.clone(); + assert_eq!(config.max_messages, 1000); + assert_eq!(config.duplicate_window, Duration::from_secs(30)); + assert_eq!(config.subjects, ["walgit.ref.>", "other.>"]); + + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_unparsable_server_url_fails_to_connect_naming_the_setting() { + let cfg = walgit_config::NatsEventsConfig { + urls: vec!["nats://bad host:not-a-port".into()], + ..Default::default() + }; + let err = walgit_nats::connect(&cfg).await.unwrap_err(); + assert!(format!("{err:#}").contains("events.nats.urls"), "{err:#}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn tags_and_deletes_carry_their_ref_type_and_zero_oids() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A), ("refs/tags/v1", ZERO, B)]).await; + rig.push(&id, &[("refs/heads/main", A, ZERO), ("refs/tags/v1", B, ZERO)]).await; + rig.push(&id, &[("refs/notes/x", ZERO, C)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 5); + + let stream = rig.stream().await; + let mut seen = Vec::new(); + for n in 1..=5 { + let raw = stream.get_raw_message(n).await.unwrap(); + let b: serde_json::Value = serde_json::from_slice(&raw.payload).unwrap(); + let get = |k: &str| b[k].as_str().unwrap().to_string(); + seen.push((get("action"), get("ref_type"), get("ref_name"), get("old"), get("new"))); + } + let row = |a: &str, t: &str, r: &str, o: &str, n: &str| { + (a.to_string(), t.to_string(), r.to_string(), o.to_string(), n.to_string()) + }; + assert_eq!( + seen, + [ + row("create", "branch", "refs/heads/main", ZERO, A), + row("create", "tag", "refs/tags/v1", ZERO, B), + row("delete", "branch", "refs/heads/main", A, ZERO), + row("delete", "tag", "refs/tags/v1", B, ZERO), + row("create", "", "refs/notes/x", ZERO, C), + ] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn many_sequential_pushes_land_in_seq_order_across_catch_ups() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let publisher = rig.publisher(NatsStart::Oldest).await; + let oid = |i: usize| format!("{i:040x}"); + let mut prev = ZERO.to_string(); + let mut expected = Vec::new(); + for i in 1..=30usize { + let next = oid(i); + let seq = rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + expected.push(msg_id(&id, seq, "refs/heads/main")); + prev = next; + if i % 7 == 0 { + publisher.catch_up(&id).await.unwrap(); + } + } + publisher.catch_up(&id).await.unwrap(); + assert_eq!(rig.ids().await, expected); +} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index f3e3c809..e66931f9 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -155,6 +155,9 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). after a restart (pinned by a test). - **A foreign message** (no `Walgit-Seq` / `Nats-Msg-Id`) as the last message on a repository's subject makes every catch-up for that repository fail, naming the subject and stream sequence, until it is removed. +- **Config adaptation.** `walgit nats` always revalidates the manifest (`wal.freshness_ttl` is forced to zero) and + never prefetches packs (`wal.prefetch_packs` is forced off), whatever `[wal]` says, because it never applies a + push itself and only reads manifest and log (`walgit_nats::publisher_config`). - **Cost.** Every wake-up for a repository reads the last message on its subject (one JetStream request) before deciding there is nothing to do, so a wiped stream is noticed without a restart. @@ -179,6 +182,13 @@ Each behaviour above has a test against a real `nats-server -js` (`crates/walgit | A NATS server restart keeps stream, subscriptions and service | `a_nats_server_restart_keeps_the_stream_the_subscriptions_and_the_service` | | `start = "head"` | `head_start_publishes_nothing_older`, `known_limitation_head_start_skips_pushes_made_while_the_publisher_was_down` | | A foreign message wedges the repository with a clear error | `a_foreign_message_on_a_repo_subject_fails_every_catch_up_with_a_clear_error` | +| A notification inside `wal.freshness_ttl` still publishes the push; no pack prefetch | `a_notification_inside_the_freshness_ttl_still_publishes_the_push`, `the_publisher_never_prefetches_packs_whatever_the_operator_configured` | +| Subject tokens round-trip odd names and never carry `.`, `*`, `>` | `odd_names_round_trip_and_never_carry_a_metacharacter`, `every_single_byte_string_round_trips` (unit, `subject.rs`) | +| Notification shapes: S3 multi-record, GCS finalize only, `store.prefix` | `an_s3_event_with_several_records_names_each_repo_once`, `a_gcs_envelope_counts_only_finalize_events`, `the_store_prefix_scopes_what_is_notified` (unit, `wake.rs`) | +| `ensure_stream = false` with no stream fails clearly; an existing stream is reused | `a_missing_stream_with_ensure_stream_off_fails_clearly_and_creates_nothing`, `an_existing_stream_with_other_settings_is_reused_not_clobbered` | +| A bad server URL fails to connect naming the setting | `an_unparsable_server_url_fails_to_connect_naming_the_setting` | +| Tags, deletes and non-branch refs carry `ref_type` and zero OIDs | `tags_and_deletes_carry_their_ref_type_and_zero_oids` | +| Many pushes land in seq order across catch-ups | `many_sequential_pushes_land_in_seq_order_across_catch_ups` | | SIGTERM exits 0 | `tests/e2e.sh` phase 3 | Not covered: Prometheus metrics (the service stats are the only counters), multi-node JetStream clusters, and From b9f2567cf183a61ba85979512a87d29990fbf7ec Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 21:33:24 -0400 Subject: [PATCH 5/8] fix(events): publish a recreated repository as a new one Signed-off-by: Yordis Prieto --- crates/walgit-nats/src/publisher.rs | 18 ++++++++- crates/walgit-nats/tests/jetstream.rs | 58 +++++++++++++++++++++++++++ docs/EVENTS.md | 13 ++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs index 9dcc1781..ce049ada 100644 --- a/crates/walgit-nats/src/publisher.rs +++ b/crates/walgit-nats/src/publisher.rs @@ -53,6 +53,7 @@ pub struct Counters { pub deduplicated: AtomicU64, pub gap_entries: AtomicU64, pub conflicts: AtomicU64, + pub rewinds: AtomicU64, pub sweep_found: AtomicU64, } @@ -64,6 +65,7 @@ impl Counters { "deduplicated": n(&self.deduplicated), "gap_entries": n(&self.gap_entries), "conflicts": n(&self.conflicts), + "rewinds": n(&self.rewinds), "sweep_found": n(&self.sweep_found), }) } @@ -162,13 +164,25 @@ impl Publisher { let last = self.last_published(&subject).await?; if locked(&self.done) .get(id) - .is_some_and(|(done_head, done_last)| *done_head >= head && *done_last == last) + .is_some_and(|(done_head, done_last)| *done_head == head && *done_last == last) { return Ok(report); } + // The stream is ahead of the WAL: the repository was deleted and created again (or the log + // rewound). What is on the subject belongs to the old incarnation, so it names no cursor; + // its stream sequence still guards the first publish. + let rewound = last.as_ref().is_some_and(|l| l.id.seq() > head); + if rewound { + self.counters.rewinds.fetch_add(1, Ordering::Relaxed); + locked(&self.cold_marks).remove(id); + tracing::warn!(repo = %id, published = last.as_ref().map_or(0, |l| l.id.seq()), head, + "the stream is ahead of the WAL: the repository was recreated; publishing it as a new one"); + } + let basis = if rewound { None } else { last.as_ref() }; + // First seq to read; `resume` is the entry the stream may hold only part of. - let (first, resume) = match &last { + let (first, resume) = match basis { Some(l) if l.id.seq() >= lowest => (l.id.seq(), Some(l)), Some(l) => { // The newest published entry is already folded away: what follows it up to the diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs index 31993f35..17298435 100644 --- a/crates/walgit-nats/tests/jetstream.rs +++ b/crates/walgit-nats/tests/jetstream.rs @@ -1173,3 +1173,61 @@ async fn many_sequential_pushes_land_in_seq_order_across_catch_ups() { publisher.catch_up(&id).await.unwrap(); assert_eq!(rig.ids().await, expected); } + +async fn published_then_recreated(rig: &Rig, publisher: &Publisher) -> RepoId { + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + let mut prev = ZERO.to_string(); + for i in 1..=3usize { + let next = format!("{i:040x}"); + rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + prev = next; + } + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + rig.registry.delete(&id).await.unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + id +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_is_published_as_a_new_one() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let id = published_then_recreated(&rig, &publisher).await; + let seq = rig.push(&id, &[("refs/heads/fresh", ZERO, A)]).await; + assert_eq!(seq, 1, "the new incarnation starts again at seq 1"); + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.head_seq, r.emitted, r.conflict), (1, 1, false), "{r:?}"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); + let ids = rig.ids().await; + assert_eq!(ids.len(), 4); + assert_eq!(ids.last().unwrap(), &msg_id(&id, 1, "refs/heads/fresh")); + let again = publisher.catch_up(&id).await.unwrap(); + assert_eq!(again.emitted, 0, "steady afterwards, not rewound again"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window() { + // PINNED: the new incarnation's first event has the old incarnation's Nats-Msg-Id + // (`t/r:1:refs/heads/main`), so inside the window the broker swallows it, the catch-up is not + // confirmed and the repository stays open until the window has passed. + let Some(nats) = start_nats().await else { return }; + let rig = Rig::with_window(&nats, NatsStart::Oldest, Some(Duration::from_secs(2))).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + let id = published_then_recreated(&rig, &publisher).await; + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false)); + assert_eq!(publisher.counters.snapshot()["deduplicated"], 1); + assert_eq!(rig.message_count().await, 3, "swallowed as a duplicate of the old one"); + + tokio::time::sleep(Duration::from_secs(4)).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false), "{r:?}"); + assert_eq!(rig.message_count().await, 4, "it landed once the window had passed"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); +} diff --git a/docs/EVENTS.md b/docs/EVENTS.md index e66931f9..0f11d899 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -155,6 +155,18 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). after a restart (pinned by a test). - **A foreign message** (no `Walgit-Seq` / `Nats-Msg-Id`) as the last message on a repository's subject makes every catch-up for that repository fail, naming the subject and stream sequence, until it is removed. +- **Deleting and recreating a repository.** The manifest has no incarnation id, so the only signal is the stream + being ahead of the WAL: when the last message on a subject names a seq above the repository's `head_seq`, the + publisher warns, counts it (`rewinds` in the service stats), ignores that message as a cursor (the first seq + follows `events.nats.start`) and still guards the first publish with the subject's real last stream sequence, so + it does not conflict. The new incarnation's `Nats-Msg-Id` (`::`) can equal the old one's: inside + `duplicate_window` the broker swallows it, the catch-up is not confirmed, the repository stays open and the + event lands on the first wake-up after the window (pinned by a test). Limitation: a recreated repository whose + head has already reached the old last seq before the publisher wakes is indistinguishable from the old one, and + its events up to that seq are never published. Consumers should treat a delete and recreate as a new stream. + The webhook bridge has the same ambiguity from the other side: its cursor lives under the repository's prefix and + is deleted with it, so the new incarnation is published from seq 1 and the dedup key `(repo, seq, ref_name)` + collides with the old incarnation's events. - **Config adaptation.** `walgit nats` always revalidates the manifest (`wal.freshness_ttl` is forced to zero) and never prefetches packs (`wal.prefetch_packs` is forced off), whatever `[wal]` says, because it never applies a push itself and only reads manifest and log (`walgit_nats::publisher_config`). @@ -189,6 +201,7 @@ Each behaviour above has a test against a real `nats-server -js` (`crates/walgit | A bad server URL fails to connect naming the setting | `an_unparsable_server_url_fails_to_connect_naming_the_setting` | | Tags, deletes and non-branch refs carry `ref_type` and zero OIDs | `tags_and_deletes_carry_their_ref_type_and_zero_oids` | | Many pushes land in seq order across catch-ups | `many_sequential_pushes_land_in_seq_order_across_catch_ups` | +| A deleted and recreated repository is detected and published as new; an id collision heals after the dedup window | `a_recreated_repository_is_published_as_a_new_one`, `a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window` | | SIGTERM exits 0 | `tests/e2e.sh` phase 3 | Not covered: Prometheus metrics (the service stats are the only counters), multi-node JetStream clusters, and From 37e1a0bc7d1b8c6c5d79a3a757b100390f65d14a Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 21:49:54 -0400 Subject: [PATCH 6/8] fix(events): keep a recreated repository's head mark across wake-ups Signed-off-by: Yordis Prieto --- crates/walgit-nats/src/publisher.rs | 26 +++++++++++++++---- crates/walgit-nats/tests/jetstream.rs | 36 +++++++++++++++++++++++++-- docs/EVENTS.md | 10 +++++--- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs index ce049ada..7ff8cae6 100644 --- a/crates/walgit-nats/src/publisher.rs +++ b/crates/walgit-nats/src/publisher.rs @@ -82,11 +82,21 @@ pub struct Publisher { /// last message for its subject. A wake-up that finds both unchanged spares the log read. The /// stream half is what notices a purged or recreated stream. Never a source of truth. done: std::sync::Mutex)>>, - /// `start = "head"`: the head a cold repository was first seen at. - cold_marks: std::sync::Mutex>, + /// The head a cold repository was first seen at (`start = "head"`), and for a recreated one, + /// which old incarnation it was seen behind. + cold_marks: std::sync::Mutex>, pub counters: Counters, } +/// A cold repository's first sighting. `behind` is the stream sequence of the old incarnation's +/// last message when the repository was recreated: the same rewind seen again keeps its mark, so +/// pushes after the first sighting are published while the old message still tops the subject. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ColdMark { + head: u64, + behind: Option, +} + fn locked(m: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) } @@ -173,9 +183,15 @@ impl Publisher { // rewound). What is on the subject belongs to the old incarnation, so it names no cursor; // its stream sequence still guards the first publish. let rewound = last.as_ref().is_some_and(|l| l.id.seq() > head); - if rewound { + let behind = last.as_ref().filter(|_| rewound).map(|l| l.stream_seq); + let seen = ColdMark { head, behind }; + let first_sighting = rewound + && locked(&self.cold_marks) + .get(id) + .is_none_or(|mark| mark.behind != behind); + if first_sighting { self.counters.rewinds.fetch_add(1, Ordering::Relaxed); - locked(&self.cold_marks).remove(id); + locked(&self.cold_marks).insert(id.clone(), seen); tracing::warn!(repo = %id, published = last.as_ref().map_or(0, |l| l.id.seq()), head, "the stream is ahead of the WAL: the repository was recreated; publishing it as a new one"); } @@ -201,7 +217,7 @@ impl Publisher { None => match self.start { NatsStart::Oldest => (lowest, None), NatsStart::Head => { - let mark = *locked(&self.cold_marks).entry(id.clone()).or_insert(head); + let mark = locked(&self.cold_marks).entry(id.clone()).or_insert(seen).head; (mark.saturating_add(1).max(lowest), None) } }, diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs index 17298435..3ee940d5 100644 --- a/crates/walgit-nats/tests/jetstream.rs +++ b/crates/walgit-nats/tests/jetstream.rs @@ -14,6 +14,7 @@ use futures::StreamExt; use walgit_config::{Config, NatsStart}; use walgit_events::refs_from_entries; use walgit_git::{ObjectFormat, RepoId}; +use walgit_nats::auth::NatsAuth; use walgit_nats::message::EventMessage; use walgit_nats::publisher::Publisher; use walgit_nats::subject::{RepoSubject, SubjectPrefix}; @@ -55,7 +56,7 @@ impl Rig { let cfg = Arc::new(cfg); let store = Arc::new(walgit_store::memory::MemoryStore::new()); let registry = walgit_wal::Registry::new(store.clone(), cfg.clone()); - let client = walgit_nats::connect(&cfg.events.nats).await.unwrap(); + let client = walgit_nats::connect(&cfg.events.nats, &NatsAuth::None).await.unwrap(); let js = jetstream::ContextBuilder::new() .timeout(Duration::from_secs(3)) .build(client.clone()); @@ -1112,7 +1113,7 @@ async fn an_unparsable_server_url_fails_to_connect_naming_the_setting() { urls: vec!["nats://bad host:not-a-port".into()], ..Default::default() }; - let err = walgit_nats::connect(&cfg).await.unwrap_err(); + let err = walgit_nats::connect(&cfg, &NatsAuth::None).await.unwrap_err(); assert!(format!("{err:#}").contains("events.nats.urls"), "{err:#}"); } @@ -1209,6 +1210,37 @@ async fn a_recreated_repository_is_published_as_a_new_one() { assert_eq!(publisher.counters.snapshot()["rewinds"], 1); } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_repository_under_head_start_publishes_pushes_after_its_first_sighting() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Head).await; + let publisher = rig.publisher(NatsStart::Head).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0, "first sighting at head 0"); + let mut prev = ZERO.to_string(); + for i in 1..=3usize { + let next = format!("{i:040x}"); + rig.push(&id, &[("refs/heads/main", &prev, &next)]).await; + prev = next; + } + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 3); + rig.registry.delete(&id).await.unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + + rig.push(&id, &[("refs/heads/before", ZERO, A)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!(r.emitted, 0, "head start: what the new incarnation held when first seen is skipped"); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1); + + rig.push(&id, &[("refs/heads/after", ZERO, B)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.emitted, r.conflict), (1, false), "{r:?}"); + assert_eq!(rig.ids().await.last().unwrap(), &msg_id(&id, 2, "refs/heads/after")); + assert_eq!(publisher.counters.snapshot()["rewinds"], 1, "the same rewind is not counted twice"); + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 0); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window() { // PINNED: the new incarnation's first event has the old incarnation's Nats-Msg-Id diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 0f11d899..89699022 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -158,7 +158,8 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). - **Deleting and recreating a repository.** The manifest has no incarnation id, so the only signal is the stream being ahead of the WAL: when the last message on a subject names a seq above the repository's `head_seq`, the publisher warns, counts it (`rewinds` in the service stats), ignores that message as a cursor (the first seq - follows `events.nats.start`) and still guards the first publish with the subject's real last stream sequence, so + follows `events.nats.start`; with `"head"` the new incarnation's head at that first sighting is kept while the + old message still tops the subject, so later pushes are published) and still guards the first publish with the subject's real last stream sequence, so it does not conflict. The new incarnation's `Nats-Msg-Id` (`::`) can equal the old one's: inside `duplicate_window` the broker swallows it, the catch-up is not confirmed, the repository stays open and the event lands on the first wake-up after the window (pinned by a test). Limitation: a recreated repository whose @@ -201,7 +202,7 @@ Each behaviour above has a test against a real `nats-server -js` (`crates/walgit | A bad server URL fails to connect naming the setting | `an_unparsable_server_url_fails_to_connect_naming_the_setting` | | Tags, deletes and non-branch refs carry `ref_type` and zero OIDs | `tags_and_deletes_carry_their_ref_type_and_zero_oids` | | Many pushes land in seq order across catch-ups | `many_sequential_pushes_land_in_seq_order_across_catch_ups` | -| A deleted and recreated repository is detected and published as new; an id collision heals after the dedup window | `a_recreated_repository_is_published_as_a_new_one`, `a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window` | +| A deleted and recreated repository is detected and published as new; an id collision heals after the dedup window | `a_recreated_repository_is_published_as_a_new_one`, `a_recreated_repository_under_head_start_publishes_pushes_after_its_first_sighting`, `a_recreated_repository_colliding_with_the_old_ids_heals_after_the_dedup_window` | | SIGTERM exits 0 | `tests/e2e.sh` phase 3 | Not covered: Prometheus metrics (the service stats are the only counters), multi-node JetStream clusters, and @@ -212,10 +213,13 @@ the Containerfile / flake packaging. sweep_interval = "5m" [events.nats] urls = ["nats://nats.internal:4222"] -credentials_file = "/etc/walgit/nats.creds" start = "oldest" ``` +Authentication is never in `walgit.toml`: `walgit nats` reads the standard NATS variables and uses the +first that is set, `NATS_CREDS` (a `.creds` file), `NATS_NKEY` (a seed), `NATS_USER` with `NATS_PASSWORD`, +then `NATS_TOKEN`; with none set it connects without credentials. Only the method is logged. + ## Consumer checklist 1. Verify `X-Walgit-Signature` (if you set a secret), then parse the array. From 826f04fe30bbf42c257de085e4d162a61f6b313a Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 21:49:55 -0400 Subject: [PATCH 7/8] feat(events): authenticate walgit nats from the NATS environment variables Signed-off-by: Yordis Prieto --- Cargo.lock | 1 + Cargo.toml | 1 + crates/walgit-config/src/lib.rs | 3 - crates/walgit-nats/Cargo.toml | 2 + crates/walgit-nats/src/auth.rs | 220 ++++++++++++++++++++++ crates/walgit-nats/src/lib.rs | 23 +-- crates/walgit-nats/src/testing.rs | 30 ++- crates/walgit-nats/tests/auth.rs | 125 ++++++++++++ crates/walgit-server/tests/events_nats.rs | 2 +- walgit.example.toml | 3 +- 10 files changed, 387 insertions(+), 23 deletions(-) create mode 100644 crates/walgit-nats/src/auth.rs create mode 100644 crates/walgit-nats/tests/auth.rs diff --git a/Cargo.lock b/Cargo.lock index 6da16f94..219881a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6006,6 +6006,7 @@ dependencies = [ "async-nats", "bytes", "futures", + "nkeys", "serde", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index b3543f3b..ca3f2483 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,6 +140,7 @@ parking_lot = "0.12" moka = { version = "0.12", features = ["sync"] } mimalloc = "0.1" tempfile = "3" +nkeys = "0.4" flate2 = "1" async-compression = { version = "0.4", features = ["tokio", "gzip"] } pin-project-lite = "0.2" diff --git a/crates/walgit-config/src/lib.rs b/crates/walgit-config/src/lib.rs index b1f609e6..b17cf8ad 100644 --- a/crates/walgit-config/src/lib.rs +++ b/crates/walgit-config/src/lib.rs @@ -671,8 +671,6 @@ pub enum NatsStart { pub struct NatsEventsConfig { /// Server URLs (`nats://host:4222`); the client connects to the first reachable one. pub urls: Vec, - /// A `.creds` or `NKey` seed file. Never logged. - pub credentials_file: Option, /// The `JetStream` stream holding the events (subjects `.ref.>`). pub stream: String, /// Create the stream when missing (file storage, `duplicate_window`). An existing stream is used as is. @@ -693,7 +691,6 @@ impl Default for NatsEventsConfig { fn default() -> Self { NatsEventsConfig { urls: vec!["nats://127.0.0.1:4222".into()], - credentials_file: None, stream: "WALGIT_EVENTS".into(), ensure_stream: true, duplicate_window: Duration::from_hours(1), diff --git a/crates/walgit-nats/Cargo.toml b/crates/walgit-nats/Cargo.toml index 19e24170..2aa9311d 100644 --- a/crates/walgit-nats/Cargo.toml +++ b/crates/walgit-nats/Cargo.toml @@ -28,6 +28,8 @@ testing = ["dep:tempfile"] [dev-dependencies] walgit-nats = { path = ".", features = ["testing"] } walgit-proto.workspace = true +nkeys.workspace = true +tempfile.workspace = true [lints] workspace = true diff --git a/crates/walgit-nats/src/auth.rs b/crates/walgit-nats/src/auth.rs new file mode 100644 index 00000000..3397f753 --- /dev/null +++ b/crates/walgit-nats/src/auth.rs @@ -0,0 +1,220 @@ +//! How `walgit nats` authenticates to NATS, resolved from the standard `NATS_*` environment +//! variables so secrets never sit in `walgit.toml`. + +use std::fmt; +use std::path::PathBuf; + +use anyhow::Context; + +pub const ENV_NATS_CREDS: &str = "NATS_CREDS"; +pub const ENV_NATS_NKEY: &str = "NATS_NKEY"; +pub const ENV_NATS_USER: &str = "NATS_USER"; +pub const ENV_NATS_PASSWORD: &str = "NATS_PASSWORD"; +pub const ENV_NATS_TOKEN: &str = "NATS_TOKEN"; + +/// NATS authentication method. +/// +/// When resolved from environment variables, priority order is: +/// 1. Credentials file (`NATS_CREDS`) +/// 2. `NKey` seed (`NATS_NKEY`) +/// 3. User/Password (`NATS_USER` + `NATS_PASSWORD`) +/// 4. Token (`NATS_TOKEN`) +/// 5. No auth +#[derive(Clone)] +pub enum NatsAuth { + Credentials(PathBuf), + NKey(String), + UserPassword { user: String, password: String }, + Token(String), + None, +} + +/// `NKey`, `password`, and `Token` are bearer credentials and must not be logged. +impl fmt::Debug for NatsAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Credentials(path) => f.debug_tuple("Credentials").field(path).finish(), + Self::NKey(_) => f.debug_tuple("NKey").field(&"").finish(), + Self::UserPassword { user, .. } => f + .debug_struct("UserPassword") + .field("user", user) + .field("password", &"") + .finish(), + Self::Token(_) => f.debug_tuple("Token").field(&"").finish(), + Self::None => f.write_str("None"), + } + } +} + +impl NatsAuth { + pub fn description(&self) -> &'static str { + match self { + Self::Credentials(_) => "credentials file", + Self::NKey(_) => "NKey", + Self::UserPassword { .. } => "user/password", + Self::Token(_) => "token", + Self::None => "none", + } + } + + /// Resolve from the process environment. + pub fn from_process_env() -> Self { + Self::from_env(|name| std::env::var(name).ok()) + } + + /// Resolve through `var`, in the priority order of [`NatsAuth`]. + pub fn from_env(var: impl Fn(&str) -> Option) -> Self { + if let Some(path) = var(ENV_NATS_CREDS) { + return Self::Credentials(PathBuf::from(path)); + } + if let Some(seed) = var(ENV_NATS_NKEY) { + return Self::NKey(seed); + } + if let (Some(user), Some(password)) = (var(ENV_NATS_USER), var(ENV_NATS_PASSWORD)) { + return Self::UserPassword { user, password }; + } + if let Some(token) = var(ENV_NATS_TOKEN) { + return Self::Token(token); + } + Self::None + } + + pub(crate) async fn connect_options(&self) -> anyhow::Result { + Ok(match self { + Self::Credentials(path) => { + async_nats::ConnectOptions::with_credentials_file(path.clone()) + .await + .with_context(|| format!("{ENV_NATS_CREDS}: {}", path.display()))? + } + Self::NKey(seed) => async_nats::ConnectOptions::with_nkey(seed.clone()), + Self::UserPassword { user, password } => { + async_nats::ConnectOptions::with_user_and_password(user.clone(), password.clone()) + } + Self::Token(token) => async_nats::ConnectOptions::with_token(token.clone()), + Self::None => async_nats::ConnectOptions::new(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn resolve(vars: &[(&str, &str)]) -> NatsAuth { + let vars: HashMap = vars + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + NatsAuth::from_env(|name| vars.get(name).cloned()) + } + + #[test] + fn no_variables_mean_no_auth() { + assert!(matches!(resolve(&[]), NatsAuth::None)); + } + + #[test] + fn credentials_take_priority() { + let auth = resolve(&[ + ("NATS_CREDS", "/path/to/creds"), + ("NATS_NKEY", "some-nkey"), + ("NATS_TOKEN", "some-token"), + ]); + assert!( + matches!(auth, NatsAuth::Credentials(p) if p == std::path::Path::new("/path/to/creds")) + ); + } + + #[test] + fn nkey_over_user_password_and_token() { + let auth = resolve(&[ + ("NATS_NKEY", "my-nkey"), + ("NATS_USER", "user"), + ("NATS_PASSWORD", "pass"), + ("NATS_TOKEN", "tok"), + ]); + assert!(matches!(auth, NatsAuth::NKey(k) if k == "my-nkey")); + } + + #[test] + fn user_password_over_token() { + let auth = resolve(&[ + ("NATS_USER", "user"), + ("NATS_PASSWORD", "pass"), + ("NATS_TOKEN", "tok"), + ]); + assert!(matches!( + auth, + NatsAuth::UserPassword { user, password } if user == "user" && password == "pass" + )); + } + + #[test] + fn token_when_nothing_else() { + assert!( + matches!(resolve(&[("NATS_TOKEN", "my-token")]), NatsAuth::Token(t) if t == "my-token") + ); + } + + #[test] + fn user_without_password_is_no_auth() { + assert!(matches!(resolve(&[("NATS_USER", "user")]), NatsAuth::None)); + assert!(matches!( + resolve(&[("NATS_PASSWORD", "pass")]), + NatsAuth::None + )); + } + + #[test] + fn description_matches_variant() { + assert_eq!( + NatsAuth::Credentials("/a".into()).description(), + "credentials file" + ); + assert_eq!(NatsAuth::NKey("k".into()).description(), "NKey"); + assert_eq!( + NatsAuth::UserPassword { + user: "u".into(), + password: "p".into() + } + .description(), + "user/password" + ); + assert_eq!(NatsAuth::Token("t".into()).description(), "token"); + assert_eq!(NatsAuth::None.description(), "none"); + } + + #[test] + fn debug_redacts_credentials() { + let nkey = format!("{:?}", NatsAuth::NKey("super-secret-nkey".into())); + assert!(!nkey.contains("super-secret-nkey")); + assert!(nkey.contains("")); + + let user_password = format!( + "{:?}", + NatsAuth::UserPassword { + user: "alice".into(), + password: "super-secret-password".into(), + } + ); + assert!(user_password.contains("alice")); + assert!(!user_password.contains("super-secret-password")); + assert!(user_password.contains("")); + + let token = format!("{:?}", NatsAuth::Token("super-secret-token".into())); + assert!(!token.contains("super-secret-token")); + assert!(token.contains("")); + } + + #[test] + fn debug_shows_non_secret_variants() { + let creds = format!( + "{:?}", + NatsAuth::Credentials(PathBuf::from("/etc/nats/app.creds")) + ); + assert!(creds.contains("/etc/nats/app.creds")); + assert_eq!(format!("{:?}", NatsAuth::None), "None"); + } +} diff --git a/crates/walgit-nats/src/lib.rs b/crates/walgit-nats/src/lib.rs index f6cb6f13..80679848 100644 --- a/crates/walgit-nats/src/lib.rs +++ b/crates/walgit-nats/src/lib.rs @@ -2,6 +2,7 @@ //! the cursor in the stream, wake-ups over NATS and a `$SRV` service API instead of HTTP. //! The event model is shared with the webhook bridge through `walgit-events`. +pub mod auth; pub mod message; pub mod publisher; pub mod service; @@ -17,24 +18,24 @@ use anyhow::Context; use async_nats::jetstream::{self, stream}; use walgit_config::{Config, NatsEventsConfig}; +use crate::auth::NatsAuth; use crate::publisher::Publisher; use crate::subject::SubjectPrefix; -/// Connect as `[events.nats]` says. The credentials path is read, never logged. -pub async fn connect(cfg: &NatsEventsConfig) -> anyhow::Result { +/// Connect to `[events.nats] urls` with `auth`. Only the method is logged, never the secret. +pub async fn connect(cfg: &NatsEventsConfig, auth: &NatsAuth) -> anyhow::Result { let addrs = cfg .urls .iter() .map(|u| async_nats::ServerAddr::from_str(u).with_context(|| format!("events.nats.urls: {u}"))) .collect::>>()?; - let mut options = async_nats::ConnectOptions::new().name("walgit"); - if let Some(path) = &cfg.credentials_file { - options = options - .credentials_file(path) - .await - .context("events.nats.credentials_file")?; - } - options.connect(addrs).await.context("connecting to NATS") + tracing::info!(urls = ?cfg.urls, auth = auth.description(), "connecting to NATS"); + auth.connect_options() + .await? + .name("walgit") + .connect(addrs) + .await + .with_context(|| format!("connecting to NATS (auth: {})", auth.description())) } /// The stream holding the events: created when missing (file storage, `.ref.>`, @@ -97,7 +98,7 @@ pub async fn run(cfg: Arc) -> anyhow::Result<()> { let store = walgit_store::open_store(&cfg).await?; let registry = walgit_wal::Registry::new(store, Arc::new(publisher_config(&cfg))); - let client = crate::connect(nats).await?; + let client = crate::connect(nats, &NatsAuth::from_process_env()).await?; let js = async_nats::jetstream::new(client.clone()); let stream = crate::open_stream(&js, nats, &prefix).await?; let publisher = Publisher::new(registry, js, stream, prefix.clone(), nats.start); diff --git a/crates/walgit-nats/src/testing.rs b/crates/walgit-nats/src/testing.rs index 7d66fe56..9d039456 100644 --- a/crates/walgit-nats/src/testing.rs +++ b/crates/walgit-nats/src/testing.rs @@ -7,16 +7,18 @@ use std::time::{Duration, Instant}; pub struct NatsServer { child: Child, port: u16, + args: Vec, pub url: String, store: tempfile::TempDir, } impl NatsServer { - fn spawn(port: u16, store: &std::path::Path) -> Child { + fn spawn(port: u16, store: &std::path::Path, args: &[String]) -> Child { #[allow(clippy::expect_used)] Command::new("nats-server") .args(["-js", "-a", "127.0.0.1", "-p", &port.to_string(), "-sd"]) .arg(store) + .args(args) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() @@ -35,14 +37,17 @@ impl NatsServer { /// When the server does not accept connections within 15 s. pub async fn restart(&mut self) { self.stop(); - self.child = Self::spawn(self.port, self.store.path()); - wait_ready(&self.url).await; + self.child = Self::spawn(self.port, self.store.path(), &self.args); + wait_ready(self.port).await; } } -async fn wait_ready(url: &str) { +async fn wait_ready(port: u16) { let t0 = Instant::now(); - while async_nats::connect(url).await.is_err() { + while tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_err() + { assert!( t0.elapsed() < Duration::from_secs(15), "nats-server did not start" @@ -65,6 +70,15 @@ impl Drop for NatsServer { /// When the server cannot be spawned or does not accept connections within 15 s. #[allow(clippy::expect_used)] pub async fn start_nats() -> Option { + start_nats_with(&[]).await +} + +/// [`start_nats`] with extra `nats-server` arguments (`--auth`, `--user`/`--pass`, ...). +/// +/// # Panics +/// When the server cannot be spawned or does not accept connections within 15 s. +#[allow(clippy::expect_used)] +pub async fn start_nats_with(args: &[&str]) -> Option { if !Command::new("nats-server") .arg("--version") .stdout(Stdio::null()) @@ -81,13 +95,15 @@ pub async fn start_nats() -> Option { .expect("local addr") .port(); let store = tempfile::tempdir().expect("tempdir"); - let child = NatsServer::spawn(port, store.path()); + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let child = NatsServer::spawn(port, store.path(), &args); let server = NatsServer { child, port, + args, url: format!("nats://127.0.0.1:{port}"), store, }; - wait_ready(&server.url).await; + wait_ready(port).await; Some(server) } diff --git a/crates/walgit-nats/tests/auth.rs b/crates/walgit-nats/tests/auth.rs new file mode 100644 index 00000000..a2d5a746 --- /dev/null +++ b/crates/walgit-nats/tests/auth.rs @@ -0,0 +1,125 @@ +//! `walgit nats` authenticates with whatever the `NATS_*` variables resolve to, against a real +//! `nats-server` that requires it. + +use std::time::Duration; + +use walgit_config::NatsEventsConfig; +use walgit_nats::auth::NatsAuth; +use walgit_nats::testing::{NatsServer, start_nats_with}; + +fn cfg(nats: &NatsServer) -> NatsEventsConfig { + NatsEventsConfig { + urls: vec![nats.url.clone()], + ..NatsEventsConfig::default() + } +} + +fn env(vars: &[(&str, &str)]) -> NatsAuth { + let vars: Vec<(String, String)> = vars + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + NatsAuth::from_env(|name| vars.iter().find(|(k, _)| k == name).map(|(_, v)| v.clone())) +} + +async fn connects(nats: &NatsServer, auth: &NatsAuth) -> bool { + let attempt = tokio::time::timeout(Duration::from_secs(5), async { + let client = walgit_nats::connect(&cfg(nats), auth).await?; + client.flush().await?; + anyhow::Ok(()) + }); + matches!(attempt.await, Ok(Ok(()))) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_token_from_the_environment_authenticates() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + assert!(connects(&nats, &env(&[("NATS_TOKEN", "s3cret-token")])).await); + assert!(!connects(&nats, &env(&[("NATS_TOKEN", "wrong")])).await); + assert!(!connects(&nats, &env(&[])).await); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_user_and_password_from_the_environment_authenticate() { + let Some(nats) = start_nats_with(&["--user", "walgit", "--pass", "s3cret-pass"]).await else { + return; + }; + assert!( + connects( + &nats, + &env(&[("NATS_USER", "walgit"), ("NATS_PASSWORD", "s3cret-pass")]) + ) + .await + ); + assert!( + !connects( + &nats, + &env(&[("NATS_USER", "walgit"), ("NATS_PASSWORD", "wrong")]) + ) + .await + ); + assert!( + !connects(&nats, &env(&[("NATS_USER", "walgit")])).await, + "a user alone is no auth" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_nkey_seed_from_the_environment_authenticates() { + let user = nkeys::KeyPair::new_user(); + let other = nkeys::KeyPair::new_user(); + let dir = tempfile::tempdir().unwrap(); + let conf = dir.path().join("nats.conf"); + std::fs::write( + &conf, + format!( + "authorization {{ users = [ {{ nkey: {} }} ] }}\n", + user.public_key() + ), + ) + .unwrap(); + let Some(nats) = start_nats_with(&["-c", conf.to_str().unwrap()]).await else { + return; + }; + assert!(connects(&nats, &env(&[("NATS_NKEY", &user.seed().unwrap())])).await); + assert!(!connects(&nats, &env(&[("NATS_NKEY", &other.seed().unwrap())])).await); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_credentials_file_wins_and_a_missing_one_fails_naming_the_variable() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + let auth = env(&[ + ("NATS_CREDS", "/nonexistent/walgit.creds"), + ("NATS_TOKEN", "s3cret-token"), + ]); + assert!(matches!(auth, NatsAuth::Credentials(_))); + let err = walgit_nats::connect(&cfg(&nats), &auth).await.unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("NATS_CREDS") && msg.contains("/nonexistent/walgit.creds"), + "{msg}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_connection_names_the_method_but_never_the_secret() { + let Some(nats) = start_nats_with(&["--auth", "s3cret-token"]).await else { + return; + }; + let auth = env(&[("NATS_TOKEN", "leaked-if-printed")]); + let err = tokio::time::timeout( + Duration::from_secs(5), + walgit_nats::connect(&cfg(&nats), &auth), + ) + .await + .expect("a rejected credential fails fast") + .unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("auth: token"), "{msg}"); + assert!(!msg.contains("leaked-if-printed"), "{msg}"); + assert!(!format!("{auth:?}").contains("leaked-if-printed")); +} diff --git a/crates/walgit-server/tests/events_nats.rs b/crates/walgit-server/tests/events_nats.rs index b6b6ede1..bea21b5a 100644 --- a/crates/walgit-server/tests/events_nats.rs +++ b/crates/walgit-server/tests/events_nats.rs @@ -52,7 +52,7 @@ impl Rig { urls: vec![nats.url.clone()], ..NatsEventsConfig::default() }; - let client = walgit_nats::connect(&nats_cfg).await?; + let client = walgit_nats::connect(&nats_cfg, &walgit_nats::auth::NatsAuth::None).await?; let js = jetstream::new(client.clone()); let prefix = SubjectPrefix::new(&nats_cfg.subject_prefix)?; Ok(Rig { diff --git a/walgit.example.toml b/walgit.example.toml index 471b8633..7f174580 100644 --- a/walgit.example.toml +++ b/walgit.example.toml @@ -205,7 +205,8 @@ lock_wait_warn = "1s" # WARN "lock wait" (lock, repo, wait_ms, reque [events.nats] # docs/EVENTS.md: read only by `walgit nats --config walgit.toml` # urls = ["nats://127.0.0.1:4222"] # the client connects to the first reachable server -# credentials_file = "/etc/walgit/nats.creds" # .creds or NKey seed file; unset = no credentials; never logged +# auth comes from the environment, first match wins: NATS_CREDS (.creds file), NATS_NKEY (seed), +# NATS_USER + NATS_PASSWORD, NATS_TOKEN; none set = no auth; secrets are never logged # stream = "WALGIT_EVENTS" # the JetStream stream holding the events (subjects ".ref.>"); it is the resume cursor # ensure_stream = true # create the stream when missing (file storage); an existing stream is used as is # duplicate_window = "1h" # dedup window (Nats-Msg-Id) of a stream `walgit nats` creates From 78396138f1b2fa1feaa21b34a155ddf6eb0b7c80 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Mon, 26 Oct 2026 22:04:17 -0400 Subject: [PATCH 8/8] fix(events): count a folded gap once until the subject moves Signed-off-by: Yordis Prieto --- crates/walgit-nats/src/publisher.rs | 9 ++++++- crates/walgit-nats/tests/jetstream.rs | 34 +++++++++++++++++++++++++++ docs/EVENTS.md | 1 + 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/walgit-nats/src/publisher.rs b/crates/walgit-nats/src/publisher.rs index 7ff8cae6..51814c93 100644 --- a/crates/walgit-nats/src/publisher.rs +++ b/crates/walgit-nats/src/publisher.rs @@ -85,6 +85,10 @@ pub struct Publisher { /// The head a cold repository was first seen at (`start = "head"`), and for a recreated one, /// which old incarnation it was seen behind. cold_marks: std::sync::Mutex>, + /// The stream sequence of the message each repository's gap was counted behind: the stream + /// keeps no cursor past it, so a head moved only by entries without ref events would otherwise + /// count the same gap again. + gaps_seen: std::sync::Mutex>, pub counters: Counters, } @@ -118,6 +122,7 @@ impl Publisher { serial: tokio::sync::Mutex::new(()), done: std::sync::Mutex::default(), cold_marks: std::sync::Mutex::default(), + gaps_seen: std::sync::Mutex::default(), counters: Counters::default(), }) } @@ -204,7 +209,9 @@ impl Publisher { // The newest published entry is already folded away: what follows it up to the // log window is lost to us. let published = l.id.seq(); - if published < readable_from { + let first_sighting = published < readable_from + && locked(&self.gaps_seen).insert(id.clone(), l.stream_seq) != Some(l.stream_seq); + if first_sighting { report.gap = Some([published + 1, readable_from]); self.counters .gap_entries diff --git a/crates/walgit-nats/tests/jetstream.rs b/crates/walgit-nats/tests/jetstream.rs index 3ee940d5..e1327212 100644 --- a/crates/walgit-nats/tests/jetstream.rs +++ b/crates/walgit-nats/tests/jetstream.rs @@ -674,6 +674,35 @@ async fn entries_folded_before_publishing_are_a_counted_gap() { assert_eq!(rig.ids().await, [msg_id(&id, 1, "refs/heads/main"), msg_id(&id, 5, "refs/heads/main")]); } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_gap_is_counted_once_while_nothing_moves_the_subject() { + let Some(nats) = start_nats().await else { return }; + let rig = Rig::new(&nats, NatsStart::Oldest).await; + let id = RepoId::new("t", "r").unwrap(); + rig.registry.create(&id, ObjectFormat::Sha1).await.unwrap(); + rig.push(&id, &[("refs/heads/main", ZERO, A)]).await; + let publisher = rig.publisher(NatsStart::Oldest).await; + assert_eq!(publisher.catch_up(&id).await.unwrap().emitted, 1); + + rig.push(&id, &[("refs/heads/main", A, B)]).await; + rig.push(&id, &[("refs/heads/main", B, C)]).await; + let handle = rig.registry.open(&id).await.unwrap(); + handle.write_checkpoint().await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.gap, r.emitted), (Some([2, 3]), 0), "{r:?}"); + + handle.publish_settings("", "test", "moves the head without a ref event").await.unwrap(); + let r = publisher.catch_up(&id).await.unwrap(); + assert!(r.head_seq > 3, "{r:?}"); + assert_eq!((r.gap, r.emitted), (None, 0), "the same gap is not reported again: {r:?}"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 2); + + rig.push(&id, &[("refs/heads/main", C, A)]).await; + let r = publisher.catch_up(&id).await.unwrap(); + assert_eq!((r.gap, r.emitted), (None, 1), "{r:?}"); + assert_eq!(publisher.counters.snapshot()["gap_entries"], 2); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn an_outage_mid_catch_up_loses_and_duplicates_nothing() { const REFS: usize = 1500; @@ -933,6 +962,11 @@ async fn the_periodic_sweep_finds_a_push_nobody_notified() { // No notification for this push: only the periodic sweep can find it. rig.push(&id, &[("refs/heads/main", A, B)]).await; rig.wait_for_messages(2).await; + // The sweep counts what it found once its catch-up returns, after the message is acknowledged. + let t0 = Instant::now(); + while publisher.counters.snapshot()["sweep_found"] == 0 && t0.elapsed() < Duration::from_secs(10) { + tokio::time::sleep(Duration::from_millis(20)).await; + } assert_eq!(publisher.counters.snapshot()["sweep_found"], 1); sweeping.abort(); } diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 89699022..311d71a7 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -189,6 +189,7 @@ Each behaviour above has a test against a real `nats-server -js` (`crates/walgit | Concurrent publishers leave each event once | `a_second_publisher_that_moved_first_stops_this_one`, `racing_publishers_leave_each_event_exactly_once` | | Losing the stream republishes the WAL window | `a_wiped_stream_is_republished_on_the_next_catch_up`, `a_purged_stream_heals_once_the_dedup_window_has_passed` | | A log gap is counted, readable entries still published | `entries_folded_before_publishing_are_a_counted_gap` | +| A gap is counted once while nothing moves the subject | `a_gap_is_counted_once_while_nothing_moves_the_subject` | | Bucket notifications wake it; junk is ignored | `bucket_notifications_wake_the_publisher_without_a_catch_up_call`, `notifications_that_name_nothing_are_ignored_and_the_loop_survives` | | The sweep finds what nobody notified | `the_periodic_sweep_finds_a_push_nobody_notified` | | Service API: catch-up, errors, stats | `notifications_and_the_service_api_wake_the_publisher`, `the_catch_up_endpoint_reports_a_real_repository`, `the_service_answers_bad_requests_and_reports_the_counters` |