diff --git a/changelog.d/branch-list-details.added.md b/changelog.d/branch-list-details.added.md new file mode 100644 index 000000000..844042dca --- /dev/null +++ b/changelog.d/branch-list-details.added.md @@ -0,0 +1,3 @@ +- `branch list --detailed`, `GET /branches?detailed=true` and MCP `branches_list` with `detailed` report each branch's head, where it started, and when it was created and last moved. See [Branches][branch-list-details]. + +[branch-list-details]: ../docs/user/branching/index.md#branch-workflow diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs index ce76f0826..aa9d51ae1 100644 --- a/crates/omnigraph-api-types/src/lib.rs +++ b/crates/omnigraph-api-types/src/lib.rs @@ -420,9 +420,58 @@ pub struct BranchCreateOutput { pub from_commit_id: String, } +/// Query parameters for listing branches. +#[derive(Debug, Clone, Default, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct BranchListQuery { + /// `true` adds `details`: each branch's head, the branch and commit it + /// started from, when it was created, and when it last moved. Defaults + /// to `false`. Reads each listed branch's latest version, and the version + /// it was created from once it has written a commit, never its commit + /// history; the names alone read neither. + #[serde(default)] + pub detailed: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct BranchListOutput { + /// Every live branch, sorted by name in byte order. pub branches: Vec, + /// Present only with `detailed=true`: one entry per branch the caller + /// may `read`, in the order of `branches`. A branch the caller may not + /// read is listed in `branches` alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub details: Option>, +} + +/// One live branch as `GET /branches?detailed=true` reports it. `from` and +/// `from_commit_id` say where the branch started, as its `POST /branches` +/// receipt did. A branch deleted and created again under the same name is a +/// new lifetime: its start and creation time are the later creation's. The +/// optional fields are absent on `main`. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchDetailOutput { + pub name: String, + /// The commit a read of the branch sees: its newest commit, or the commit + /// it started at while it has written none. + pub head: CommitOutput, + /// The branch this one started from, as that branch was named then; it + /// may since have been deleted or recreated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from: Option, + /// The commit the branch started at. A later merge does not move it; + /// `GET /branches/merge-base` answers where two branches last met. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_commit_id: Option, + /// When the branch was created, as Unix epoch microseconds recorded to + /// the second. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schema(example = 1714000000000000i64)] + pub created_at: Option, + /// When the head last moved, as Unix epoch microseconds: the time of the + /// branch's newest commit, or `created_at` while it has written none. + #[schema(example = 1714000000000000i64)] + pub updated_at: i64, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] @@ -3125,6 +3174,30 @@ pub fn diff_output(page: &omnigraph::changes::GraphDiffPage) -> DiffOutput { } } +/// A detailed branch listing (`detailed=true`): names and details, each in +/// byte order. +pub fn branch_list_output(listing: &omnigraph::db::BranchListing) -> BranchListOutput { + let mut branches = listing.branches.clone(); + branches.sort(); + let mut details: Vec<_> = listing.details.iter().map(branch_detail_output).collect(); + details.sort_by(|left, right| left.name.cmp(&right.name)); + BranchListOutput { + branches, + details: Some(details), + } +} + +pub fn branch_detail_output(detail: &omnigraph::db::BranchDetail) -> BranchDetailOutput { + BranchDetailOutput { + name: detail.name.clone(), + head: commit_output(&detail.head), + from: detail.from.clone(), + from_commit_id: detail.from_commit_id.clone(), + created_at: detail.created_at, + updated_at: detail.updated_at, + } +} + pub fn merge_base_output( source: &str, target: &str, diff --git a/crates/omnigraph-catalog/src/branch_detail.rs b/crates/omnigraph-catalog/src/branch_detail.rs new file mode 100644 index 000000000..6176ed8bd --- /dev/null +++ b/crates/omnigraph-catalog/src/branch_detail.rs @@ -0,0 +1,197 @@ +//! What a detailed branch listing reports about each live graph branch: its +//! head, the branch and commit its current lifetime started from, and when +//! that lifetime was created. Every fact is read from the branch refs and each +//! branch's own `__manifest`; nothing is stored for the listing. + +use std::sync::Arc; + +use futures::StreamExt; +use lance::Dataset; +use lance::dataset::refs::BranchContents; + +use crate::branch_control::list_live_manifest_branch_contents; +use crate::error::{OmniError, Result}; +use crate::history::ExtentCache; +use crate::layout::open_manifest_dataset_with_session; +use crate::state::{GraphLineageRow, exact_head, read_manifest_rows_projected}; + +/// One live named graph branch: its logical name and the native ref, with +/// that ref's contents, of its current lifetime. +pub(crate) struct LiveBranch { + pub(crate) name: String, + pub(crate) native: String, + pub(crate) contents: BranchContents, +} + +/// The live named graph branches of a `__manifest`, sorted by name: each live +/// native ref mapped to its one logical name, `main` excluded. A name with +/// more than one live incarnation fails loudly. The one enumeration both +/// branch listings use, so names and details agree on what is live. +pub(crate) async fn live_named_branches(dataset: &Dataset) -> Result> { + let mut named: Vec<_> = list_live_manifest_branch_contents(dataset) + .await? + .into_iter() + .filter(|(native, _)| native != "main") + .map(|(native, contents)| LiveBranch { + name: crate::branch_names::logical_branch_name(&native).to_string(), + native, + contents, + }) + .collect(); + named.sort_by(|left, right| left.name.cmp(&right.name)); + if let Some(pair) = named.windows(2).find(|pair| pair[0].name == pair[1].name) { + return Err(OmniError::manifest_conflict(format!( + "branch '{}' has more than one live native incarnation; run cleanup before using it", + pair[0].name + ))); + } + Ok(named) +} + +/// How many branches a listing reads at once. +const BRANCH_DETAIL_READ_CONCURRENCY: usize = 8; + +/// One listing of the live graph branches. +#[derive(Debug, Clone)] +pub struct GraphBranchListing { + /// Every live branch, `main` first and the rest by name. + pub branches: Vec, + /// One entry per branch the caller admitted, in the order of `branches`. + pub details: Vec, +} + +/// One live graph branch as a detailed listing reports it. +#[derive(Debug, Clone)] +pub struct GraphBranchDetail { + pub name: String, + /// The head record of the branch's latest `__manifest` version: its newest + /// commit, or the commit it was created at while it has written none. + pub head: GraphLineageRow, + /// Whether the branch's current lifetime wrote `head`. + pub head_is_own: bool, + /// The logical name of the branch the current lifetime was created from, + /// as that branch was named then; it may since have been deleted or + /// recreated. `None` on main. + pub from: Option, + /// The head commit of the `__manifest` version the lifetime was created + /// from: the commit it started at. `None` on main. + pub from_commit_id: Option, + /// `BranchContents::create_at` of the lifetime's native ref, in Unix + /// seconds; `None` on main. + pub created_at_seconds: Option, +} + +/// Every live graph branch, with details of those `can_read` admits. One open +/// of main's `__manifest` and one listing of its refs; then, for each admitted +/// branch, a checkout of its latest version and one scan of its rows, and when +/// the branch has written a commit, the checkout and scan of the version it was +/// created from. A branch whose ref is gone before its own reads keeps its name +/// and has no details: it was deleted after the listing. +pub async fn list_graph_branches_detailed( + root_uri: &str, + control_session: &Arc, + can_read: &mut (dyn FnMut(&str) -> bool + Send + '_), +) -> Result { + let root = root_uri.trim_end_matches('/'); + let main = open_manifest_dataset_with_session(root, None, control_session).await?; + let named = live_named_branches(&main).await?; + let branches = std::iter::once("main".to_string()) + .chain(named.iter().map(|branch| branch.name.clone())) + .collect(); + let mut details = Vec::with_capacity(named.len() + 1); + if can_read("main") { + let head = read_manifest_rows_projected(&main).await?.head; + details.push(GraphBranchDetail { + name: "main".to_string(), + head_is_own: exact_head(&head, None).is_some(), + head, + from: None, + from_commit_id: None, + created_at_seconds: None, + }); + } + let admitted: Vec<_> = named + .into_iter() + .filter(|branch| can_read(&branch.name)) + .collect(); + let legacy = ExtentCache::default(); + let mut reads = futures::stream::iter(admitted) + .map(|branch| { + let (main, legacy) = (&main, &legacy); + async move { branch_detail(root, main, legacy, branch).await } + }) + .buffered(BRANCH_DETAIL_READ_CONCURRENCY); + while let Some(detail) = reads.next().await { + details.extend(detail?); + } + Ok(GraphBranchListing { branches, details }) +} + +async fn branch_detail( + root: &str, + main: &Dataset, + legacy: &ExtentCache, + branch: LiveBranch, +) -> Result> { + let LiveBranch { + name: logical, + native, + contents, + } = branch; + match read_branch(root, main, legacy, &native, &contents).await { + Ok((head, head_is_own, from_commit_id)) => { + let from = contents + .parent_branch + .as_deref() + .map(crate::branch_names::logical_branch_name) + .unwrap_or("main") + .to_string(); + Ok(Some(GraphBranchDetail { + name: logical, + from: Some(from), + from_commit_id: Some(from_commit_id), + head, + head_is_own, + created_at_seconds: Some(contents.create_at), + })) + } + // The listing saw this lifetime live, and a native name is never + // reused. A read that fails once its ref is gone reports a branch + // deleted since; any other failure is the read's own. + Err(error) => match crate::branch_control::get_branch_identifier(main, &native).await { + Err(lance::Error::RefNotFound { .. }) => Ok(None), + _ => Err(error), + }, + } +} + +/// The latest head of the lifetime `native`, whether the lifetime wrote it, +/// and the commit the lifetime started at: the head of the version it was +/// created from, which is still its head while it has written none. +async fn read_branch( + root: &str, + main: &Dataset, + legacy: &ExtentCache, + native: &str, + contents: &BranchContents, +) -> Result<(GraphLineageRow, bool, String)> { + let branch = main + .checkout_branch(native) + .await + .map_err(OmniError::storage)?; + let head = read_manifest_rows_projected(&branch).await?.head; + if exact_head(&head, Some(native)).is_none() { + let started_at = head.graph_commit_id.clone(); + return Ok((head, false, started_at)); + } + let created = branch + .checkout_version(contents.parent_version) + .await + .map_err(OmniError::storage)?; + let started_at = + match crate::legacy::at_version(root, &created.session(), legacy, &created).await? { + None => read_manifest_rows_projected(&created).await?.head, + Some(at) => at.served(Some(native), contents.parent_version)?.commit, + }; + Ok((head, true, started_at.graph_commit_id)) +} diff --git a/crates/omnigraph-catalog/src/lib.rs b/crates/omnigraph-catalog/src/lib.rs index 8523d4e80..66f6ffea9 100644 --- a/crates/omnigraph-catalog/src/lib.rs +++ b/crates/omnigraph-catalog/src/lib.rs @@ -14,6 +14,7 @@ pub(crate) use omnigraph_core::{ branch_control, branch_names, dataset_index, error, handle_cache, instrumentation, lance_access, lance_clone, manifest_request, metadata, seams, staging, storage, }; +pub mod branch_detail; pub mod commit_graph; mod commit; @@ -63,6 +64,7 @@ pub mod retention; pub mod state; pub mod unattached; pub mod unpublished; +pub use branch_detail::{GraphBranchDetail, GraphBranchListing, list_graph_branches_detailed}; pub use branch_names::is_merge_input_tag; pub use migrations::{ UNATTACHED_BRANCHES_STAMP, UNPUBLISHED_COMMITS_STAMP, require_unattached_branches, @@ -2668,23 +2670,10 @@ impl ManifestCoordinator { /// Logical graph branches, `main` first. Each live native ref maps to /// exactly one logical name; a duplicate incarnation fails loudly. pub async fn list_graph_branches(&self) -> Result> { - let branches = list_live_manifest_branch_contents(&self.dataset).await?; - let mut names = Vec::with_capacity(branches.len()); - let mut seen = HashSet::with_capacity(branches.len()); - for native in branches.keys().filter(|name| *name != "main") { - let logical = crate::branch_names::logical_branch_name(native).to_string(); - if !seen.insert(logical.clone()) { - return Err(OmniError::manifest_conflict(format!( - "branch '{logical}' has more than one live native incarnation; run cleanup \ - before using it" - ))); - } - names.push(logical); - } - names.sort(); - let mut all = vec!["main".to_string()]; - all.extend(names); - Ok(all) + let named = branch_detail::live_named_branches(&self.dataset).await?; + Ok(std::iter::once("main".to_string()) + .chain(named.into_iter().map(|branch| branch.name)) + .collect()) } } diff --git a/crates/omnigraph-cli/src/client.rs b/crates/omnigraph-cli/src/client.rs index 1548240c5..42e33f327 100644 --- a/crates/omnigraph-cli/src/client.rs +++ b/crates/omnigraph-cli/src/client.rs @@ -16,11 +16,11 @@ use omnigraph_api_types::{ ChangeOutput, ChangeRequest, CommitChangesOutput, CommitListOutput, CommitOutput, DiffOutput, EntityKind, ExportRequest, GraphBatchLoadOutput, GraphDiscoveryOutput, GraphListOutput, InvokeStoredQueryRequest, MergeBaseOutput, QueriesCatalogOutput, QueryRequest, ReadOutput, - SchemaApplyOutput, SchemaOutput, SettingsRequest, SnapshotOutput, branch_list_read_output, - branch_read_output, change_baseline_output, change_feed_output, change_scope, - commit_changes_output, commit_output, diff_output, graph_batch_load_receipt_output, - load_executed_output, merge_base_output, read_output, schema_apply_output, show_read_output, - snapshot_payload, + SchemaApplyOutput, SchemaOutput, SettingsRequest, SnapshotOutput, branch_list_output, + branch_list_read_output, branch_read_output, change_baseline_output, change_feed_output, + change_scope, commit_changes_output, commit_output, diff_output, + graph_batch_load_receipt_output, load_executed_output, merge_base_output, read_output, + schema_apply_output, show_read_output, snapshot_payload, }; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::query::ast::{BranchCreateFrom, BranchRead, BranchWrite, CommitWrite}; @@ -582,7 +582,41 @@ impl GraphClient { let session = Self::open_read_session(uri, binding, &[]).await?; let mut branches = session.branch_list().await?; branches.sort(); - Ok(BranchListOutput { branches }) + Ok(BranchListOutput { + branches, + details: None, + }) + } + GraphClient::Catalogue => Err(catalogue_scope_refusal("branch list")), + } + } + + /// [`Self::branch_list`] with each readable branch's details + /// (`GET /branches?detailed=true`). + pub(crate) async fn branch_list_detailed(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + response_limit, + .. + } => { + remote_json_bounded( + http, + Method::GET, + remote_url(base_url, &["branches"], &[("detailed", "true")])?, + None, + token.as_deref(), + None, + *response_limit, + ) + .await + } + GraphClient::Embedded { uri, binding, .. } => { + let session = Self::open_read_session(uri, binding, &[]).await?; + let listing = session.branch_list_detailed(&mut |_| true).await?; + Ok(branch_list_output(&listing)) } GraphClient::Catalogue => Err(catalogue_scope_refusal("branch list")), } diff --git a/crates/omnigraph-cli/src/commands/history/args.rs b/crates/omnigraph-cli/src/commands/history/args.rs index fffa6f453..0f7b640a6 100644 --- a/crates/omnigraph-cli/src/commands/history/args.rs +++ b/crates/omnigraph-cli/src/commands/history/args.rs @@ -57,6 +57,10 @@ pub(crate) enum BranchCommand { }, /// List branches List { + /// Also report each branch's head, the branch and commit it started + /// from, when it was created, and when it last moved + #[arg(long)] + detailed: bool, #[arg(long)] json: bool, }, diff --git a/crates/omnigraph-cli/src/commands/history/branch.rs b/crates/omnigraph-cli/src/commands/history/branch.rs index e21af3aae..ea32bc5d8 100644 --- a/crates/omnigraph-cli/src/commands/history/branch.rs +++ b/crates/omnigraph-cli/src/commands/history/branch.rs @@ -3,7 +3,7 @@ use super::prepare::{ChangeFilter, Pagination}; use super::{BranchCommand, HistoryScope, run_diff}; use crate::client::{self, GraphClient}; use crate::helpers::{confirm_destructive, resolve_branch}; -use crate::output::{write_json, write_merge_base_human}; +use crate::output::{write_branch_list_human, write_json, write_merge_base_human}; use color_eyre::eyre::Result; use omnigraph::db::{ReadTarget, SnapshotId}; use std::io::Write; @@ -45,17 +45,19 @@ pub(crate) async fn run_branch( )?; } } - BranchCommand::List { json } => { + BranchCommand::List { detailed, json } => { let client = GraphClient::from_target(scope.operator, scope.target, scope.graph, None, stderr) .await?; - let payload = client.branch_list().await?; + let payload = if detailed { + client.branch_list_detailed().await? + } else { + client.branch_list().await? + }; if json { write_json(&payload, stdout)?; } else { - for branch in payload.branches { - writeln!(stdout, "{}", branch)?; - } + write_branch_list_human(&payload, stdout)?; } } BranchCommand::Delete { name, json } => { diff --git a/crates/omnigraph-cli/src/output.rs b/crates/omnigraph-cli/src/output.rs index de5a97c3c..7156db9f4 100644 --- a/crates/omnigraph-cli/src/output.rs +++ b/crates/omnigraph-cli/src/output.rs @@ -3,7 +3,7 @@ use super::*; use crate::app::Completion; -use omnigraph_api_types::CommitListOutput; +use omnigraph_api_types::{BranchListOutput, CommitListOutput}; pub(crate) fn graph_type_subject(type_key: &str) -> String { if let Some(type_name) = type_key.strip_prefix("node:") { @@ -1039,6 +1039,32 @@ pub(crate) fn write_change_human(output: &ChangeOutput, writer: &mut dyn Write) Ok(()) } +/// One line per branch: its name, then with details its head, where it +/// started, and its creation and last-move times, under the keys the JSON +/// output uses. A branch without details (one the caller may not read) writes +/// its name alone. +pub(crate) fn write_branch_list_human( + list: &BranchListOutput, + writer: &mut dyn Write, +) -> Result<()> { + let details = list.details.as_deref().unwrap_or_default(); + for branch in &list.branches { + let Some(detail) = details.iter().find(|detail| &detail.name == branch) else { + writeln!(writer, "{branch}")?; + continue; + }; + let mut line = format!("{branch} head={}", detail.head.graph_commit_id); + if let (Some(from), Some(from_commit_id)) = (&detail.from, &detail.from_commit_id) { + line.push_str(&format!(" from={from} from_commit_id={from_commit_id}")); + } + if let Some(created_at) = detail.created_at { + line.push_str(&format!(" created_at={created_at}")); + } + writeln!(writer, "{line} updated_at={}", detail.updated_at)?; + } + Ok(()) +} + pub(crate) fn write_commit_list_human( list: &CommitListOutput, writer: &mut dyn Write, diff --git a/crates/omnigraph-cli/tests/integration/cli_data.rs b/crates/omnigraph-cli/tests/integration/cli_data.rs index af09b63f8..298285fb5 100644 --- a/crates/omnigraph-cli/tests/integration/cli_data.rs +++ b/crates/omnigraph-cli/tests/integration/cli_data.rs @@ -4339,6 +4339,37 @@ fn branch_list_outputs_sorted_branches() { .collect::>(); assert_eq!(lines, vec!["alpha", "main", "zeta"]); + + // `--detailed` keeps one line per branch, in the same order, and adds the + // head, where the branch started and the times. + let detailed = output_success( + cli() + .arg("branch") + .arg("list") + .arg("--detailed") + .arg("--target") + .arg(&graph), + ); + let stdout = stdout_string(&detailed); + let lines = stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::>(); + assert_eq!(lines.len(), 3, "{stdout}"); + let head = lines[1] + .strip_prefix("main head=") + .and_then(|rest| rest.split(' ').next()) + .unwrap_or_else(|| panic!("{stdout}")); + for (line, name) in [(lines[0], "alpha"), (lines[2], "zeta")] { + assert!( + line.starts_with(&format!( + "{name} head={head} from=main from_commit_id={head} created_at=" + )), + "{stdout}" + ); + assert!(line.contains(" updated_at="), "{stdout}"); + } } #[test] diff --git a/crates/omnigraph-cli/tests/integration/parity_matrix.rs b/crates/omnigraph-cli/tests/integration/parity_matrix.rs index c37283499..f48af9c86 100644 --- a/crates/omnigraph-cli/tests/integration/parity_matrix.rs +++ b/crates/omnigraph-cli/tests/integration/parity_matrix.rs @@ -321,6 +321,13 @@ fn parity_branch_list() { let p = parity(); let (l, r) = p.run(&["branch", "list", "--json"]); assert_parity("branch list", &l, &r); + let (l, r) = p.run(&["branch", "list", "--detailed", "--json"]); + assert_parity("branch list --detailed", &l, &r); + assert!( + parse_stdout_json(&l)["details"][0]["head"]["graph_commit_id"].is_string(), + "{}", + String::from_utf8_lossy(&l.stdout) + ); } #[test] diff --git a/crates/omnigraph-mcp/src/arguments.rs b/crates/omnigraph-mcp/src/arguments.rs index 1657cede8..3249dab89 100644 --- a/crates/omnigraph-mcp/src/arguments.rs +++ b/crates/omnigraph-mcp/src/arguments.rs @@ -13,6 +13,13 @@ pub struct GraphArguments { } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] +pub struct BranchesListArguments { + pub graph: String, + #[serde(default)] + pub detailed: bool, +} +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BranchArguments { pub graph: String, #[serde(default)] @@ -310,7 +317,7 @@ pub enum ToolCall { Read(ReadArguments), Snapshot(BranchArguments), Access(AccessArguments), - BranchesList(GraphArguments), + BranchesList(BranchesListArguments), CommitsList(CommitsListArguments), CommitsGet(CommitArguments), CommitsChanges(CommitChangesArguments), diff --git a/crates/omnigraph-mcp/src/catalog.rs b/crates/omnigraph-mcp/src/catalog.rs index 62daafaf3..3cf9da32f 100644 --- a/crates/omnigraph-mcp/src/catalog.rs +++ b/crates/omnigraph-mcp/src/catalog.rs @@ -125,8 +125,8 @@ const TOOLS: &[ToolSpec] = &[ ToolSpec { name: "branches_list", title: "List branches", - description: "List a graph's branch names.", - input: graph_schema, + description: "List a graph's branch names. With `detailed`, also each readable branch's head commit, the branch and commit it started from, and when it was created and last moved.", + input: branches_list_schema, output: Some(branches_output_schema), effect: Effect::Read, gate: Gate::Always, @@ -377,6 +377,13 @@ pub(super) fn graph_schema() -> Value { arguments(json!({}), &[]) } +pub(super) fn branches_list_schema() -> Value { + arguments( + json!({"detailed": {"type": "boolean", "description": "`true` adds `details`: for each branch the caller may read, its `head` commit, the branch (`from`) and commit (`from_commit_id`) it started from, and `created_at` and `updated_at`. Defaults to `false`."}}), + &[], + ) +} + pub(super) fn branch_schema() -> Value { arguments( json!({"branch": text_schema("Branch. Defaults to `main`.")}), @@ -674,7 +681,21 @@ pub(super) fn access_output_schema() -> Value { pub(super) fn branches_output_schema() -> Value { json!({ "type": "object", - "properties": {"branches": {"type": "array", "items": {"type": "string"}}}, + "properties": { + "branches": {"type": "array", "items": {"type": "string"}}, + "details": {"type": "array", "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "head": {"type": "object"}, + "from": {"type": "string"}, + "from_commit_id": {"type": "string"}, + "created_at": {"type": "integer"}, + "updated_at": {"type": "integer"}, + }, + "required": ["name", "head", "updated_at"], + }}, + }, "required": ["branches"], }) } diff --git a/crates/omnigraph-mcp/src/lib.rs b/crates/omnigraph-mcp/src/lib.rs index baf5af5a5..de3175975 100644 --- a/crates/omnigraph-mcp/src/lib.rs +++ b/crates/omnigraph-mcp/src/lib.rs @@ -13,10 +13,11 @@ mod settings; pub use arguments::{ AccessArguments, BlobClearArguments, BlobStatArguments, BranchArguments, BranchCreateArguments, - BranchDeleteArguments, BranchDiffArguments, BranchMergeArguments, ChangeFilters, - ChangesPollArguments, CommitArguments, CommitChangesArguments, CommitsListArguments, - DiffArguments, GraphArguments, InvalidArguments, LoadArguments, MergeBaseArguments, - MutateArguments, QueryArguments, ReadArguments, ToolCall, ToolRequest, change_query, + BranchDeleteArguments, BranchDiffArguments, BranchMergeArguments, BranchesListArguments, + ChangeFilters, ChangesPollArguments, CommitArguments, CommitChangesArguments, + CommitsListArguments, DiffArguments, GraphArguments, InvalidArguments, LoadArguments, + MergeBaseArguments, MutateArguments, QueryArguments, ReadArguments, ToolCall, ToolRequest, + change_query, }; pub use resources::{ResourceOutput, ResourceRequest}; pub use result::{HostError, bounded_result}; diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index 287024862..190097dc8 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -2800,8 +2800,10 @@ pub(crate) async fn server_load_ndjson( get, path = "/branches", tag = "branches", + params(api::BranchListQuery), responses( (status = 200, description = "List of branches", body = BranchListOutput), + (status = 400, description = "Invalid `detailed`", body = ErrorOutput), (status = 401, description = "Unauthorized", body = ErrorOutput), (status = 403, description = "Forbidden", body = ErrorOutput), (status = 500, description = "Internal failure (code `internal`) before a successful response", body = ErrorOutput), @@ -2812,12 +2814,54 @@ pub(crate) async fn server_load_ndjson( /// /// Returns branch names sorted by name in byte order. Read-only. The GQ statement /// `branch list` on `POST /query` runs the same body. +/// +/// `detailed=true` adds `details`: for each branch the caller may `read`, its +/// head commit, the branch and commit it started from, when it was created, +/// and when it last moved. Each listed branch costs a read of its latest +/// version, and of the version it was created from once it has written a +/// commit; no commit history is read. pub(crate) async fn server_branch_list( Extension(handle): Extension, actor: Option>, + ApiQuery(query): ApiQuery, ) -> std::result::Result, ApiError> { - let branches = branch_list_body(&handle, actor.as_ref().map(|Extension(actor)| actor)).await?; - Ok(Json(BranchListOutput { branches })) + let actor = actor.as_ref().map(|Extension(actor)| actor); + if !query.detailed { + let branches = branch_list_body(&handle, actor).await?; + return Ok(Json(BranchListOutput { + branches, + details: None, + })); + } + authorize_scope_free_read(&handle, actor)?; + let policy = handle.policy.as_deref(); + // A policy failure (a missing bearer, an evaluation error) is not a + // denial: it stops the listing, and its own error answers. + let mut failure = None; + let mut can_read = |branch: &str| { + let request = PolicyRequest { + action: PolicyAction::Read, + branch: Some(branch.to_string()), + target_branch: None, + }; + match authorize(actor, policy, request) { + Ok(Authz::Allowed) => true, + Ok(Authz::Denied(_)) => false, + Err(error) => { + failure.get_or_insert(error); + false + } + } + }; + let listing = handle + .engine + .branch_list_detailed(&mut can_read) + .await + .map_err(ApiError::from_omni)?; + if let Some(error) = failure { + return Err(error); + } + Ok(Json(api::branch_list_output(&listing))) } /// The scope-free `read` decision shared by `GET /branches`, `branch list`, diff --git a/crates/omnigraph-server/src/mcp.rs b/crates/omnigraph-server/src/mcp.rs index 090941c51..a17878472 100644 --- a/crates/omnigraph-server/src/mcp.rs +++ b/crates/omnigraph-server/src/mcp.rs @@ -460,8 +460,14 @@ impl ServerHost { } ToolCall::BranchesList(args) => { let (handle, actor) = self.graph(&args.graph, actor, &graph)?; - let output = - handlers::server_branch_list(Extension(handle), Some(Extension(actor))).await?; + let output = handlers::server_branch_list( + Extension(handle), + Some(Extension(actor)), + ApiQuery(crate::api::BranchListQuery { + detailed: args.detailed, + }), + ) + .await?; bounded_result(&output.0) } ToolCall::CommitsList(args) => { @@ -835,8 +841,12 @@ impl ServerHost { } ResourceRequest::Branches(name) => { let (handle, actor) = self.graph(&name, actor, &graph)?; - let output = - handlers::server_branch_list(Extension(handle), Some(Extension(actor))).await?; + let output = handlers::server_branch_list( + Extension(handle), + Some(Extension(actor)), + ApiQuery(Default::default()), + ) + .await?; Ok(("application/json", pretty(&output.0)?)) } } diff --git a/crates/omnigraph-server/tests/integration/auth_policy.rs b/crates/omnigraph-server/tests/integration/auth_policy.rs index 103273158..1fd05bbb3 100644 --- a/crates/omnigraph-server/tests/integration/auth_policy.rs +++ b/crates/omnigraph-server/tests/integration/auth_policy.rs @@ -3083,12 +3083,14 @@ async fn branch_list_is_denied_by_a_branch_scoped_read_rule_as_the_route_is() { &format!("{statement} under a protected-only read rule"), ); } - let (status, body) = json_response(&app, get_request(&g("/branches"), "team-token")).await; - assert_forbidden( - status, - body, - "GET /branches under a protected-only read rule", - ); + for route in ["/branches", "/branches?detailed=true"] { + let (status, body) = json_response(&app, get_request(&g(route), "team-token")).await; + assert_forbidden( + status, + body, + &format!("GET {route} under a protected-only read rule"), + ); + } } const SHOW_SOURCES: [&str; 1] = ["show all;"]; diff --git a/crates/omnigraph-server/tests/integration/data_routes.rs b/crates/omnigraph-server/tests/integration/data_routes.rs index 54aaf216a..674195c01 100644 --- a/crates/omnigraph-server/tests/integration/data_routes.rs +++ b/crates/omnigraph-server/tests/integration/data_routes.rs @@ -5701,6 +5701,31 @@ async fn remote_branch_list_create_merge_flow_works() { assert_eq!(change_body["branch"], "feature"); assert_eq!(change_body["affected_nodes"], 1); + // `detailed=true` adds what `GET /commits?branch=` would otherwise be read + // for, per branch, and says where each branch started as its create + // receipt did; the plain listing stays names only. + assert!(list_body.get("details").is_none(), "{list_body}"); + let (status, detailed) = get_json(&app, g("/branches?detailed=true")).await; + assert_eq!(status, StatusCode::OK, "{detailed}"); + assert_eq!(detailed["branches"], json!(["feature", "main"])); + let (_, main_commits) = get_json(&app, g("/commits?branch=main")).await; + let main_head = &main_commits["commits"][0]; + let main = &detailed["details"][1]; + assert_eq!(main["name"], "main"); + assert_eq!(&main["head"], main_head); + for absent in ["from", "from_commit_id", "created_at"] { + assert!(main.get(absent).is_none(), "{main}"); + } + assert_eq!(main["updated_at"], main_head["created_at"]); + let feature = &detailed["details"][0]; + assert_eq!(feature["name"], "feature"); + assert_eq!(feature["head"], change_body["commit"]); + assert_eq!(feature["updated_at"], change_body["commit"]["created_at"]); + assert_eq!(feature["from"], create_body["from"]); + assert_eq!(feature["from_commit_id"], create_body["from_commit_id"]); + assert_eq!(feature["from_commit_id"], main_head["graph_commit_id"]); + assert!(feature["created_at"].as_i64().unwrap() > 0, "{feature}"); + let read_main_before = QueryRequest { query: fs::read_to_string(fixture("test.gq")).unwrap(), name: Some("get_person".to_string()), diff --git a/crates/omnigraph-server/tests/integration/mcp.rs b/crates/omnigraph-server/tests/integration/mcp.rs index 02aa7b673..43a257c4c 100644 --- a/crates/omnigraph-server/tests/integration/mcp.rs +++ b/crates/omnigraph-server/tests/integration/mcp.rs @@ -1098,6 +1098,11 @@ async fn mcp_read_parity_tools_equal_their_routes() { json!({"graph":"default"}), "/graphs/default/branches".to_string(), ), + ( + "branches_list", + json!({"graph":"default","detailed":true}), + "/graphs/default/branches?detailed=true".to_string(), + ), ( "access", json!({"graph":"default"}), diff --git a/crates/omnigraph-server/tests/integration/openapi.rs b/crates/omnigraph-server/tests/integration/openapi.rs index d6186fbd5..c4539f93e 100644 --- a/crates/omnigraph-server/tests/integration/openapi.rs +++ b/crates/omnigraph-server/tests/integration/openapi.rs @@ -1047,6 +1047,7 @@ const EXPECTED_SCHEMAS: &[&str] = &[ "BranchCreateOutput", "BranchCreateRequest", "BranchDeleteOutput", + "BranchDetailOutput", "BranchListOutput", "BranchMergeOutcome", "BranchMergeOutput", diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 8217af43a..cabc1fa81 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -16,12 +16,12 @@ pub use commit_graph::{GraphCommit, GraphCommitKind}; pub use graph_coordinator::{ReadTarget, ResolvedTarget, SnapshotId}; pub use manifest::{DatasetEntry, DatasetUpdate}; pub use omnigraph::{ - BranchCreated, BuiltIndex, BuiltIndexAction, BuiltIndexKind, COMMIT_LIST_DEFAULT_PAGE_COMMITS, - COMMIT_LIST_MAX_PAGE_COMMITS, CleanupPolicyOptions, CollectorCost, CollectorPathSnapshot, - CollectorReport, CollectorRowSummary, CommitListPage, DatasetCleanupStats, - DatasetOptimizeStats, DatasetRepairStats, EXPORT_CHUNK_MAX_BYTES, ExportCut, - FullTextIndexRebuildResult, GraphCreateReconciliation, IndexBuildResult, InitOptions, - MergeOutcome, MergeResult, Omnigraph, OpenMode, PendingIndex, PreparedGraphCreate, + BranchCreated, BranchDetail, BranchListing, BuiltIndex, BuiltIndexAction, BuiltIndexKind, + COMMIT_LIST_DEFAULT_PAGE_COMMITS, COMMIT_LIST_MAX_PAGE_COMMITS, CleanupPolicyOptions, + CollectorCost, CollectorPathSnapshot, CollectorReport, CollectorRowSummary, CommitListPage, + DatasetCleanupStats, DatasetOptimizeStats, DatasetRepairStats, EXPORT_CHUNK_MAX_BYTES, + ExportCut, FullTextIndexRebuildResult, GraphCreateReconciliation, IndexBuildResult, + InitOptions, MergeOutcome, MergeResult, Omnigraph, OpenMode, PendingIndex, PreparedGraphCreate, PreparedSchemaApply, PreparedSchemaSettlement, ReadCheck, RebuiltFullTextIndex, RepairAction, RepairClassification, RepairOptions, RepairStats, RetainedManifestVersions, SYSTEM_COLUMNS_PREFLIGHT, SchemaApplyReconciliation, SchemaApplyResult, SchemaApplySettlement, diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 195e49a2a..895eb3966 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -4102,6 +4102,35 @@ impl Omnigraph { self.coordinator.read().await.branch_list().await } + /// [`Self::branch_list`] with the details of each branch `can_read` + /// admits: its head, the branch and commit its current lifetime started + /// from, and when that lifetime was created. One listing of the branch + /// refs; `can_read` is asked once per branch, before any read of that + /// branch. Each admitted branch costs a read of its + /// latest `__manifest` version and, when it has written a commit, of the + /// version it was created from; never a read of its lineage. A branch + /// deleted after the listing keeps its name and has no details. + pub async fn branch_list_detailed( + &self, + can_read: &mut ReadCheck<'_>, + ) -> Result { + self.ensure_schema_state_valid().await?; + let listing = crate::db::manifest::list_graph_branches_detailed( + self.uri(), + &self.control_session(), + can_read, + ) + .await?; + Ok(BranchListing { + branches: listing.branches, + details: listing + .details + .into_iter() + .map(BranchDetail::from_catalog) + .collect(), + }) + } + pub async fn branch_delete(&self, name: &str) -> Result<()> { self.branch_delete_as(name, None).await } @@ -5231,9 +5260,68 @@ pub(crate) fn schema_for_table_key(catalog: &Catalog, table_key: &str) -> Result /// A caller's answer to "may it read this branch?", asked once per two-point /// graph diff side and, on a continuation, per commit the page token names -/// that its side no longer resolves to; see [`Omnigraph::diff_page_readable`]. +/// that its side no longer resolves to ([`Omnigraph::diff_page_readable`]), +/// and once per listed branch ([`Omnigraph::branch_list_detailed`]). pub type ReadCheck<'a> = dyn FnMut(&str) -> bool + Send + 'a; +/// The live graph branches, as [`Omnigraph::branch_list_detailed`] lists them. +#[derive(Debug, Clone)] +pub struct BranchListing { + /// Every live branch, `main` first and the rest by name. + pub branches: Vec, + /// One entry per branch the caller may read, in the order of `branches`. + pub details: Vec, +} + +/// One live graph branch, as [`Omnigraph::branch_list_detailed`] reports it. +/// `from` and `from_commit_id` name where the branch started, as its create +/// receipt ([`BranchCreated`]) did. A branch deleted and created again under +/// the same name is a new lifetime: its start and creation time are the new +/// lifetime's. +#[derive(Debug, Clone)] +pub struct BranchDetail { + pub name: String, + /// The commit a read of the branch sees: its newest commit, or the commit + /// it started at while it has written none. + pub head: GraphCommit, + /// The branch this one started from, as that branch was named then; it + /// may since have been deleted or recreated. `None` on `main`. + pub from: Option, + /// The commit the branch started at. Fixed for the branch's lifetime: + /// unlike the merge base, a later merge does not move it. `None` on + /// `main`. + pub from_commit_id: Option, + /// When the branch was created, in Unix microseconds recorded to the + /// second; `None` on `main`. + pub created_at: Option, + /// When the head last moved, in Unix microseconds: the time of the + /// branch's newest commit, or `created_at` while it has written none. + pub updated_at: i64, +} + +impl BranchDetail { + fn from_catalog(detail: crate::db::manifest::GraphBranchDetail) -> Self { + let head = crate::db::commit_graph::graph_commit_from_manifest_row(detail.head); + let created_at = detail.created_at_seconds.map(|seconds| { + i64::try_from(seconds) + .unwrap_or(i64::MAX) + .saturating_mul(1_000_000) + }); + let updated_at = match created_at { + Some(created_at) if !detail.head_is_own => created_at, + _ => head.created_at, + }; + Self { + name: detail.name, + head, + from: detail.from, + from_commit_id: detail.from_commit_id, + created_at, + updated_at, + } + } +} + /// How a diff continuation reached a commit its token names, which is how /// it opens the commit. enum DiffReach { diff --git a/crates/omnigraph/src/db/upgrade/tests.rs b/crates/omnigraph/src/db/upgrade/tests.rs index 39ab4cb9d..feafb51ec 100644 --- a/crates/omnigraph/src/db/upgrade/tests.rs +++ b/crates/omnigraph/src/db/upgrade/tests.rs @@ -521,6 +521,38 @@ async fn assert_converted(source: &Source) { let dataset = open(root, Some(branch)).await.unwrap(); assert_eq!(dataset.version().version, version, "{branch}"); } + // A detailed listing serves each converted branch's start from the legacy + // history: `feature` wrote its own head, so its start is read at the + // pre-upgrade version it was created from; `child` was forked from the + // retired `gone`. + let listing = crate::db::manifest::list_graph_branches_detailed( + root, + &crate::lance_access::control_session(), + &mut |_| true, + ) + .await + .unwrap(); + let details: Vec<_> = listing + .details + .iter() + .map(|detail| { + ( + detail.name.as_str(), + detail.from.as_deref(), + detail.from_commit_id.clone(), + detail.head.graph_commit_id.clone(), + ) + }) + .collect(); + assert_eq!( + details, + [ + ("main", None, None, id(4)), + ("child", Some("gone"), Some(id(3)), id(3)), + ("feature", Some("main"), Some(id(2)), id(6)), + ("fresh", Some("feature"), Some(id(5)), id(5)), + ] + ); assert_eq!( root_objects(source.dir.path()), root_objects_of(source), diff --git a/crates/omnigraph/tests/bound_storage.rs b/crates/omnigraph/tests/bound_storage.rs index 4641a513d..d601cad67 100644 --- a/crates/omnigraph/tests/bound_storage.rs +++ b/crates/omnigraph/tests/bound_storage.rs @@ -251,6 +251,10 @@ async fn bound_handle_workload_child() { ) .await .unwrap(); + // A detailed listing reads `feature`'s latest `__manifest` version and the + // version it was created from, both through the handle's storage. + let listing = db.branch_list_detailed(&mut |_| true).await.unwrap(); + assert_eq!(listing.details.len(), 2); let merged = db.branch_merge("feature", "main").await.unwrap(); assert_eq!(merged.outcome, MergeOutcome::Merged); db.branch_delete("feature").await.unwrap(); diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index a2ad48426..a7fa17325 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -341,6 +341,7 @@ const READ_ONLY_SURFACES: &[(&str, &str)] = &[ ("db/omnigraph.rs", "graph_index"), ("blob.rs", "read_blob_at"), ("db/omnigraph.rs", "branch_list"), + ("db/omnigraph.rs", "branch_list_detailed"), ("db/omnigraph.rs", "get_commit"), ("db/omnigraph.rs", "commit_target_branch"), ("db/omnigraph.rs", "list_commits"), diff --git a/crates/omnigraph/tests/integration/branch_control_cost.rs b/crates/omnigraph/tests/integration/branch_control_cost.rs index 49a57f32b..1690a21f3 100644 --- a/crates/omnigraph/tests/integration/branch_control_cost.rs +++ b/crates/omnigraph/tests/integration/branch_control_cost.rs @@ -21,7 +21,11 @@ use std::future::Future; -use crate::helpers::cost::{IoCounts, cost_harness, local_graph, measure}; +use crate::helpers::cost::{ + DEEP_HISTORY_DEPTH, IoCounts, assert_flat, cost_harness, local_graph, measure, s3_graph, +}; +use crate::helpers::{MUTATION_QUERIES, mixed_params}; +use omnigraph::db::BranchListing; /// Run an async test body on a thread with a large stack (see module docs). fn on_big_stack(body: impl FnOnce() -> F + Send + 'static) @@ -110,6 +114,154 @@ fn branch_delete_manifest_reads_bounded_per_surviving_branch() { }); } +/// What `branch_list_detailed` may spend per listed branch, from a measured +/// curve. Scans are the same on every backend: one of the branch's latest +/// `__manifest` version, and one of the version it was created from once it has +/// written a commit. Object reads differ by backend, so each pins its own. +struct DetailedListingCost { + /// `__manifest` object reads per branch that has written none. + unwritten_reads: u64, + /// `__manifest` object reads per branch that has written a commit. + written_reads: u64, +} + +/// A local store counts the listing, the ref read and the version resolution of +/// each checkout; Lance reads the local files themselves without the counting +/// wrapper. +const LOCAL_DETAILED_LISTING: DetailedListingCost = DetailedListingCost { + unwritten_reads: 1, + written_reads: 3, +}; + +/// Every read is a request on an object store; measured on an S3 emulator. +const S3_DETAILED_LISTING: DetailedListingCost = DetailedListingCost { + unwritten_reads: 5, + written_reads: 8, +}; + +async fn insert_on(db: &omnigraph::Session, branch: &str, name: &str) { + db.mutate( + branch, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", 30)]), + ) + .await + .unwrap(); +} + +async fn measured_details(db: &omnigraph::Session) -> (BranchListing, IoCounts) { + let (listing, io) = measure(db.branch_list_detailed(&mut |_| true)).await; + (listing.unwrap(), io) +} + +/// A detailed branch listing reads each listed branch's `__manifest` a fixed +/// number of times, never its lineage: one scan per branch that has written +/// none, two per branch that has, within the backend's read budget, and a +/// branch and main that wrote `DEEP_HISTORY_DEPTH` commits cost what they cost +/// after one. +#[test] +fn branch_list_detailed_reads_bounded_per_branch_and_flat_in_history() { + on_big_stack(|| { + cost_harness(async { + let dir = tempfile::tempdir().unwrap(); + assert_branch_list_detailed_cost(local_graph(&dir).await, LOCAL_DETAILED_LISTING).await; + }) + }); +} + +/// The same gate on an object store, where every `__manifest` read is a +/// request. Skips without `OMNIGRAPH_S3_TEST_BUCKET`. +#[test] +fn branch_list_detailed_reads_bounded_per_branch_and_flat_in_history_s3() { + on_big_stack(|| { + cost_harness(async { + let Some(db) = s3_graph("branch-details").await else { + eprintln!("skipping: OMNIGRAPH_S3_TEST_BUCKET is not set"); + return; + }; + assert_branch_list_detailed_cost(db, S3_DETAILED_LISTING).await; + }) + }); +} + +async fn assert_branch_list_detailed_cost(db: omnigraph::Session, budget: DetailedListingCost) { + db.branch_create("unwritten").await.unwrap(); + let (_, before) = measured_details(&db).await; + db.branch_create("unwritten_2").await.unwrap(); + let (_, after) = measured_details(&db).await; + let (reads, scans) = ( + after.manifest_reads - before.manifest_reads, + after.manifest_scan_count - before.manifest_scan_count, + ); + eprintln!("one added unwritten branch: LIST DETAILED manifest_reads +{reads} scans +{scans}"); + assert_eq!(scans, 1, "a branch that has written none is scanned once"); + assert!( + reads <= budget.unwritten_reads, + "{reads} __manifest reads for a branch that has written none (budget {})", + budget.unwritten_reads + ); + db.branch_delete("unwritten").await.unwrap(); + db.branch_delete("unwritten_2").await.unwrap(); + + let mut curve: Vec<(u64, IoCounts)> = Vec::new(); + let mut created = 0u64; + for n in [2u64, 6] { + while created < n { + let name = format!("b{created}"); + db.branch_create(&name).await.unwrap(); + insert_on(&db, &name, &format!("{name}_0")).await; + created += 1; + } + let (listing, io) = measured_details(&db).await; + assert_eq!(listing.details.len() as u64, n + 1); + eprintln!( + "written branches={n} (+main): LIST DETAILED manifest_reads={} manifest_scans={} \ + internal_opens={}", + io.manifest_reads, io.manifest_scan_count, io.internal_open_count + ); + curve.push((n, io)); + } + let added = curve[1].0 - curve[0].0; + let scans = curve[1].1.manifest_scan_count - curve[0].1.manifest_scan_count; + assert_eq!( + scans, + 2 * added, + "a branch that has written a commit is scanned twice: its latest and its creation \ + version, not its lineage", + ); + let reads = (curve[1].1.manifest_reads - curve[0].1.manifest_reads).div_ceil(added); + assert!( + reads <= budget.written_reads, + "branch_list_detailed read {reads} __manifest objects per written branch (budget {})", + budget.written_reads + ); + + for i in 1..DEEP_HISTORY_DEPTH { + insert_on(&db, "b0", &format!("b0_{i}")).await; + insert_on(&db, "main", &format!("main_{i}")).await; + } + let (_, deep) = measured_details(&db).await; + eprintln!( + "after {DEEP_HISTORY_DEPTH} commits on b0 and main: LIST DETAILED manifest_reads={} \ + manifest_scans={}", + deep.manifest_reads, deep.manifest_scan_count + ); + let depth_curve = [(1, curve[1].1), (DEEP_HISTORY_DEPTH, deep)]; + assert_flat( + &depth_curve, + |io| io.manifest_reads, + SLOPE_SLACK, + "branch_list_detailed __manifest reads against branch history depth", + ); + assert_flat( + &depth_curve, + |io| io.manifest_scan_count, + 0, + "branch_list_detailed __manifest scans against branch history depth", + ); +} + /// Budget for a start at a past commit: it probes every object of each pin /// its source head does not also hold, so its requests grow with the /// fragments of those pins. On this fixture each earlier insert adds one diff --git a/crates/omnigraph/tests/integration/branching.rs b/crates/omnigraph/tests/integration/branching.rs index 9c12d0378..970b3c6e5 100644 --- a/crates/omnigraph/tests/integration/branching.rs +++ b/crates/omnigraph/tests/integration/branching.rs @@ -10,7 +10,10 @@ use lance::Dataset; use lance_index::is_system_index; use omnigraph::db::commit_graph::CommitGraph; -use omnigraph::db::{BranchCreated, MergeOutcome, Omnigraph, ReadTarget, Snapshot, SnapshotId}; +use omnigraph::db::{ + BranchCreated, BranchDetail, GraphCommit, MergeOutcome, Omnigraph, ReadTarget, Snapshot, + SnapshotId, +}; use omnigraph::error::{ BranchStartRefusal, CompletionEvidence, ManifestErrorKind, MergeConflictKind, OmniError, }; @@ -283,6 +286,250 @@ async fn branch_create_open_list_and_lazy_branching_work() { assert_eq!(count_rows(&main, "node:Person").await, 4); } +/// The newest commit `list_commits` reports for `branch`. +async fn newest_commit(db: &Omnigraph, branch: &str) -> GraphCommit { + db.list_commits(Some(branch)).await.unwrap().remove(0) +} + +/// Every branch's detail, keyed by name, from a listing that may read all. +async fn detailed_branches(db: &Omnigraph) -> BTreeMap { + let listing = db.branch_list_detailed(&mut |_| true).await.unwrap(); + let names: Vec<_> = listing.details.iter().map(|d| d.name.clone()).collect(); + assert_eq!(names, listing.branches, "every readable branch has details"); + listing + .details + .into_iter() + .map(|detail| (detail.name.clone(), detail)) + .collect() +} + +fn unix_micros_now() -> i64 { + let elapsed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + i64::try_from(elapsed.as_micros()).unwrap() +} + +/// Where a branch's details say it started: `from` and `from_commit_id`. +fn started(detail: &BranchDetail) -> (Option<&str>, Option<&str>) { + (detail.from.as_deref(), detail.from_commit_id.as_deref()) +} + +/// What a create receipt says the new branch started at. +fn receipt(created: &BranchCreated) -> (Option<&str>, Option<&str>) { + ( + Some(created.from.as_str()), + Some(created.from_commit_id.as_str()), + ) +} + +/// A branch's details answer what a client would otherwise derive from its +/// whole commit list: its head, the branch and commit its current lifetime +/// started from, as its create receipt said, when it was created and when it +/// last moved. The start is fixed for the lifetime, so a merge moves the merge +/// base but not the start; deleting and recreating the name starts a new +/// lifetime with its own start and creation time; cleanup leaves every start +/// readable. +#[tokio::test] +async fn branch_list_detailed_reports_head_start_and_lifetime() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + let insert = |name: &str| mixed_params(&[("$name", name)], &[("$age", 30)]); + let loaded = newest_commit(&db, "main").await; + + let before_create = unix_micros_now() / 1_000_000 * 1_000_000; + let created = db + .branch_create_from(ReadTarget::branch("main"), "feature") + .await + .unwrap(); + let after_create = unix_micros_now(); + let details = detailed_branches(&db).await; + let main = &details["main"]; + assert_eq!(main.head, loaded); + assert_eq!((started(main), main.created_at), ((None, None), None)); + assert_eq!(main.updated_at, loaded.created_at); + let unwritten = &details["feature"]; + assert_eq!( + unwritten.head, loaded, + "an unwritten branch reads its start" + ); + assert_eq!(started(unwritten), receipt(&created)); + assert_eq!(created.from_commit_id, loaded.graph_commit_id); + let created_at = unwritten.created_at.unwrap(); + assert!( + (before_create..=after_create).contains(&created_at), + "{before_create} <= {created_at} <= {after_create}" + ); + assert_eq!(unwritten.updated_at, created_at); + + mutate_main(&db, MUTATION_QUERIES, "insert_person", &insert("Mia")) + .await + .unwrap(); + mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "insert_person", + &insert("Eve"), + ) + .await + .unwrap(); + let written = newest_commit(&db, "feature").await; + assert_eq!( + written.parent_commit_id.as_deref(), + Some(loaded.graph_commit_id.as_str()) + ); + let child_created = db + .branch_create_from(ReadTarget::branch("feature"), "child") + .await + .unwrap(); + assert_eq!(child_created.from_commit_id, written.graph_commit_id); + mutate_branch( + &db, + "child", + MUTATION_QUERIES, + "insert_person", + &insert("Kid"), + ) + .await + .unwrap(); + // A branch started at a past commit reads that commit as its start, from + // the version it was cloned at, once it has written its own head. + let at_load = db + .branch_create_from( + ReadTarget::snapshot(SnapshotId::new(loaded.graph_commit_id.clone())), + "at_load", + ) + .await + .unwrap(); + mutate_branch( + &db, + "at_load", + MUTATION_QUERIES, + "insert_person", + &insert("Old"), + ) + .await + .unwrap(); + let details = detailed_branches(&db).await; + let feature = &details["feature"]; + assert_eq!(feature.head, written); + assert_eq!(feature.updated_at, written.created_at); + assert_eq!( + started(feature), + receipt(&created), + "a written branch keeps the commit it started at" + ); + let child = &details["child"]; + assert_eq!(child.head, newest_commit(&db, "child").await); + assert_eq!(started(child), receipt(&child_created)); + let at_load_detail = &details["at_load"]; + assert_eq!(at_load_detail.head, newest_commit(&db, "at_load").await); + assert_eq!(started(at_load_detail), receipt(&at_load)); + assert_eq!(at_load.from_commit_id, loaded.graph_commit_id); + + let main_head = newest_commit(&db, "main").await; + db.branch_merge("main", "feature").await.unwrap(); + let merged = newest_commit(&db, "feature").await; + assert_eq!( + merged.merged_parent_commit_id.as_deref(), + Some(main_head.graph_commit_id.as_str()) + ); + let base = db.merge_base("feature", "main").await.unwrap().merge_base; + assert_eq!(base, main_head, "the merge moved the merge base"); + let details = detailed_branches(&db).await; + assert_eq!(details["feature"].head, merged); + assert_eq!( + started(&details["feature"]), + receipt(&created), + "a merge does not move the start" + ); + + db.branch_delete("feature").await.unwrap(); + let recreated_receipt = db + .branch_create_from(ReadTarget::branch("main"), "feature") + .await + .unwrap(); + assert_eq!(recreated_receipt.from_commit_id, main_head.graph_commit_id); + let details = detailed_branches(&db).await; + assert_eq!( + details.keys().collect::>(), + ["at_load", "child", "feature", "main"] + ); + let recreated = &details["feature"]; + assert_eq!(started(recreated), receipt(&recreated_receipt)); + assert_eq!(recreated.head, main_head); + assert!(recreated.created_at.unwrap() >= created_at); + assert_eq!( + started(&details["child"]), + receipt(&child_created), + "`from` names the source as it was named then" + ); + + mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "insert_person", + &insert("Ann"), + ) + .await + .unwrap(); + db.cleanup(omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }) + .await + .unwrap(); + let details = detailed_branches(&db).await; + assert_eq!(started(&details["feature"]), receipt(&recreated_receipt)); + assert_eq!(started(&details["child"]), receipt(&child_created)); + assert_eq!(started(&details["at_load"]), receipt(&at_load)); + + let mut asked = Vec::new(); + let listing = db + .branch_list_detailed(&mut |name: &str| { + asked.push(name.to_string()); + name != "child" + }) + .await + .unwrap(); + assert_eq!( + asked, + ["main", "at_load", "child", "feature"], + "asked once per branch" + ); + assert_eq!(listing.branches, ["main", "at_load", "child", "feature"]); + assert_eq!( + listing + .details + .iter() + .map(|d| d.name.as_str()) + .collect::>(), + ["main", "at_load", "feature"], + "a branch the caller may not read lists by name alone" + ); + + // A fast-forward into a branch that has written nothing publishes a new + // commit on it: the head and `updated_at` follow that commit, and the + // start stays where the branch was created. + let ff_created = db + .branch_create_from(ReadTarget::branch("main"), "ff") + .await + .unwrap(); + mutate_main(&db, MUTATION_QUERIES, "insert_person", &insert("Fay")) + .await + .unwrap(); + let outcome = db.branch_merge("main", "ff").await.unwrap(); + assert_eq!(outcome.outcome, MergeOutcome::FastForward); + let fast_forwarded = newest_commit(&db, "ff").await; + let details = detailed_branches(&db).await; + let ff = &details["ff"]; + assert_eq!(ff.head, fast_forwarded); + assert_eq!(started(ff), receipt(&ff_created)); + assert_eq!(ff.updated_at, fast_forwarded.created_at); +} + #[tokio::test] async fn explicit_target_query_reads_multiple_branches_from_one_handle() { let dir = tempfile::tempdir().unwrap(); diff --git a/docs/dev/writes.md b/docs/dev/writes.md index f9971b22f..e2d61d17a 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -531,6 +531,17 @@ Cached write admission checks the ref's identifier and retirement metadata through the existing lookup, with no additional storage request. Cold branch enumeration reads active refs; creation does not scan retired histories. +A detailed listing (`list_graph_branches_detailed` in `omnigraph-catalog`) +derives every field from the same authority and writes nothing: each live +ref's `BranchContents` (`parent_branch`, `parent_version`, `create_at`), the +head row of the branch's latest `__manifest` version and, once the branch has +written a commit, the head row of its clone at `parent_version`, the commit it +started at (a start at a past commit clones that commit's version, so the +same read answers it). It reads no lineage, so its cost per branch is flat in +history (`branch_control_cost.rs`). The collector keeps each live branch's +oldest own `__manifest` version (`retained_versions` with `keep_base`), which +is that clone; a reclamation that dropped it would lose the start. + Stable table/incarnation identity, not `table_key`, determines whether a registration, rename, tombstone, or pointer belongs to the same lifetime. diff --git a/docs/user/branching/index.md b/docs/user/branching/index.md index a0ca3cae4..419b9e707 100644 --- a/docs/user/branching/index.md +++ b/docs/user/branching/index.md @@ -28,6 +28,19 @@ omnigraph branch list --target ./graph.omni omnigraph branch delete review/abandoned --target ./graph.omni ``` +`branch list --detailed` (served: `GET /branches?detailed=true`; MCP: +`branches_list` with `detailed`) adds a `details` array. Per branch it reports +`head`, the commit a read sees; `from` and `from_commit_id`, the branch and +commit it started from, as its create receipt named them (a later merge moves +the merge base, not these); `created_at`, recorded to the second; and +`updated_at`, the newest commit's time, or `created_at` while the branch has +written none. Times are Unix microseconds. `main` omits `from`, +`from_commit_id` and `created_at`. A branch deleted and created again reports +its new creation. Each branch costs a read of its own manifest, two once it has +written a commit, whatever its history, and the branches are not read as one +cut. The served listing needs the `read` that `GET /branches` needs, and has +details only for branches the caller may `read`. + Creating a branch defaults to `main` when `--from` is omitted. To start a branch at a past commit instead of a branch head, use `--from-commit`; see [start a branch at a commit](#start-a-branch-at-a-commit). A load can create a diff --git a/docs/user/cli/reference.md b/docs/user/cli/reference.md index f28a7f554..bb4c52fcc 100644 --- a/docs/user/cli/reference.md +++ b/docs/user/cli/reference.md @@ -43,7 +43,7 @@ operations on nonlocal storage. The retired `--quiet` cannot hide the destinatio | `mutate` | Run an insert/update/delete query (`as commit` leaves it unpublished; `as unattached branch [on ""]` records a snapshot that never publishes), or a `branch create`, `branch delete`, `branch merge`, `commit publish`, `commit drop`, or [`load csv`](../mutations/index.md#csv-loading) statement | direct or served | | `load` | Load graph JSONL in `overwrite`, `append`, or `merge` mode | direct or served | | `blob get/stat/put/clear` | Read, inspect, replace or clear one Blob cell | direct or served | -| `branch create/list/delete/merge/diff/merge-base` | Manage graph branches; `create --from-commit` starts a branch at a commit, `merge --as-commit` leaves the merge unpublished, `merge --dry-run` reports the outcome or conflicts and writes nothing, `diff` reviews what a branch changed since its merge base, `merge-base` prints that base | direct or served | +| `branch create/list/delete/merge/diff/merge-base` | Manage graph branches; `create --from-commit` starts a branch at a commit, `list --detailed` adds each branch's head, start and times, `merge --as-commit` leaves the merge unpublished, `merge --dry-run` reports the outcome or conflicts and writes nothing, `diff` reviews what a branch changed since its merge base, `merge-base` prints that base | direct or served | | `snapshot` | Show a branch snapshot: `internal_schema_version`, `graph_manifest_version`, and the datasets of one captured graph version | direct or served | | `commit list/show/changes/publish/drop` | Inspect history (`list --limit`/`--page-token` for one page of it, `list --unpublished` for a branch's unpublished commits, `list --unattached` for its live unattached snapshots; the two flags exclude each other and paging) or one commit's entity changes; publish an unpublished commit onto its recorded target branch (an unattached snapshot refuses with `unattached_cannot_publish`), or drop it (an unattached snapshot takes every descendant with it); `publish` and `drop` take `--as`, the reads refuse it | direct or served | | `diff` | Net entity changes between two points: branches or commit ids (`--from`, `--to`) | direct or served | diff --git a/docs/user/operations/mcp.md b/docs/user/operations/mcp.md index c783debfb..cb902c287 100644 --- a/docs/user/operations/mcp.md +++ b/docs/user/operations/mcp.md @@ -194,7 +194,7 @@ for a deployment whose policy was written with an LLM caller in mind. | `read` (opt-in) | `graph`, `query`, optional `name`, `params`, `branch`, `snapshot` | `POST /graphs/{graph_id}/query` | `read`; `change` for `explain load csv` | | `snapshot` | `graph`, optional `branch` | `GET /graphs/{graph_id}/snapshot` | `read` | | `access` | `graph`, optional `branch`, `target_branch` | `GET /graphs/{graph_id}/access` | none | -| `branches_list` | `graph` | `GET /graphs/{graph_id}/branches` | `read` | +| `branches_list` | `graph`, optional `detailed` | `GET /graphs/{graph_id}/branches` | `read`; with `detailed`, details only for branches the caller may `read` | | `branches_diff` | `graph`, `source`, optional `target`, `kind`, `type`, `op`, `page_token`, `limit` | `GET /graphs/{graph_id}/branches/diff` | `read` on both branches | | `branches_merge_base` | `graph`, `source`, optional `target` | `GET /graphs/{graph_id}/branches/merge-base` | `read` on both branches | | `commits_list` | `graph`, optional `branch`, `listing`, `limit`, `page_token` | `GET /graphs/{graph_id}/commits` | `read` | diff --git a/docs/user/operations/server.md b/docs/user/operations/server.md index c1256e52d..087ff8fb3 100644 --- a/docs/user/operations/server.md +++ b/docs/user/operations/server.md @@ -221,7 +221,7 @@ describes sign-in, which resource identifier to choose, and the endpoint's tools | `/graphs/{graph_id}/queries` | List and invoke stored queries, including conditional mutations | | `/graphs/{graph_id}/load`, `/load/ndjson` | Bounded batch loading | | `/graphs/{graph_id}/blob` | GET/HEAD one Blob cell; PUT/DELETE to replace or clear it | -| `/graphs/{graph_id}/branches` | Branch management and merge, a merge dry run (`dry_run: true`), a branch's diff against its merge base (`GET /branches/diff`) and that merge base (`GET /branches/merge-base`) | +| `/graphs/{graph_id}/branches` | Branch management and merge, a detailed listing (`GET /branches?detailed=true`), a merge dry run (`dry_run: true`), a branch's diff against its merge base (`GET /branches/diff`) and that merge base (`GET /branches/merge-base`) | | `/graphs/{graph_id}/snapshot`, `/commits` | Snapshot (`internal_schema_version`, `graph_manifest_version`, and the datasets of one captured graph version), history (paged with `limit` and `page_token`), per-commit changes, and the unpublished commits of a branch (`GET /commits?listing=unpublished`, `POST /commits/{commit_id}/publish`, `DELETE /commits/{commit_id}`) | | `/graphs/{graph_id}/changes`, `/diff` | Poll a branch feed, establish a baseline, or diff two points (branches or commit ids) | | `/graphs/{graph_id}/schema` | Show the accepted schema | diff --git a/openapi.json b/openapi.json index 23f0d2afd..53951aa0b 100644 --- a/openapi.json +++ b/openapi.json @@ -3937,7 +3937,7 @@ "branches" ], "summary": "List all branches.", - "description": "Returns branch names sorted by name in byte order. Read-only. The GQ statement\n`branch list` on `POST /query` runs the same body.", + "description": "Returns branch names sorted by name in byte order. Read-only. The GQ statement\n`branch list` on `POST /query` runs the same body.\n\n`detailed=true` adds `details`: for each branch the caller may `read`, its\nhead commit, the branch and commit it started from, when it was created,\nand when it last moved. Each listed branch costs a read of its latest\nversion, and of the version it was created from once it has written a\ncommit; no commit history is read.", "operationId": "branchList", "parameters": [ { @@ -3949,6 +3949,15 @@ "type": "string" } }, + { + "name": "detailed", + "in": "query", + "description": "`true` adds `details`: each branch's head, the branch and commit it\nstarted from, when it was created, and when it last moved. Defaults\nto `false`. Reads each listed branch's latest version, and the version\nit was created from once it has written a commit, never its commit\nhistory; the names alone read neither.", + "required": false, + "schema": { + "type": "boolean" + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -3994,7 +4003,7 @@ } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Invalid `detailed`; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -13144,6 +13153,53 @@ } } }, + "BranchDetailOutput": { + "type": "object", + "description": "One live branch as `GET /branches?detailed=true` reports it. `from` and\n`from_commit_id` say where the branch started, as its `POST /branches`\nreceipt did. A branch deleted and created again under the same name is a\nnew lifetime: its start and creation time are the later creation's. The\noptional fields are absent on `main`.", + "required": [ + "name", + "head", + "updated_at" + ], + "properties": { + "created_at": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "When the branch was created, as Unix epoch microseconds recorded to\nthe second.", + "example": 1714000000000000 + }, + "from": { + "type": [ + "string", + "null" + ], + "description": "The branch this one started from, as that branch was named then; it\nmay since have been deleted or recreated." + }, + "from_commit_id": { + "type": [ + "string", + "null" + ], + "description": "The commit the branch started at. A later merge does not move it;\n`GET /branches/merge-base` answers where two branches last met." + }, + "head": { + "$ref": "#/components/schemas/CommitOutput", + "description": "The commit a read of the branch sees: its newest commit, or the commit\nit started at while it has written none." + }, + "name": { + "type": "string" + }, + "updated_at": { + "type": "integer", + "format": "int64", + "description": "When the head last moved, as Unix epoch microseconds: the time of the\nbranch's newest commit, or `created_at` while it has written none.", + "example": 1714000000000000 + } + } + }, "BranchListOutput": { "type": "object", "required": [ @@ -13154,7 +13210,18 @@ "type": "array", "items": { "type": "string" - } + }, + "description": "Every live branch, sorted by name in byte order." + }, + "details": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/BranchDetailOutput" + }, + "description": "Present only with `detailed=true`: one entry per branch the caller\nmay `read`, in the order of `branches`. A branch the caller may not\nread is listed in `branches` alone." } } }, diff --git a/skills/omnigraph/references/commands.md b/skills/omnigraph/references/commands.md index c25189ebb..f121807c5 100644 --- a/skills/omnigraph/references/commands.md +++ b/skills/omnigraph/references/commands.md @@ -138,7 +138,9 @@ omnigraph branch delete --target $REPO All support `--json`. `--from` and `--into` default to `main`. `branch create --from-commit ` starts the branch at a published commit instead. `--delete-branch` removes the source only after a successful merge publication. -A served `branch delete --json` needs `--yes`. +A served `branch delete --json` needs `--yes`. `branch list --detailed` adds +each branch's head, the branch (`from`) and commit (`from_commit_id`) it +started from, and `created_at`/`updated_at`, without reading commit history. The same operations are GQ branch statements, useful when a client already sends `.gq` source: