diff --git a/Cargo.lock b/Cargo.lock index 2dd52ee23..8ca24cd80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1476,7 +1476,7 @@ dependencies = [ "tempfile", "tinyagents-harness", "tinyinference-llm", - "tinytools", + "tinytools 0.4.1", "tokio", "tracing", ] @@ -1504,8 +1504,8 @@ dependencies = [ "tinyagents-definition", "tinyinference-embeddings", "tinyinference-llm", - "tinytools", - "tinytools-agent", + "tinytools 0.4.1", + "tinytools-agent 0.4.1", "tokio", "tracing", "uuid", @@ -1534,7 +1534,7 @@ dependencies = [ "tinyagents-session", "tinyinference-embeddings", "tinyinference-llm", - "tinytools", + "tinytools 0.4.1", "tokio", ] @@ -1559,7 +1559,7 @@ dependencies = [ "tinyagents-runtime", "tinyagents-session", "tinyinference-llm", - "tinytools", + "tinytools 0.4.1", "tokio", "uuid", ] @@ -1576,7 +1576,7 @@ dependencies = [ "tinyagents-definition", "tinyagents-harness", "tinyinference-llm", - "tinytools", + "tinytools 0.4.1", "tokio", ] @@ -1593,7 +1593,7 @@ dependencies = [ "tinyagents-harness", "tinyagents-session", "tinyinference-llm", - "tinytools", + "tinytools 0.4.1", "tokio", ] @@ -1652,7 +1652,7 @@ dependencies = [ "sha2", "thiserror", "tinyinference-core", - "tinytools-agent", + "tinytools-agent 0.3.0", "tokio", "tracing", ] @@ -1670,6 +1670,17 @@ dependencies = [ [[package]] name = "tinytools" version = "0.3.0" +source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "serde_json", +] + +[[package]] +name = "tinytools" +version = "0.4.1" dependencies = [ "anyhow", "async-trait", @@ -1680,11 +1691,22 @@ dependencies = [ [[package]] name = "tinytools-agent" version = "0.3.0" +source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" +dependencies = [ + "regex", + "serde", + "serde_json", + "tinytools 0.3.0", +] + +[[package]] +name = "tinytools-agent" +version = "0.4.1" dependencies = [ "regex", "serde", "serde_json", - "tinytools", + "tinytools 0.4.1", "tracing", ] diff --git a/crates/tinyagents-graph/Cargo.toml b/crates/tinyagents-graph/Cargo.toml index 71f5b2dc2..1bcbdd448 100644 --- a/crates/tinyagents-graph/Cargo.toml +++ b/crates/tinyagents-graph/Cargo.toml @@ -19,7 +19,7 @@ tinyagents-harness = { path = "../tinyagents-harness", version = "2.1.2", defaul "langfuse", ] } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs"] } tracing = { workspace = true } diff --git a/crates/tinyagents-harness/Cargo.toml b/crates/tinyagents-harness/Cargo.toml index 918db3b99..5c1e7bd0d 100644 --- a/crates/tinyagents-harness/Cargo.toml +++ b/crates/tinyagents-harness/Cargo.toml @@ -28,10 +28,10 @@ sha2 = "0.11" thiserror = "2" tracing = { workspace = true } tinyagents-definition = { path = "../tinyagents-definition", version = "2.1.2" } -tinytools-agent = { path = "../../vendor/tinytools/crates/tinytools-agent", version = "0.3.0", default-features = false } +tinytools-agent = { path = "../../vendor/tinytools/crates/tinytools-agent", version = "0.4.1", default-features = false } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } tinyinference-embeddings = { path = "../../vendor/tinyinference/crates/tinyinference-embeddings", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs", "io-util", "process"] } tempfile = { workspace = true } wait-timeout = { version = "0.2", optional = true } diff --git a/crates/tinyagents-harness/src/agent_loop/tools.rs b/crates/tinyagents-harness/src/agent_loop/tools.rs index bdce90fe5..cd5439528 100644 --- a/crates/tinyagents-harness/src/agent_loop/tools.rs +++ b/crates/tinyagents-harness/src/agent_loop/tools.rs @@ -284,16 +284,22 @@ impl AgentHarness { if !self.policy.discovery.enabled { return crate::tool::discover::DeferredCatalog::default(); } - let mut schemas = self + let mut entries = self .tools - .deferred_schemas() + .deferred_schemas_with_families() .into_iter() - .filter(|schema| host_allows(&schema.name)) + .filter(|(schema, _)| host_allows(&schema.name)) .collect::>(); if let Some(preparation) = &self.policy.tool_schemas { - schemas = crate::tool::prepare_tool_schemas(&schemas, preparation); + let families: Vec> = + entries.iter().map(|(_, family)| family.clone()).collect(); + let schemas: Vec<_> = entries.into_iter().map(|(schema, _)| schema).collect(); + entries = crate::tool::prepare_tool_schemas(&schemas, preparation) + .into_iter() + .zip(families) + .collect(); } - crate::tool::discover::DeferredCatalog::build(schemas) + crate::tool::discover::DeferredCatalog::build_with_families(entries) } /// Resolves the discovery bridge for one call, when it is one. @@ -303,7 +309,7 @@ impl AgentHarness { /// admission continues with it, and `None` untouched for any other name. /// A malformed `tool_call` payload is answered with a tool error rather /// than passed on, so the model can correct it. - fn answer_discovery_bridge( + async fn answer_discovery_bridge( &self, ctx: &RunContext, status: &mut HarnessRunStatus, @@ -334,11 +340,13 @@ impl AgentHarness { return Ok(None); } if call.name == TOOL_SEARCH_NAME { - let (result, matched) = crate::tool::discover::answer_tool_search( + let answer = crate::tool::discover::answer_tool_search( &catalog, &self.policy.discovery, &call.arguments, - ); + ) + .await; + let ranking = answer.ranking; // `query` is model-supplied tool-call content, same privacy // class as a normal tool call's arguments, so it honors the same // `RunPolicy::capture.tool_io` gate (default `false`, payload @@ -355,10 +363,18 @@ impl AgentHarness { } else { String::new() }, - matched, + matched: answer.matched, + ranker: ranking + .as_ref() + .map(|r| r.ranker.to_string()) + .unwrap_or_default(), + top_confidence: ranking.as_ref().and_then(|r| r.top_confidence), + fallback: ranking.as_ref().and_then(|r| r.fallback.clone()), + shadow_matched: ranking.as_ref().and_then(|r| r.shadow_names.clone()), + latency_ms: ranking.as_ref().map_or(0, |r| r.latency_ms), }); status.set_last_event(record.id); - return Ok(Some(ResolvedToolCall::Answered(result))); + return Ok(Some(ResolvedToolCall::Answered(answer.result))); } match crate::tool::discover::unwrap_tool_call(&call.arguments) { Ok((name, arguments)) => { @@ -517,7 +533,7 @@ impl AgentHarness { // a call the provider could not parse is left for the recovery below. if call.invalid.is_none() && self.tools.dispatch(&call.name).is_none() - && let Some(answered) = self.answer_discovery_bridge(ctx, status, call)? + && let Some(answered) = self.answer_discovery_bridge(ctx, status, call).await? { return Ok(answered); } diff --git a/crates/tinyagents-harness/src/events/types.rs b/crates/tinyagents-harness/src/events/types.rs index 4052069b0..d3182bb31 100644 --- a/crates/tinyagents-harness/src/events/types.rs +++ b/crates/tinyagents-harness/src/events/types.rs @@ -139,6 +139,24 @@ pub enum AgentEvent { query: String, /// Number of deferred tools returned. matched: usize, + /// Which ranker's answer was served: `"bm25"`, or the host ranker's + /// [`tinytools::ToolRanker::kind`]. Empty when the query was rejected + /// before ranking. + #[serde(default)] + ranker: String, + /// The best hit's calibrated confidence, when the ranker gave one. + #[serde(default, skip_serializing_if = "Option::is_none")] + top_confidence: Option, + /// Why the host ranker was not served, when one was active. + #[serde(default, skip_serializing_if = "Option::is_none")] + fallback: Option, + /// The BM25 ranking, when the policy asked to compare it against the + /// served one. + #[serde(default, skip_serializing_if = "Option::is_none")] + shadow_matched: Option>, + /// Wall time of the ranking, in milliseconds. + #[serde(default)] + latency_ms: u64, }, /// The model invoked a deferred tool through the intrinsic `tool_call` diff --git a/crates/tinyagents-harness/src/tool/discover/README.md b/crates/tinyagents-harness/src/tool/discover/README.md index 868042ee8..5e3f3d1dc 100644 --- a/crates/tinyagents-harness/src/tool/discover/README.md +++ b/crates/tinyagents-harness/src/tool/discover/README.md @@ -6,10 +6,9 @@ this file is the map of the module. | File | Owns | |---------------|----------------------------------------------------------------------| -| `types.rs` | `ToolDiscoveryPolicy` (the knobs), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted), `DeferredTool` | -| `index.rs` | `Bm25Index` + `tokenize` — ranking over `(sort_key, text)` pairs, knows nothing about tools | +| `types.rs` | `ToolDiscoveryPolicy` (the knobs, the host `ranker` and `DiscoveryRankMode`), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted, ranked through the policy), `DeferredTool`, `RankedSearch` | | `manifest.rs` | `render_manifest` — the budgeted listing inside `tool_search`'s description: full → names → count | -| `bridge.rs` | The two intrinsic tools: `bridge_schemas`, `answer_tool_search`, `unwrap_tool_call` | +| `bridge.rs` | The two intrinsic tools: `bridge_schemas`, `answer_tool_search` (async; returns a `SearchAnswer`), `unwrap_tool_call` | | `test.rs` | Unit tests for all of the above | The agent loop (`agent_loop/run_loop.rs`, `agent_loop/tools.rs`) is the only @@ -27,3 +26,8 @@ Invariants worth keeping: the bridge is enabled; a `Hidden` tool is never callable by the model. - The manifest is bounded by `manifest_token_budget`; the search answer clips descriptions to 500 chars and `limit` to `max_limit`. +- A search never fails. The host ranker (`ToolDiscoveryPolicy::ranker`, + any `tinytools::ToolRanker`) is served when active; on error or an empty + answer BM25 answers instead and `RankedSearch::fallback` says why. + `DiscoveryRankMode::Compare` serves the host ranker and carries the BM25 + ranking alongside for comparison. BM25 itself lives in `tinytools::rank`. diff --git a/crates/tinyagents-harness/src/tool/discover/bridge.rs b/crates/tinyagents-harness/src/tool/discover/bridge.rs index cd5c5895b..10d5156d6 100644 --- a/crates/tinyagents-harness/src/tool/discover/bridge.rs +++ b/crates/tinyagents-harness/src/tool/discover/bridge.rs @@ -11,10 +11,10 @@ use serde_json::{Value, json}; use tinyinference_llm::tool::{ToolFormat, ToolSchema}; -use tinytools::ToolResult; +use tinytools::{RankContext, ToolResult}; use super::manifest::render_manifest; -use super::types::{DeferredCatalog, ToolDiscoveryPolicy}; +use super::types::{DeferredCatalog, RankedSearch, ToolDiscoveryPolicy}; /// Name of the intrinsic search bridge. pub const TOOL_SEARCH_NAME: &str = "tool_search"; @@ -96,26 +96,42 @@ fn tool_call_schema() -> ToolSchema { } } -/// Answers a `tool_search` call against the run's catalogue, returning the -/// result to hand the model and how many tools it named. -#[must_use] -pub fn answer_tool_search( +/// What a `tool_search` produced: the result to hand the model plus the +/// facts the loop reports in its `ToolSearched` event. +#[derive(Debug)] +pub struct SearchAnswer { + /// The tool result the model sees. + pub result: ToolResult, + /// How many tools it named. + pub matched: usize, + /// Which ranker's answer was served, and how it went. `None` when the + /// query was rejected before ranking. + pub ranking: Option, +} + +/// Answers a `tool_search` call against the run's catalogue. +/// +/// Ranks as `policy` says — the host ranker when one is active, BM25 +/// otherwise or on failure — and returns the full schema of every hit so +/// the model can call it. +pub async fn answer_tool_search( catalog: &DeferredCatalog, policy: &ToolDiscoveryPolicy, arguments: &Value, -) -> (ToolResult, usize) { +) -> SearchAnswer { let query = arguments .get("query") .and_then(Value::as_str) .unwrap_or_default() .trim(); if query.is_empty() { - return ( - ToolResult::error(format!( + return SearchAnswer { + result: ToolResult::error(format!( "`{TOOL_SEARCH_NAME}` needs a `query` describing what you want to do." )), - 0, - ); + matched: 0, + ranking: None, + }; } let (default_limit, max_limit) = policy.effective_limits(); let limit = arguments @@ -125,16 +141,24 @@ pub fn answer_tool_search( usize::try_from(n).unwrap_or(usize::MAX).clamp(1, max_limit) }); - let matches = catalog.search(query, limit); + let ranking = catalog + .rank(policy, query, &RankContext::empty(), limit) + .await; + let matches: Vec<&ToolSchema> = ranking + .names + .iter() + .filter_map(|name| catalog.get(name)) + .collect(); if matches.is_empty() { - return ( - ToolResult::success(format!( + return SearchAnswer { + result: ToolResult::success(format!( "No deferred tool matches \"{query}\". {} tool(s) are searchable; everything \ else you can use is already in your tool list.", catalog.len() )), - 0, - ); + matched: 0, + ranking: Some(ranking), + }; } let payload: Vec = matches .iter() @@ -148,13 +172,14 @@ pub fn answer_tool_search( .collect(); let rendered = serde_json::to_string_pretty(&payload).unwrap_or_else(|_| "[]".to_string()); let matched = payload.len(); - ( - ToolResult::success(format!( + SearchAnswer { + result: ToolResult::success(format!( "{matched} match(es). Invoke one with `{TOOL_CALL_NAME}` {{\"name\", \"arguments\"}} \ or by its own name, using the parameters shown.\n{rendered}" )), matched, - ) + ranking: Some(ranking), + } } /// Unwraps a `tool_call` payload into the real `(name, arguments)` pair. diff --git a/crates/tinyagents-harness/src/tool/discover/index.rs b/crates/tinyagents-harness/src/tool/discover/index.rs deleted file mode 100644 index 1a86dbb07..000000000 --- a/crates/tinyagents-harness/src/tool/discover/index.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! BM25 ranking over deferred tool declarations. -//! -//! Names nothing from the rest of the harness: it takes `(sort_key, text)` -//! pairs and returns ranked indices, so the tool-specific half (what text is -//! searchable, what a hit looks like) stays in [`super::types`]. -//! -//! Hand-rolled rather than the `bm25` crate: the arithmetic is ~80 lines and -//! the harness is a dependency floor for every embedder. - -use std::collections::HashMap; - -/// Words that carry no capability meaning, dropped from a query before ranking. -/// -/// A document-frequency threshold alone cannot do this on a small corpus: -/// with three deferred tools, "a" may appear in exactly one description and -/// so rank as the *most* distinguishing term in "send a calendar invite". -/// Deliberately short and English-only — it can only remove terms, so a -/// description in another language ranks exactly as it would without it. -/// Words that could name a capability ("up", as in "look up") are left in. -const STOPWORDS: &[&str] = &[ - "a", "an", "and", "are", "as", "at", "be", "but", "by", "can", "do", "for", "from", "how", "i", - "if", "in", "into", "is", "it", "its", "me", "my", "of", "on", "or", "our", "so", "that", - "the", "their", "them", "then", "there", "these", "they", "this", "to", "was", "we", "were", - "what", "when", "which", "who", "will", "with", "would", "you", "your", -]; - -/// BM25 term-frequency saturation (the standard default). -const K1: f64 = 1.2; -/// BM25 length normalisation (the standard default). -const B: f64 = 0.75; - -/// Splits text into search terms. -/// -/// Splits on non-alphanumerics **and** on a lower→upper transition, so -/// `memory_hybrid_search` and `readWorkflowResource` both yield the words a -/// person would type. Without the camel-case rule a query for "workflow" -/// misses a tool whose only mention of it is inside an identifier. -pub fn tokenize(text: &str) -> Vec { - let mut out = Vec::new(); - let mut current = String::new(); - let mut previous_lower = false; - for ch in text.chars() { - if ch.is_alphanumeric() { - if ch.is_uppercase() && previous_lower && !current.is_empty() { - out.push(std::mem::take(&mut current)); - } - current.extend(ch.to_lowercase()); - previous_lower = ch.is_lowercase() || ch.is_numeric(); - } else if !current.is_empty() { - out.push(std::mem::take(&mut current)); - previous_lower = false; - } - } - if !current.is_empty() { - out.push(current); - } - out -} - -/// A ranked corpus of short documents, identified by their index into the -/// slice the caller built the index from. -#[derive(Default, Debug, Clone)] -pub struct Bm25Index { - documents: Vec, - document_frequency: HashMap, - average_length: f64, -} - -#[derive(Debug, Clone)] -struct Document { - /// Used only to break score ties deterministically. - sort_key: String, - tokens: Vec, -} - -impl Bm25Index { - /// Builds from `(sort_key, searchable_text)` pairs, in caller order. - /// - /// `sort_key` breaks ties; make it the id the caller would print, so two - /// identical queries produce identical output. An unstable order would make - /// a model's transcript non-reproducible for no benefit. - pub fn build<'a>(documents: impl IntoIterator) -> Self { - let documents: Vec = documents - .into_iter() - .map(|(sort_key, text)| Document { - sort_key: sort_key.to_string(), - tokens: tokenize(text), - }) - .collect(); - - let mut document_frequency: HashMap = HashMap::new(); - for doc in &documents { - let mut seen: Vec<&str> = Vec::new(); - for token in &doc.tokens { - if !seen.contains(&token.as_str()) { - seen.push(token); - *document_frequency.entry(token.clone()).or_insert(0) += 1; - } - } - } - - let total: usize = documents.iter().map(|d| d.tokens.len()).sum(); - let average_length = if documents.is_empty() { - 0.0 - } else { - total as f64 / documents.len() as f64 - }; - - Self { - documents, - document_frequency, - average_length, - } - } - - /// `true` when the corpus holds no documents. - #[must_use] - pub fn is_empty(&self) -> bool { - self.documents.is_empty() - } - - /// Number of documents in the corpus. - #[must_use] - pub fn len(&self) -> usize { - self.documents.len() - } - - /// Ranks against `query`, best first, returning **indices** into the corpus. - /// - /// Only documents scoring above zero are returned. Padding the list out to - /// `limit` with unrelated entries would spend exactly the tokens deferral - /// exists to save, and would invite the model to call something unrelated - /// to what it asked for. - #[must_use] - pub fn search(&self, query: &str, limit: usize) -> Vec { - let terms = self.significant(tokenize(query)); - if terms.is_empty() || self.documents.is_empty() { - return Vec::new(); - } - let mut scored: Vec<(f64, usize)> = self - .documents - .iter() - .enumerate() - .map(|(index, doc)| (self.score(doc, &terms), index)) - .filter(|(score, _)| *score > 0.0) - .collect(); - - scored.sort_by(|a, b| { - b.0.partial_cmp(&a.0) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| { - self.documents[a.1] - .sort_key - .cmp(&self.documents[b.1].sort_key) - }) - }); - scored.into_iter().take(limit).map(|(_, i)| i).collect() - } - - /// Drops query terms too common in this corpus to mean anything. - /// - /// The IDF below carries the standard `+ 1`, which keeps a term present in - /// every document at a small **positive** weight — without it a corpus of - /// one document scores every term at zero and nothing is ever findable. - /// The cost is that "a" and "the" score, so a document-frequency filter - /// runs first: drop the term when `df >= max(2, ceil(0.8 * n))`. The floor - /// of 2 makes it inert on a one-document corpus; [`STOPWORDS`] covers the - /// small-corpus case the threshold cannot. - fn significant(&self, terms: Vec) -> Vec { - let n = self.documents.len(); - if n == 0 { - return Vec::new(); - } - let threshold = std::cmp::max(2, (0.8 * n as f64).ceil() as usize); - terms - .into_iter() - .filter(|term| !STOPWORDS.contains(&term.as_str())) - .filter(|term| *self.document_frequency.get(term).unwrap_or(&0) < threshold) - .collect() - } - - fn score(&self, doc: &Document, terms: &[String]) -> f64 { - let length = doc.tokens.len() as f64; - let count = self.documents.len() as f64; - terms - .iter() - .map(|term| { - let frequency = doc.tokens.iter().filter(|t| *t == term).count() as f64; - if frequency == 0.0 { - return 0.0; - } - let df = *self.document_frequency.get(term).unwrap_or(&0) as f64; - // Standard BM25 IDF with the +1 that keeps a term present in - // every document at a small positive weight (see `significant`). - let idf = ((count - df + 0.5) / (df + 0.5) + 1.0).ln(); - let normalised = frequency * (K1 + 1.0) - / (frequency + K1 * (1.0 - B + B * length / self.average_length.max(1.0))); - idf * normalised - }) - .sum() - } -} diff --git a/crates/tinyagents-harness/src/tool/discover/mod.rs b/crates/tinyagents-harness/src/tool/discover/mod.rs index 63f5b1b33..2ca1136b6 100644 --- a/crates/tinyagents-harness/src/tool/discover/mod.rs +++ b/crates/tinyagents-harness/src/tool/discover/mod.rs @@ -4,25 +4,31 @@ //! See `README.md` in this directory for the design and the cache rationale. //! The pieces: //! -//! - [`DeferredCatalog`] — the run's deferred schemas, BM25-indexed. +//! - [`DeferredCatalog`] — the run's deferred schemas, BM25-indexed, ranked +//! by the host's [`tinytools::ToolRanker`] when one is installed. //! - [`ToolDiscoveryPolicy`] — the knobs, carried on -//! [`crate::runtime::RunPolicy::discovery`]. +//! [`crate::runtime::RunPolicy::discovery`], including the ranker and the +//! [`DiscoveryRankMode`] that says how it is used. //! - [`bridge_schemas`] / [`answer_tool_search`] / [`unwrap_tool_call`] — the //! two intrinsic bridge tools the agent loop advertises and answers. //! - [`render_manifest`] — the budgeted listing inside `tool_search`'s //! description. mod bridge; -mod index; mod manifest; mod types; pub use bridge::{ - TOOL_CALL_NAME, TOOL_SEARCH_NAME, answer_tool_search, bridge_schemas, unwrap_tool_call, + SearchAnswer, TOOL_CALL_NAME, TOOL_SEARCH_NAME, answer_tool_search, bridge_schemas, + unwrap_tool_call, }; -pub use index::{Bm25Index, tokenize}; +// The BM25 arithmetic lives in `tinytools::rank` now, so a host ranks with the +// same index the bridge does; the old paths keep resolving. pub use manifest::{MANIFEST_DESCRIPTION_CHARS, first_sentence, render_manifest}; -pub use types::{DeferredCatalog, DeferredTool, ToolDiscoveryPolicy}; +pub use tinytools::rank::{Bm25Index, tokenize}; +pub use types::{ + DeferredCatalog, DeferredTool, DiscoveryRankMode, RankedSearch, ToolDiscoveryPolicy, +}; #[cfg(test)] mod test; diff --git a/crates/tinyagents-harness/src/tool/discover/test.rs b/crates/tinyagents-harness/src/tool/discover/test.rs index c2c2b1419..2fbd75d9d 100644 --- a/crates/tinyagents-harness/src/tool/discover/test.rs +++ b/crates/tinyagents-harness/src/tool/discover/test.rs @@ -147,62 +147,267 @@ fn bridge_schemas_are_byte_stable_across_builds() { assert_eq!(a, b); } -#[test] -fn answer_tool_search_returns_full_schemas_for_hits() { +#[tokio::test] +async fn answer_tool_search_returns_full_schemas_for_hits() { let policy = ToolDiscoveryPolicy::default(); - let (result, matched) = answer_tool_search( + let SearchAnswer { + result, + matched, + ranking, + } = answer_tool_search( &catalog(), &policy, &json!({"query": "read a pdf", "limit": 1}), - ); + ) + .await; assert!(!result.is_error); assert_eq!(matched, 1); + let ranking = ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert!(ranking.fallback.is_none()); + assert!(ranking.shadow_names.is_none()); let text = result.text(); assert!(text.starts_with("1 match(es).")); assert!(text.contains("\"name\": \"pdf_read\"")); assert!(text.contains("\"path\"")); } -#[test] -fn answer_tool_search_clamps_limit_and_handles_misses() { +#[tokio::test] +async fn answer_tool_search_clamps_limit_and_handles_misses() { let policy = ToolDiscoveryPolicy { max_limit: 2, ..ToolDiscoveryPolicy::default() }; - let (_, matched) = answer_tool_search( + let answer = answer_tool_search( &catalog(), &policy, &json!({"query": "pdf invite quote symbol attendees", "limit": 50}), - ); - assert!(matched <= 2); + ) + .await; + assert!(answer.matched <= 2); - let (result, matched) = answer_tool_search(&catalog(), &policy, &json!({"query": "zzzz qqqq"})); - assert!(!result.is_error); - assert_eq!(matched, 0); - assert!(result.text().starts_with("No deferred tool matches")); + let answer = answer_tool_search(&catalog(), &policy, &json!({"query": "zzzz qqqq"})).await; + assert!(!answer.result.is_error); + assert_eq!(answer.matched, 0); + assert!(answer.result.text().starts_with("No deferred tool matches")); - let (result, _) = answer_tool_search(&catalog(), &policy, &json!({"query": " "})); - assert!(result.is_error); + let answer = answer_tool_search(&catalog(), &policy, &json!({"query": " "})).await; + assert!(answer.result.is_error); + assert!(answer.ranking.is_none()); } /// Regression: `max_limit: 0` used to reach `usize::clamp(1, 0)`, which /// panics because its minimum exceeds its maximum — a model-supplied numeric /// `limit` could crash the process. It must instead clamp against a /// normalized effective maximum of at least 1. -#[test] -fn answer_tool_search_does_not_panic_on_a_zero_max_limit() { +#[tokio::test] +async fn answer_tool_search_does_not_panic_on_a_zero_max_limit() { let policy = ToolDiscoveryPolicy { max_limit: 0, default_limit: 5, ..ToolDiscoveryPolicy::default() }; - let (result, matched) = answer_tool_search( + let answer = answer_tool_search( &catalog(), &policy, &json!({"query": "pdf invite quote symbol attendees", "limit": 50}), + ) + .await; + assert!(!answer.result.is_error); + assert!( + answer.matched <= 1, + "effective max_limit must clamp to at least 1" ); - assert!(!result.is_error); - assert!(matched <= 1, "effective max_limit must clamp to at least 1"); +} + +/// A ranker that answers from a script: `Ok(keys)` or a failure. +struct ScriptedRanker { + answer: Result, &'static str>, + calls: std::sync::atomic::AtomicUsize, +} + +impl ScriptedRanker { + fn returning(keys: Vec<&'static str>) -> std::sync::Arc { + std::sync::Arc::new(Self { + answer: Ok(keys), + calls: std::sync::atomic::AtomicUsize::new(0), + }) + } + + fn failing(reason: &'static str) -> std::sync::Arc { + std::sync::Arc::new(Self { + answer: Err(reason), + calls: std::sync::atomic::AtomicUsize::new(0), + }) + } + + fn calls(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl tinytools::ToolRanker for ScriptedRanker { + fn kind(&self) -> &'static str { + "scripted" + } + + async fn rank( + &self, + _intent: &str, + _context: &tinytools::RankContext, + candidates: &[tinytools::RankCandidate], + limit: usize, + ) -> Result, tinytools::RankError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Every candidate carries the family the catalogue was built with. + assert!( + candidates + .iter() + .all(|c| c.family.as_deref() == Some("fam")) + ); + match &self.answer { + Ok(keys) => Ok(keys + .iter() + .take(limit) + .enumerate() + .map(|(i, key)| tinytools::RankHit { + key: (*key).to_string(), + score: 0.9 - i as f64 * 0.1, + confidence: Some(0.9 - i as f64 * 0.1), + }) + .collect()), + Err(reason) => Err(tinytools::RankError::Backend { + reason: (*reason).to_string(), + }), + } + } +} + +fn catalog_with_families() -> DeferredCatalog { + DeferredCatalog::build_with_families( + catalog() + .schemas() + .cloned() + .map(|schema| (schema, Some("fam".to_string()))) + .collect(), + ) +} + +#[tokio::test] +async fn host_ranker_is_served_with_its_confidence() { + let ranker = ScriptedRanker::returning(vec!["stock_quote", "pdf_read"]); + let policy = ToolDiscoveryPolicy::default().with_ranker(ranker.clone()); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf", "limit": 3}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "scripted"); + assert_eq!(ranking.names, vec!["stock_quote", "pdf_read"]); + assert_eq!(ranking.top_confidence, Some(0.9)); + assert!(ranking.fallback.is_none()); + assert!( + ranking.shadow_names.is_none(), + "no shadow outside compare mode" + ); + assert_eq!(answer.matched, 2); + assert!(answer.result.text().contains("\"name\": \"stock_quote\"")); + assert_eq!(ranker.calls(), 1); +} + +#[tokio::test] +async fn host_ranker_failure_falls_back_to_bm25_and_says_why() { + let policy = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::failing("503")); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!( + ranking.fallback.as_deref(), + Some("scripted failed: ranker backend failed: 503") + ); + assert_eq!(answer.matched, 1); +} + +#[tokio::test] +async fn host_ranker_empty_answer_falls_back_to_bm25() { + let policy = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::returning(vec![])); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!( + ranking.fallback.as_deref(), + Some("scripted returned no match") + ); +} + +#[tokio::test] +async fn compare_mode_serves_the_ranker_and_reports_bm25_alongside() { + let ranker = ScriptedRanker::returning(vec!["stock_quote"]); + let policy = ToolDiscoveryPolicy::default() + .with_ranker(ranker.clone()) + .with_rank_mode(DiscoveryRankMode::Compare); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "scripted"); + assert_eq!(ranking.names, vec!["stock_quote"]); + assert_eq!(ranking.shadow_names, Some(vec!["pdf_read".to_string()])); +} + +#[tokio::test] +async fn bm25_mode_ignores_an_installed_ranker() { + let ranker = ScriptedRanker::returning(vec!["stock_quote"]); + let policy = ToolDiscoveryPolicy::default() + .with_ranker(ranker.clone()) + .with_rank_mode(DiscoveryRankMode::Bm25); + assert!(policy.active_ranker().is_none()); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!(ranker.calls(), 0); +} + +#[test] +fn a_ranker_hit_naming_an_unknown_tool_is_dropped_from_the_answer() { + // `answer_tool_search` resolves names through `catalog.get`; a key the + // ranker invented never reaches the model. Pinned through the sync + // lookup so the guarantee is visible without a runtime. + assert!(catalog().get("invented").is_none()); +} + +#[test] +fn policy_equality_and_debug_compare_ranker_kinds_only() { + let a = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::returning(vec![])); + let b = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::failing("x")); + assert_eq!(a, b, "same kind, same knobs"); + assert_ne!(a, ToolDiscoveryPolicy::default()); + assert!(format!("{a:?}").contains("Some(\"scripted\")")); } /// Regression: the `tool_search` schema advertised `"minimum": 1, "maximum": diff --git a/crates/tinyagents-harness/src/tool/discover/types.rs b/crates/tinyagents-harness/src/tool/discover/types.rs index 0c3d0448f..2704f62fa 100644 --- a/crates/tinyagents-harness/src/tool/discover/types.rs +++ b/crates/tinyagents-harness/src/tool/discover/types.rs @@ -1,8 +1,10 @@ //! Types for on-demand tool discovery. -use tinyinference_llm::tool::ToolSchema; +use std::{fmt, sync::Arc}; -use super::index::Bm25Index; +use tinyinference_llm::tool::ToolSchema; +use tinytools::rank::Bm25Index; +use tinytools::{Bm25Ranker, RankCandidate, RankContext, RankError, ToolRanker}; /// How the agent loop exposes [`tinytools::ToolExposure::Deferred`] tools. /// @@ -13,7 +15,7 @@ use super::index::Bm25Index; /// array therefore stays byte-identical for the whole run, which is what a /// provider prompt cache keys on; revealing a schema costs one tool result, /// not a cache miss on every later turn. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone)] pub struct ToolDiscoveryPolicy { /// Whether the bridge tools are offered at all. /// @@ -33,6 +35,30 @@ pub struct ToolDiscoveryPolicy { /// Ceiling on the model-supplied `limit`, so one call cannot undo the /// saving by asking for everything. pub max_limit: usize, + /// What ranks the catalogue against a `tool_search` query. + /// + /// `None` is the built-in [`Bm25Ranker`]: free, deterministic, no network. + /// A host with a decision model or an embedding index installs it here and + /// chooses how it is used with [`Self::rank_mode`]. Whatever is installed, + /// a ranker failure falls back to BM25 — a search that errors would leave + /// every deferred tool unreachable for the turn. + pub ranker: Option>, + /// How [`Self::ranker`] and the built-in BM25 are combined. + pub rank_mode: DiscoveryRankMode, +} + +/// How a `tool_search` answer is produced when a host ranker is installed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum DiscoveryRankMode { + /// Serve the host ranker; fall back to BM25 when it fails. The default. + #[default] + Ranker, + /// Ignore the host ranker and serve BM25, as if none were installed. + Bm25, + /// Run both, serve the host ranker (BM25 on failure), and report the + /// BM25 ranking alongside in the `ToolSearched` event so the two can be + /// compared on live traffic without changing what the model sees. + Compare, } impl ToolDiscoveryPolicy { @@ -51,6 +77,31 @@ impl ToolDiscoveryPolicy { let default = self.default_limit.clamp(1, max); (default, max) } + + /// Installs a host ranker, keeping the current [`Self::rank_mode`]. + #[must_use] + pub fn with_ranker(mut self, ranker: Arc) -> Self { + self.ranker = Some(ranker); + self + } + + /// Sets how the host ranker is used. + #[must_use] + pub fn with_rank_mode(mut self, mode: DiscoveryRankMode) -> Self { + self.rank_mode = mode; + self + } + + /// The host ranker in force, or `None` when BM25 answers alone — either + /// because none is installed or because [`Self::rank_mode`] is + /// [`DiscoveryRankMode::Bm25`]. + #[must_use] + pub fn active_ranker(&self) -> Option<&Arc> { + match self.rank_mode { + DiscoveryRankMode::Bm25 => None, + DiscoveryRankMode::Ranker | DiscoveryRankMode::Compare => self.ranker.as_ref(), + } + } } impl Default for ToolDiscoveryPolicy { @@ -60,22 +111,52 @@ impl Default for ToolDiscoveryPolicy { manifest_token_budget: 4_000, default_limit: 5, max_limit: 20, + ranker: None, + rank_mode: DiscoveryRankMode::default(), } } } +impl fmt::Debug for ToolDiscoveryPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ToolDiscoveryPolicy") + .field("enabled", &self.enabled) + .field("manifest_token_budget", &self.manifest_token_budget) + .field("default_limit", &self.default_limit) + .field("max_limit", &self.max_limit) + .field("ranker", &self.ranker.as_ref().map(|r| r.kind())) + .field("rank_mode", &self.rank_mode) + .finish() + } +} + +impl PartialEq for ToolDiscoveryPolicy { + /// Two policies are equal when every knob matches and the installed + /// rankers are of the same kind; a ranker has no identity beyond that. + fn eq(&self, other: &Self) -> bool { + self.enabled == other.enabled + && self.manifest_token_budget == other.manifest_token_budget + && self.default_limit == other.default_limit + && self.max_limit == other.max_limit + && self.rank_mode == other.rank_mode + && self.ranker.as_ref().map(|r| r.kind()) == other.ranker.as_ref().map(|r| r.kind()) + } +} + /// One deferred tool as the catalogue sees it. #[derive(Clone, Debug)] pub struct DeferredTool { /// The model-facing schema (host-injected arguments already projected out). pub schema: ToolSchema, + /// The pack, toolkit, or server the tool came from, when the host said. + pub family: Option, /// `name` (also split into words) + description + top-level property /// names: the text the ranker sees. searchable: String, } impl DeferredTool { - fn from_schema(schema: ToolSchema) -> Self { + fn from_schema(schema: ToolSchema, family: Option) -> Self { let mut searchable = String::with_capacity(schema.description.len() + 64); searchable.push_str(&schema.name); searchable.push(' '); @@ -92,7 +173,22 @@ impl DeferredTool { searchable.push_str(key); } } - Self { schema, searchable } + Self { + schema, + family, + searchable, + } + } + + /// This tool as a [`RankCandidate`]: keyed by name, summarised by its + /// searchable text. + #[must_use] + pub fn candidate(&self) -> RankCandidate { + RankCandidate { + key: self.schema.name.clone(), + family: self.family.clone(), + summary: self.searchable.clone(), + } } } @@ -107,12 +203,39 @@ pub struct DeferredCatalog { index: Bm25Index, } +/// How a [`DeferredCatalog::rank`] answer was produced. +#[derive(Clone, Debug, PartialEq)] +pub struct RankedSearch { + /// Names of the matched tools, best first. + pub names: Vec, + /// [`ToolRanker::kind`] of what produced `names`. + pub ranker: &'static str, + /// The best hit's calibrated confidence, when the ranker gave one. + pub top_confidence: Option, + /// Why the host ranker was not served, when it was installed and active. + pub fallback: Option, + /// The BM25 ranking, when [`DiscoveryRankMode::Compare`] asked for it and + /// BM25 was not what was served. + pub shadow_names: Option>, + /// Wall time of the ranking, in milliseconds. + pub latency_ms: u64, +} + impl DeferredCatalog { - /// Indexes `schemas`, sorting them by name. + /// Indexes `schemas`, sorting them by name. No families. #[must_use] - pub fn build(mut schemas: Vec) -> Self { - schemas.sort_by(|left, right| left.name.cmp(&right.name)); - let tools: Vec = schemas.into_iter().map(DeferredTool::from_schema).collect(); + pub fn build(schemas: Vec) -> Self { + Self::build_with_families(schemas.into_iter().map(|schema| (schema, None)).collect()) + } + + /// Indexes `(schema, family)` pairs, sorting them by name. + #[must_use] + pub fn build_with_families(mut entries: Vec<(ToolSchema, Option)>) -> Self { + entries.sort_by(|left, right| left.0.name.cmp(&right.0.name)); + let tools: Vec = entries + .into_iter() + .map(|(schema, family)| DeferredTool::from_schema(schema, family)) + .collect(); let index = Bm25Index::build( tools .iter() @@ -138,6 +261,11 @@ impl DeferredCatalog { self.tools.iter().map(|tool| &tool.schema) } + /// Every deferred tool, sorted by name. + pub fn tools(&self) -> impl Iterator { + self.tools.iter() + } + /// Looks a deferred tool up by exact name. #[must_use] pub fn get(&self, name: &str) -> Option<&ToolSchema> { @@ -147,8 +275,8 @@ impl DeferredCatalog { .map(|index| &self.tools[index].schema) } - /// Ranks the catalogue against `query`, best first, at most `limit` hits. - /// Only positively scored tools are returned. + /// Ranks the catalogue against `query` with BM25, best first, at most + /// `limit` hits. Only positively scored tools are returned. #[must_use] pub fn search(&self, query: &str, limit: usize) -> Vec<&ToolSchema> { self.index @@ -157,4 +285,77 @@ impl DeferredCatalog { .map(|index| &self.tools[index].schema) .collect() } + + /// Ranks the catalogue as `policy` says: the host ranker when one is + /// active, BM25 otherwise, and BM25 as the fallback when the host ranker + /// fails or returns nothing it is sure of. + /// + /// Never errors: the catalogue always has BM25 to answer with, and a + /// search that fails would leave every deferred tool unreachable for the + /// turn. What went wrong is reported in [`RankedSearch::fallback`]. + pub async fn rank( + &self, + policy: &ToolDiscoveryPolicy, + query: &str, + context: &RankContext, + limit: usize, + ) -> RankedSearch { + let started = std::time::Instant::now(); + let Some(ranker) = policy.active_ranker() else { + return RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: None, + shadow_names: None, + latency_ms: elapsed_ms(started), + }; + }; + let candidates: Vec = + self.tools.iter().map(DeferredTool::candidate).collect(); + let hosted = ranker.rank(query, context, &candidates, limit).await; + let shadow_names = (policy.rank_mode == DiscoveryRankMode::Compare) + .then(|| self.search_names(query, limit)); + match hosted { + Ok(hits) if !hits.is_empty() => RankedSearch { + top_confidence: hits.first().and_then(|hit| hit.confidence), + names: hits.into_iter().map(|hit| hit.key).collect(), + ranker: ranker.kind(), + fallback: None, + shadow_names, + latency_ms: elapsed_ms(started), + }, + Ok(_) => RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: Some(format!("{} returned no match", ranker.kind())), + shadow_names: None, + latency_ms: elapsed_ms(started), + }, + Err(error) => RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: Some(describe_failure(ranker.kind(), &error)), + shadow_names: None, + latency_ms: elapsed_ms(started), + }, + } + } + + fn search_names(&self, query: &str, limit: usize) -> Vec { + self.search(query, limit) + .into_iter() + .map(|schema| schema.name.clone()) + .collect() + } +} + +fn describe_failure(kind: &str, error: &RankError) -> String { + format!("{kind} failed: {error}") +} + +fn elapsed_ms(started: std::time::Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) } diff --git a/crates/tinyagents-harness/src/tool/mod.rs b/crates/tinyagents-harness/src/tool/mod.rs index e898085c3..1113432d5 100644 --- a/crates/tinyagents-harness/src/tool/mod.rs +++ b/crates/tinyagents-harness/src/tool/mod.rs @@ -320,6 +320,27 @@ impl ToolRegistry { self.schemas_with_exposure(tinytools::ToolExposure::Deferred) } + /// [`Self::deferred_schemas`] paired with each tool's + /// [`tinytools::Tool::family`], name-sorted, so the discovery catalogue + /// can say where a hit came from. + #[must_use] + pub fn deferred_schemas_with_families( + &self, + ) -> Vec<(tinyinference_llm::tool::ToolSchema, Option)> { + let mut entries: Vec<_> = self + .tools + .values() + .map(|dispatch| dispatch.tool()) + .filter(|tool| tool.exposure() == tinytools::ToolExposure::Deferred) + .map(|tool| { + let family = tool.family().map(str::to_owned); + (provider_schema(tool.as_ref()), family) + }) + .collect(); + entries.sort_by(|left, right| left.0.name.cmp(&right.0.name)); + entries + } + fn schemas_with_exposure( &self, exposure: tinytools::ToolExposure, diff --git a/crates/tinyagents-integration-tests/Cargo.toml b/crates/tinyagents-integration-tests/Cargo.toml index 70de25121..741185832 100644 --- a/crates/tinyagents-integration-tests/Cargo.toml +++ b/crates/tinyagents-integration-tests/Cargo.toml @@ -26,7 +26,7 @@ tinyagents-registry = { path = "../tinyagents-registry" } tinyagents-session = { path = "../tinyagents-session" } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } tinyinference-embeddings = { path = "../../vendor/tinyinference/crates/tinyinference-embeddings", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread", "time", "test-util"] } [features] diff --git a/crates/tinyagents-orchestration/Cargo.toml b/crates/tinyagents-orchestration/Cargo.toml index 1157dc5b2..fa2c0f1e0 100644 --- a/crates/tinyagents-orchestration/Cargo.toml +++ b/crates/tinyagents-orchestration/Cargo.toml @@ -21,7 +21,7 @@ tinyagents-harness = { path = "../tinyagents-harness", version = "2.1.2", defaul tinyagents-session = { path = "../tinyagents-session", version = "2.1.2", default-features = false } tinyagents-runtime = { path = "../tinyagents-runtime", version = "2.1.2", default-features = false } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } tokio = { workspace = true, features = ["sync", "rt", "macros"] } uuid = { version = "1", features = ["v4"] } diff --git a/crates/tinyagents-registry/Cargo.toml b/crates/tinyagents-registry/Cargo.toml index d58f44066..a8d674d9a 100644 --- a/crates/tinyagents-registry/Cargo.toml +++ b/crates/tinyagents-registry/Cargo.toml @@ -16,7 +16,7 @@ serde_json = { workspace = true } tinyagents-definition = { path = "../tinyagents-definition", version = "2.1.2" } tinyagents-harness = { path = "../tinyagents-harness", version = "2.1.2", default-features = false } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } [features] default = [] diff --git a/crates/tinyagents-runtime/Cargo.toml b/crates/tinyagents-runtime/Cargo.toml index 1fdaf79ef..96dcf9a18 100644 --- a/crates/tinyagents-runtime/Cargo.toml +++ b/crates/tinyagents-runtime/Cargo.toml @@ -15,7 +15,7 @@ thiserror = "2" tinyagents-harness = { path = "../tinyagents-harness", version = "2.1.2" } tinyagents-session = { path = "../tinyagents-session", version = "2.1.2" } tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" } -tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" } +tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" } tokio = { workspace = true, features = ["macros", "rt", "sync"] } [dev-dependencies] diff --git a/docs/modules/harness/tool-discovery.md b/docs/modules/harness/tool-discovery.md index 6cfbd7edd..17ed2fb3b 100644 --- a/docs/modules/harness/tool-discovery.md +++ b/docs/modules/harness/tool-discovery.md @@ -35,10 +35,11 @@ a tool it does not know it needs. When a run has at least one deferred tool (after the host allow-list), the agent loop appends two intrinsic tools **after** the name-sorted direct set: -- `tool_search { query, limit }` — BM25 over name, split identifier, description - and top-level property names. Returns up to `limit` matches (default 5, max - 20) as `{name, description, parameters}` with the **full** schema, or a - "no match" note. Its description embeds a manifest of every deferred tool: +- `tool_search { query, limit }` — ranks the catalogue's name, split + identifier, description, top-level property names and `Tool::family`. + Returns up to `limit` matches (default 5, max 20) as `{name, description, + parameters}` with the **full** schema, or a "no match" note. See + [Ranking](#ranking) for what does the ranking. Its description embeds a manifest of every deferred tool: `- name: first sentence (≤ 60 chars)`, degrading to names only, then to a bare count, until it fits `ToolDiscoveryPolicy::manifest_token_budget` (default 4,000 tokens). @@ -55,6 +56,27 @@ max_limit }`). With `enabled: false` deferred tools are neither advertised nor searchable, but a direct call by name still runs: deferral only ever subtracts from the wire, never from what the host registered. +### Ranking + +`DeferredCatalog::rank` answers as `ToolDiscoveryPolicy` says: + +- With no `ranker` installed (the default), BM25 from `tinytools::rank` — + free, deterministic, no network. `Bm25Index` and `tokenize` moved to that + crate so a host ranks with the same arithmetic the bridge does. +- With a host `ranker: Arc` (a decision model such + as `tinytools-jev`, or an embedding index), `rank_mode` decides: + `Ranker` serves it; `Bm25` ignores it; `Compare` serves it and reports the + BM25 ranking alongside in `ToolSearched.shadow_matched` so the two can be + judged on live traffic without changing what the model sees. +- A host ranker that fails or returns nothing **falls back to BM25** and + the reason lands in `ToolSearched.fallback`. A search never errors: an + error would leave every deferred tool unreachable for the turn. + +Every hit the ranker names is resolved through `catalog.get`, so a key the +ranker invented never reaches the model. The catalogue is what the ranker +sees; the model's `query` is the only intent, with an empty `RankContext` — +the model already distilled the turn into it. + ### Why a bridge and not hydration OpenClaw appends a revealed schema to the request's `tools` for the rest of the @@ -114,8 +136,11 @@ this flag. exposure-narrowing middleware (`ToolPolicyMiddleware::before_model`, dynamic/contextual selection) can still shrink an individual request below it. -- `ToolSearched { call_id, query, matched }` and - `DeferredToolCall { call_id, tool_name }` — every discovery, auditable. +- `ToolSearched { call_id, query, matched, ranker, top_confidence, fallback, + shadow_matched, latency_ms }` and `DeferredToolCall { call_id, tool_name }` + — every discovery, auditable: which ranker answered, how sure it was, why + a host ranker was not served, and what BM25 would have said in compare + mode. `query` is recorded only under `RunPolicy::capture.tool_io`. ## Schema budgets diff --git a/vendor/tinytools b/vendor/tinytools index 970bfae38..fbd7bd4bb 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 970bfae384945317d6e901beb5c4e00c2028c0c6 +Subproject commit fbd7bd4bbada687d354d26be7c1bb0ba11191417