diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 95dd933..5a18d16 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -37,6 +37,7 @@ use crate::parse::call_object::{AliasPolicy, read_calls}; use crate::parse::json_values::{ extract_first_json_value_with_end, extract_json_values, find_json_end, }; +use crate::repair::args::from_call_object; use crate::repair::json::{recover_object, strip_code_fence}; use crate::types::{CallSource, ParseOptions, ParsedToolCall}; @@ -94,6 +95,19 @@ static NAMED_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { .ok() }); +/// The malformed name-parameter spelling emitted after JSON arguments by +/// `DeepSeek` when its DSML stream leaks into a text `` block. +static DSML_NAME_PARAMETER_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?is)^\s*(?:\s*)*<[||]{1,2}\s*DSML\s*[||]{1,2}\s*parameter\s+name\s*=\s*"name"\s*:\s*"([^"]+)"[^>]*(?:>|})"#, + ) + .ok() +}); + +/// `DeepSeek`'s group terminator, used to leave following assistant prose intact. +static DSML_CALLS_CLOSE_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?is)").ok()); + /// First match of `re` in `haystack`, as `(start, end)`. fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize)> { re.as_ref() @@ -340,6 +354,14 @@ impl Tagged { }) .or_else(|| extract_first_json_value_with_end(after)); if let Some((value, consumed)) = recovered { + let (dsml_calls, dsml_end) = recover_dsml_calls(after); + if !dsml_calls.is_empty() { + return Probe::Found(Block { + start: opener.start, + end: body_start + dsml_end, + decoded: Decoded::Calls(dsml_calls), + }); + } let calls = read_calls( &value, AliasPolicy::Marked, @@ -382,6 +404,55 @@ impl Tagged { } } +/// Recovers JSON argument objects followed by `DeepSeek`'s name parameter. +/// JSON boundaries are found before scanning markup, so tag-like text inside +/// string arguments cannot be mistaken for DSML framing. +fn recover_dsml_calls(after: &str) -> (Vec, usize) { + let Some(re) = DSML_NAME_PARAMETER_RE.as_ref() else { + return (Vec::new(), 0); + }; + let mut calls = Vec::new(); + let mut cursor = 0; + while cursor < after.len() { + let Some((arguments, json_end)) = extract_first_json_value_with_end(&after[cursor..]) + else { + break; + }; + let json_end = cursor + json_end; + let metadata = &after[json_end..]; + let inline_name = arguments.get("name").and_then(serde_json::Value::as_str); + let name_match = re.captures(metadata); + let name = inline_name.or_else(|| { + name_match + .as_ref() + .and_then(|m| m.get(1).map(|v| v.as_str())) + }); + let Some(name) = name.map(str::trim) else { + break; + }; + if name.is_empty() { + break; + } + let name = name.to_owned(); + let args = from_call_object(&arguments); + calls.push(ParsedToolCall::new(name, args, CallSource::TaggedJson)); + // An inline name identifies this JSON value, but does not frame a + // later value in trailing text. Continue only when DSML explicitly + // connects the values with a name parameter. + let Some(name_match) = name_match else { + cursor = json_end; + break; + }; + cursor = json_end + name_match.get(0).map_or(0, |m| m.end()); + } + let end = if calls.is_empty() { + 0 + } else { + find_re(&DSML_CALLS_CLOSE_RE, &after[cursor..]).map_or(cursor, |(_, end)| cursor + end) + }; + (calls, end) +} + /// Whether a tag-family marker is a closer (``, `<|/tool_call|>`). /// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and /// tab), so a marker like `<\n/tool_call>` — which the regex matches as one diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 22ccd91..2a8372f 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -80,6 +80,33 @@ fn unclosed_tag_with_balanced_json_still_recovers() { assert_eq!(calls.len(), 1); } +#[test] +fn unclosed_tag_recovers_dsml_parameter_calls() { + let response = concat!( + "\n", + r#"{"arguments":{"path":"work/extract.py","content":"t = re.sub(r'', ' ', t, flags=re.S)"}}"#, + "\n", + "<|DSML| parameter name=\"name\":\"file_write\"}\n", + "\n", + "<|DSML| invoke>\n", + r#"{"arguments":{"command":"ls work"}}"#, + "\n", + "<|DSML| parameter name=\"name\":\"shell\"}\n", + "\n", + "Done.", + ); + let (text, calls) = parse(response); + assert_eq!(text, "Done."); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "file_write"); + assert_eq!( + calls[0].arguments["content"], + "t = re.sub(r'', ' ', t, flags=re.S)" + ); + assert_eq!(calls[1].name, "shell"); + assert_eq!(calls[1].arguments["command"], "ls work"); +} + #[test] fn unclosed_tag_recovery_preserves_unrelated_markup_after_the_json() { // No tag-family marker exists anywhere after this opener, so a `` @@ -632,12 +659,8 @@ fn a_bare_dsml_invoke_carries_its_name_in_the_body() { /// A block that decodes to nothing must not bury the calls after it. /// /// Verbatim from a `deepseek` turn: an unterminated `` whose body is -/// `{"arguments":{…}}` with no name — genuinely unrecoverable, since the name -/// survived only in a corrupted `<|DSML| parameter name="name":"file_write"}` -/// line and inventing one is never right — followed by a complete -/// `<|DSML| invoke>`. The failed block used to run to end-of-text and take -/// the good call with it, so the whole response parsed as prose and both calls -/// were lost. +/// `{"arguments":{…}}` with a name in a corrupted DSML parameter line, +/// followed by a complete `<|DSML| invoke>`. Both calls should survive. #[test] fn an_undecodable_block_does_not_swallow_the_call_after_it() { let raw = concat!( @@ -656,11 +679,12 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() { let outcome = super::parse_known(raw, &["file_write", "shell"]); assert_eq!( outcome.calls.len(), - 1, - "the well-formed call survives its malformed neighbour: {:?}", + 2, + "both calls survive their mixed framing: {:?}", outcome.calls ); - assert_eq!(outcome.calls[0].name, "shell"); + assert_eq!(outcome.calls[0].name, "file_write"); + assert_eq!(outcome.calls[1].name, "shell"); // Reduced to the essential shape, so a future change that reintroduces the // swallow fails here with less noise. @@ -717,6 +741,30 @@ fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() { assert_eq!(outcome.calls[0].name, "shell"); } +#[test] +fn unclosed_inline_name_does_not_scan_later_json_from_narrative() { + let raw = concat!( + "{\"name\":\"first\",\"arguments\":{}} prose ", + "{\"name\":\"shell\",\"arguments\":{\"command\":\"ls\"}}" + ); + let (text, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "first"); + assert!(text.contains("prose")); +} + +#[test] +fn unclosed_inline_name_decodes_stringified_and_aliased_arguments() { + for raw in [ + r#"{"name":"echo","arguments":"{\"value\":\"x\"}"}"#, + r#"{"name":"echo","args":{"value":"x"}}"#, + ] { + let (_, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"})); + } +} + #[test] fn a_named_invoke_precedes_a_later_bare_invoke_closer() { let raw = concat!(