From e03cef21f62041952841964b2cc9ca2e84723cf4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 30 Sep 2026 09:39:13 +0300 Subject: [PATCH 1/4] Recover mixed DeepSeek DSML tool calls Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 66 +++++++++++++++++++ .../tinytools-agent/src/parse/test/tagged.rs | 42 +++++++++--- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 95dd933..4a8c6af 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -94,6 +94,19 @@ static NAMED_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { .ok() }); +/// The malformed name-parameter spelling emitted after JSON arguments by +/// `DeepSeek` when its DSML stream leaks into a text `` block. +static DSML_NAME_PARAMETER_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?is)^\s*(?:\s*)*<[|｜]{1,2}\s*DSML\s*[|｜]{1,2}\s*parameter\s+name\s*=\s*"name"\s*:\s*"([^"]+)"[^>]*(?:>|})"#, + ) + .ok() +}); + +/// `DeepSeek`'s group terminator, used to leave following assistant prose intact. +static DSML_CALLS_CLOSE_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?is)").ok()); + /// First match of `re` in `haystack`, as `(start, end)`. fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize)> { re.as_ref() @@ -340,6 +353,14 @@ impl Tagged { }) .or_else(|| extract_first_json_value_with_end(after)); if let Some((value, consumed)) = recovered { + let (dsml_calls, dsml_end) = recover_dsml_calls(after); + if !dsml_calls.is_empty() { + return Probe::Found(Block { + start: opener.start, + end: body_start + dsml_end, + decoded: Decoded::Calls(dsml_calls), + }); + } let calls = read_calls( &value, AliasPolicy::Marked, @@ -382,6 +403,51 @@ impl Tagged { } } +/// Recovers JSON argument objects followed by `DeepSeek`'s name parameter. +/// JSON boundaries are found before scanning markup, so tag-like text inside +/// string arguments cannot be mistaken for DSML framing. +fn recover_dsml_calls(after: &str) -> (Vec, usize) { + let Some(re) = DSML_NAME_PARAMETER_RE.as_ref() else { + return (Vec::new(), 0); + }; + let mut calls = Vec::new(); + let mut cursor = 0; + while cursor < after.len() { + let Some((arguments, json_end)) = extract_first_json_value_with_end(&after[cursor..]) + else { + break; + }; + let json_end = cursor + json_end; + let metadata = &after[json_end..]; + let inline_name = arguments.get("name").and_then(serde_json::Value::as_str); + let name_match = re.captures(metadata); + let name = inline_name.or_else(|| { + name_match + .as_ref() + .and_then(|m| m.get(1).map(|v| v.as_str())) + }); + let Some(name) = name.map(str::trim) else { + break; + }; + if name.is_empty() { + break; + } + let name = name.to_owned(); + let args = arguments.get("arguments").cloned().unwrap_or(arguments); + calls.push(ParsedToolCall::new(name, args, CallSource::TaggedJson)); + cursor = json_end + + name_match + .and_then(|m| m.get(0).map(|v| v.end())) + .unwrap_or(0); + } + let end = if calls.is_empty() { + 0 + } else { + find_re(&DSML_CALLS_CLOSE_RE, &after[cursor..]).map_or(cursor, |(_, end)| cursor + end) + }; + (calls, end) +} + /// Whether a tag-family marker is a closer (``, `<|/tool_call|>`). /// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and /// tab), so a marker like `<\n/tool_call>` — which the regex matches as one diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 22ccd91..d7842c6 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -80,6 +80,33 @@ fn unclosed_tag_with_balanced_json_still_recovers() { assert_eq!(calls.len(), 1); } +#[test] +fn unclosed_tag_recovers_dsml_parameter_calls() { + let response = concat!( + "\n", + r#"{"arguments":{"path":"work/extract.py","content":"t = re.sub(r'', ' ', t, flags=re.S)"}}"#, + "\n", + "<｜DSML｜ parameter name=\"name\":\"file_write\"}\n", + "\n", + "<｜DSML｜ invoke>\n", + r#"{"arguments":{"command":"ls work"}}"#, + "\n", + "<｜DSML｜ parameter name=\"name\":\"shell\"}\n", + "\n", + "Done.", + ); + let (text, calls) = parse(response); + assert_eq!(text, "Done."); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "file_write"); + assert_eq!( + calls[0].arguments["content"], + "t = re.sub(r'', ' ', t, flags=re.S)" + ); + assert_eq!(calls[1].name, "shell"); + assert_eq!(calls[1].arguments["command"], "ls work"); +} + #[test] fn unclosed_tag_recovery_preserves_unrelated_markup_after_the_json() { // No tag-family marker exists anywhere after this opener, so a `` @@ -632,12 +659,8 @@ fn a_bare_dsml_invoke_carries_its_name_in_the_body() { /// A block that decodes to nothing must not bury the calls after it. /// /// Verbatim from a `deepseek` turn: an unterminated `` whose body is -/// `{"arguments":{…}}` with no name — genuinely unrecoverable, since the name -/// survived only in a corrupted `<｜DSML｜ parameter name="name":"file_write"}` -/// line and inventing one is never right — followed by a complete -/// `<｜DSML｜ invoke>`. The failed block used to run to end-of-text and take -/// the good call with it, so the whole response parsed as prose and both calls -/// were lost. +/// `{"arguments":{…}}` with a name in a corrupted DSML parameter line, +/// followed by a complete `<｜DSML｜ invoke>`. Both calls should survive. #[test] fn an_undecodable_block_does_not_swallow_the_call_after_it() { let raw = concat!( @@ -656,11 +679,12 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() { let outcome = super::parse_known(raw, &["file_write", "shell"]); assert_eq!( outcome.calls.len(), - 1, - "the well-formed call survives its malformed neighbour: {:?}", + 2, + "both calls survive their mixed framing: {:?}", outcome.calls ); - assert_eq!(outcome.calls[0].name, "shell"); + assert_eq!(outcome.calls[0].name, "file_write"); + assert_eq!(outcome.calls[1].name, "shell"); // Reduced to the essential shape, so a future change that reintroduces the // swallow fails here with less noise. From c5d4f3d7c216fb23c695c4712e2dc341e1d12407 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 30 Sep 2026 10:06:55 +0300 Subject: [PATCH 2/4] fix(parse): stop scanning past unclosed inline name in DSML recovery When a DSML tool call uses an inline name but no explicit name parameter, the recovery loop was incorrectly advancing the cursor past the JSON value and scanning later narrative text for additional calls. This caused spurious extra calls to be extracted from prose that happened to contain JSON-like content. The fix stops the loop after the first call when no name match is present, and also applies the `from_call_object` helper to properly decode stringified and aliased arguments. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/parse/grammar/tagged.rs | 13 ++++++---- .../tinytools-agent/src/parse/test/tagged.rs | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 4a8c6af..213f542 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -37,6 +37,7 @@ use crate::parse::call_object::{AliasPolicy, read_calls}; use crate::parse::json_values::{ extract_first_json_value_with_end, extract_json_values, find_json_end, }; +use crate::repair::args::from_call_object; use crate::repair::json::{recover_object, strip_code_fence}; use crate::types::{CallSource, ParseOptions, ParsedToolCall}; @@ -433,12 +434,16 @@ fn recover_dsml_calls(after: &str) -> (Vec, usize) { break; } let name = name.to_owned(); - let args = arguments.get("arguments").cloned().unwrap_or(arguments); + let args = from_call_object(&arguments); calls.push(ParsedToolCall::new(name, args, CallSource::TaggedJson)); + // An inline name identifies this JSON value, but does not frame a + // later value in trailing text. Continue only when DSML explicitly + // connects the values with a name parameter. + let Some(name_match) = name_match else { + break; + }; cursor = json_end - + name_match - .and_then(|m| m.get(0).map(|v| v.end())) - .unwrap_or(0); + + name_match.get(0).map_or(0, |m| m.end()); } let end = if calls.is_empty() { 0 diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index d7842c6..2a8372f 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -741,6 +741,30 @@ fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() { assert_eq!(outcome.calls[0].name, "shell"); } +#[test] +fn unclosed_inline_name_does_not_scan_later_json_from_narrative() { + let raw = concat!( + "{\"name\":\"first\",\"arguments\":{}} prose ", + "{\"name\":\"shell\",\"arguments\":{\"command\":\"ls\"}}" + ); + let (text, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "first"); + assert!(text.contains("prose")); +} + +#[test] +fn unclosed_inline_name_decodes_stringified_and_aliased_arguments() { + for raw in [ + r#"{"name":"echo","arguments":"{\"value\":\"x\"}"}"#, + r#"{"name":"echo","args":{"value":"x"}}"#, + ] { + let (_, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"})); + } +} + #[test] fn a_named_invoke_precedes_a_later_bare_invoke_closer() { let raw = concat!( From c07a515695c2483d103df3e6b8a48fc23e03149e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 30 Sep 2026 10:07:09 +0300 Subject: [PATCH 3/4] fix(parse): fix cursor offset calculation in DSML recovery The cursor offset was incorrectly split across two lines, causing a syntax error in the recovery logic for DSML tool calls. This change consolidates the expression onto a single line to ensure correct parsing and offset computation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/grammar/tagged.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 213f542..bd594b5 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -442,8 +442,7 @@ fn recover_dsml_calls(after: &str) -> (Vec, usize) { let Some(name_match) = name_match else { break; }; - cursor = json_end - + name_match.get(0).map_or(0, |m| m.end()); + cursor = json_end + name_match.get(0).map_or(0, |m| m.end()); } let end = if calls.is_empty() { 0 From 0e1f94c7a38e20d502ce97f7229b8bf5b85067e4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 30 Sep 2026 10:07:46 +0300 Subject: [PATCH 4/4] chore: files changed crates/tinytools-agent/src/parse/grammar/tagged.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/grammar/tagged.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index bd594b5..5a18d16 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -440,6 +440,7 @@ fn recover_dsml_calls(after: &str) -> (Vec, usize) { // later value in trailing text. Continue only when DSML explicitly // connects the values with a name parameter. let Some(name_match) = name_match else { + cursor = json_end; break; }; cursor = json_end + name_match.get(0).map_or(0, |m| m.end());