Recover mixed DeepSeek DSML tool calls

crates/tinytools-agent/src/parse/grammar/tagged.rs

@@ -37,6 +37,7 @@ use crate::parse::call_object::{AliasPolicy, read_calls};
 use crate::parse::json_values::{
     extract_first_json_value_with_end, extract_json_values, find_json_end,
 };
+use crate::repair::args::from_call_object;
 use crate::repair::json::{recover_object, strip_code_fence};
 use crate::types::{CallSource, ParseOptions, ParsedToolCall};
 
@@ -94,6 +95,19 @@ static NAMED_INVOKE_OPEN_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
     .ok()
 });
 
+/// The malformed name-parameter spelling emitted after JSON arguments by
+/// `DeepSeek` when its DSML stream leaks into a text `<tool_call>` block.
+static DSML_NAME_PARAMETER_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
+    Regex::new(
+        r#"(?is)^\s*(?:</[|｜]{1,2}\s*DSML\s*[|｜]{1,2}\s*parameter\s*>\s*)*<[|｜]{1,2}\s*DSML\s*[|｜]{1,2}\s*parameter\s+name\s*=\s*"name"\s*:\s*"([^"]+)"[^>]*(?:>|})"#,
+    )
+    .ok()
+});
+
+/// `DeepSeek`'s group terminator, used to leave following assistant prose intact.
+static DSML_CALLS_CLOSE_RE: LazyLock<Option<Regex>> =
+    LazyLock::new(|| Regex::new(r"(?is)</[|｜]{1,2}\s*DSML\s*[|｜]{1,2}\s*calls\s*>").ok());
+
 /// First match of `re` in `haystack`, as `(start, end)`.
 fn find_re(re: &LazyLock<Option<Regex>>, haystack: &str) -> Option<(usize, usize)> {
     re.as_ref()
@@ -340,6 +354,14 @@ impl Tagged {
             })
             .or_else(|| extract_first_json_value_with_end(after));
         if let Some((value, consumed)) = recovered {
+            let (dsml_calls, dsml_end) = recover_dsml_calls(after);
+            if !dsml_calls.is_empty() {
+                return Probe::Found(Block {
+                    start: opener.start,
+                    end: body_start + dsml_end,
+                    decoded: Decoded::Calls(dsml_calls),
+                });
+            }
             let calls = read_calls(
                 &value,
                 AliasPolicy::Marked,
@@ -382,6 +404,55 @@ impl Tagged {
     }
 }
 
+/// Recovers JSON argument objects followed by `DeepSeek`'s name parameter.
+/// JSON boundaries are found before scanning markup, so tag-like text inside
+/// string arguments cannot be mistaken for DSML framing.
+fn recover_dsml_calls(after: &str) -> (Vec<ParsedToolCall>, usize) {
+    let Some(re) = DSML_NAME_PARAMETER_RE.as_ref() else {
+        return (Vec::new(), 0);
+    };
+    let mut calls = Vec::new();
+    let mut cursor = 0;
+    while cursor < after.len() {
+        let Some((arguments, json_end)) = extract_first_json_value_with_end(&after[cursor..])
+        else {
+            break;
+        };
+        let json_end = cursor + json_end;
+        let metadata = &after[json_end..];
+        let inline_name = arguments.get("name").and_then(serde_json::Value::as_str);
+        let name_match = re.captures(metadata);
+        let name = inline_name.or_else(|| {
+            name_match
+                .as_ref()
+                .and_then(|m| m.get(1).map(|v| v.as_str()))
+        });
+        let Some(name) = name.map(str::trim) else {
+            break;
+        };
+        if name.is_empty() {
+            break;
+        }
+        let name = name.to_owned();
+        let args = from_call_object(&arguments);
+        calls.push(ParsedToolCall::new(name, args, CallSource::TaggedJson));
+        // An inline name identifies this JSON value, but does not frame a
+        // later value in trailing text. Continue only when DSML explicitly
+        // connects the values with a name parameter.
+        let Some(name_match) = name_match else {
+            cursor = json_end;
+            break;
+        };
+        cursor = json_end + name_match.get(0).map_or(0, |m| m.end());
+    }
+    let end = if calls.is_empty() {
+        0
+    } else {
+        find_re(&DSML_CALLS_CLOSE_RE, &after[cursor..]).map_or(cursor, |(_, end)| cursor + end)
+    };
+    (calls, end)
+}
+
 /// Whether a tag-family marker is a closer (`</tool_call>`, `<|/tool_call|>`).
 /// Skips the same whitespace class `TAG_RE`'s `\s` does (not just space and
 /// tab), so a marker like `<\n/tool_call>` — which the regex matches as one

crates/tinytools-agent/src/parse/test/tagged.rs

@@ -80,6 +80,33 @@ fn unclosed_tag_with_balanced_json_still_recovers() {
     assert_eq!(calls.len(), 1);
 }
 
+#[test]
+fn unclosed_tag_recovers_dsml_parameter_calls() {
+    let response = concat!(
+        "<tool_call>\n",
+        r#"{"arguments":{"path":"work/extract.py","content":"t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S)"}}"#,
+        "</｜DSML｜ parameter>\n",
+        "<｜DSML｜ parameter name=\"name\":\"file_write\"}</｜DSML｜ parameter>\n",
+        "</｜DSML｜ invoke>\n",
+        "<｜DSML｜ invoke>\n",
+        r#"{"arguments":{"command":"ls work"}}"#,
+        "</｜DSML｜ parameter>\n",
+        "<｜DSML｜ parameter name=\"name\":\"shell\"}</｜DSML｜ parameter>\n",
+        "</｜DSML｜ invoke>\n",
+        "</｜DSML｜ calls>Done.",
+    );
+    let (text, calls) = parse(response);
+    assert_eq!(text, "Done.");
+    assert_eq!(calls.len(), 2);
+    assert_eq!(calls[0].name, "file_write");
+    assert_eq!(
+        calls[0].arguments["content"],
+        "t = re.sub(r'<script.*?</script>', ' ', t, flags=re.S)"
+    );
+    assert_eq!(calls[1].name, "shell");
+    assert_eq!(calls[1].arguments["command"], "ls work");
+}
+
 #[test]
 fn unclosed_tag_recovery_preserves_unrelated_markup_after_the_json() {
     // No tag-family marker exists anywhere after this opener, so a `</div>`
@@ -632,12 +659,8 @@ fn a_bare_dsml_invoke_carries_its_name_in_the_body() {
 /// A block that decodes to nothing must not bury the calls after it.
 ///
 /// Verbatim from a `deepseek` turn: an unterminated `<tool_call>` whose body is
-/// `{"arguments":{…}}` with no name — genuinely unrecoverable, since the name
-/// survived only in a corrupted `<｜DSML｜ parameter name="name":"file_write"}`
-/// line and inventing one is never right — followed by a complete
-/// `<｜DSML｜ invoke>`. The failed block used to run to end-of-text and take
-/// the good call with it, so the whole response parsed as prose and both calls
-/// were lost.
+/// `{"arguments":{…}}` with a name in a corrupted DSML parameter line,
+/// followed by a complete `<｜DSML｜ invoke>`. Both calls should survive.
 #[test]
 fn an_undecodable_block_does_not_swallow_the_call_after_it() {
     let raw = concat!(
@@ -656,11 +679,12 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() {
     let outcome = super::parse_known(raw, &["file_write", "shell"]);
     assert_eq!(
         outcome.calls.len(),
-        1,
-        "the well-formed call survives its malformed neighbour: {:?}",
+        2,
+        "both calls survive their mixed framing: {:?}",
         outcome.calls
     );
-    assert_eq!(outcome.calls[0].name, "shell");
+    assert_eq!(outcome.calls[0].name, "file_write");
+    assert_eq!(outcome.calls[1].name, "shell");
 
     // Reduced to the essential shape, so a future change that reintroduces the
     // swallow fails here with less noise.
@@ -717,6 +741,30 @@ fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() {
     assert_eq!(outcome.calls[0].name, "shell");
 }
 
+#[test]
+fn unclosed_inline_name_does_not_scan_later_json_from_narrative() {
+    let raw = concat!(
+        "<tool_call>{\"name\":\"first\",\"arguments\":{}} prose ",
+        "{\"name\":\"shell\",\"arguments\":{\"command\":\"ls\"}}"
+    );
+    let (text, calls) = parse(raw);
+    assert_eq!(calls.len(), 1, "{calls:?}");
+    assert_eq!(calls[0].name, "first");
+    assert!(text.contains("prose"));
+}
+
+#[test]
+fn unclosed_inline_name_decodes_stringified_and_aliased_arguments() {
+    for raw in [
+        r#"<tool_call>{"name":"echo","arguments":"{\"value\":\"x\"}"}"#,
+        r#"<tool_call>{"name":"echo","args":{"value":"x"}}"#,
+    ] {
+        let (_, calls) = parse(raw);
+        assert_eq!(calls.len(), 1, "{calls:?}");
+        assert_eq!(calls[0].arguments, serde_json::json!({"value": "x"}));
+    }
+}
+
 #[test]
 fn a_named_invoke_precedes_a_later_bare_invoke_closer() {
     let raw = concat!(