diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 1b4249bec..b5da1fa66 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -807,6 +807,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s // and 500s a request can earn before it has produced anything keep their status codes -- and a stream. void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { + // A non-causal decision pack has no decode to run (npu::decision_only_refusal): say so before any work. + if (e.fwd && e.fwd->noncausal()) { + res.status = 400; + res.set_content(npu::decision_only_refusal(e.id, e.head != nullptr).dump(), "application/json"); + return; + } bool wants_stream = false; try { const json body = json::parse(req.body); diff --git a/app/npu_systemone.h b/app/npu_systemone.h index 22623f7cc..2a6c0bfd4 100644 --- a/app/npu_systemone.h +++ b/app/npu_systemone.h @@ -124,6 +124,22 @@ inline nlohmann::json systemone_question_error(const std::string& kind, const st {"message", message}}; } +// What a text route answers on a pack that cannot generate text. A decision pack that declares +// `attention_mode: noncausal_full_attention` encodes a whole prompt in one pass and reads an answer off the last +// position; there is no token-by-token decode to run, so `/v1/chat/completions` and `/v1/completions` can never +// succeed on it. They used to try, and failed inside the forward with HTTP 500 and the forward's own wording +// ("forward step failed: attention.mode noncausal_full_attention encodes a prompt in one batched pass, but this +// call starts at position 23 ..."; measured on FastFlowLM's D3-Nano-NPU2, engine#1786). That is the client's +// request against the wrong route, so it is a 400 that names the right one. `has_head` says whether the decision +// route is actually up on this serve, because pointing a client at a route that answers 503 is no help. +inline nlohmann::json decision_only_refusal(const std::string& model, bool has_head) { + const std::string where = has_head ? "send decisions to POST /v1/systemone" + : "and this serve has no decision head loaded, so it has no route for it"; + return {{"error", + {{"message", model + " is a decision model (non-causal attention): it does not generate text; " + where}, + {"type", "decision_model_error"}}}}; +} + inline std::string systemone_length_message(size_t tokens, size_t limit) { return "the question prompt has " + std::to_string(tokens) + " tokens, over the maximum context length of " + std::to_string(limit) + " tokens; nothing was truncated"; diff --git a/docs/npu.md b/docs/npu.md index 85857e81f..922997b93 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2107,6 +2107,11 @@ forward-serve` serves that too, from the same process and the same weights: | `POST /v1/systemone` (alias `POST /v1/decisions`) | `{"state": …, "questions": {id: question}}` → `{"model", "answers": {id: answer}, "usage"}` | | `GET /v1/systemone/models` | the decision model this build serves: name, description, `max_input_tokens`, `modalities` | +A pack that declares `attention_mode: noncausal_full_attention` (below) is a decision model and nothing else: +it encodes a prompt in one pass and has no token-by-token decode. On such a pack `/v1/chat/completions` and +`/v1/completions` answer **400** with `type: decision_model_error` and a message that names `/v1/systemone` +(or says there is no decision head, when the serve loaded none), before any work is done. + `questions` is an **object**, and its ids come back as the keys of `answers` in request order. Question types are `choice` (1–255 options), `noul` and `score` (2–10 ordered levels); a decision runs one backbone prefill per question, pools the **last position's hidden state after the final norm** (`pooling: last`), diff --git a/tests/npu_chatml_test.cpp b/tests/npu_chatml_test.cpp index d43909570..dd407c1d1 100644 --- a/tests/npu_chatml_test.cpp +++ b/tests/npu_chatml_test.cpp @@ -154,6 +154,21 @@ int main(int argc, char** argv) { check(pin_probe_messages(oreq).size() == 2, "the same for the ordered JSON the template path uses"); } + // A text route on a decision-only pack (engine#1786, D3-Nano-NPU2): a 400 that names the right route + { + const nlohmann::json up = onebit::npu::decision_only_refusal("D3-Nano-NPU2", true); + const std::string m = up["error"]["message"].get(); + check(m.find("D3-Nano-NPU2") == 0 && m.find("does not generate text") != std::string::npos && + m.find("POST /v1/systemone") != std::string::npos, + "the refusal names the model, says it does not generate text, and names the decision route"); + check(up["error"]["type"] == "decision_model_error", "and carries a type a client can match on"); + const std::string none = onebit::npu::decision_only_refusal("x", false)["error"]["message"].get(); + check(none.find("POST /v1/systemone") == std::string::npos && none.find("no decision head") != std::string::npos, + "with no head loaded it does not send the client to a route that would answer 503"); + check(m.find("forward step failed") == std::string::npos && m.find("position") == std::string::npos, + "none of the forward's internal wording reaches the client"); + } + if (failures == 0) std::printf("npu_chatml: ok\n"); return failures == 0 ? 0 : 1; }