From 1adceaf5746604ac0429457df2e93259381f9590 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:42:43 -0400 Subject: [PATCH] serve: logprobs normalise over the tokenizer's bound, not the head's padded width (Fixes #2272) A padded head stores rows past the vocabulary (engine#2026: Qwen3 has 151,936 rows against 151,669 ids), and those rows are not tokens. logprobs_of() normalised over every element of the row it was given, so every reported logprob was divided by a partition function that included the padding's mass -- a model-dependent shift, not a constant, and invisible to any comparison between two paths that share the helper. sample_token() already bounded itself with token_bound(); this makes the logprob report do the same, including the top_logprobs candidate list, which could otherwise offer an id the tokenizer does not have. Measured: with the bound ignored (a mutant), two pre-existing arms and both new ones fail -- FAIL bounded greedy sample picks the best real id FAIL the bound holds with a mask that allows padding FAIL a padded row: the logprob is the vocabulary-only softmax, not the padding's FAIL a padded row: no id at or past the vocabulary appears in top_logprobs and with it: npu_sampling: ok. The new arms compute the expected value from a vocabulary-only softmax built in the test, not through the helper, so they cannot agree with a wrong helper by construction. --- app/forward_serve.cpp | 2 +- app/npu_sampling.h | 26 ++++++++++++++++++++------ tests/npu_sampling_test.cpp | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 5fe87824d..fa9204753 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -559,7 +559,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // Raw distribution: sample_token() takes logits by const reference, so this is // pre-top_k/top_p/temperature and the grammar mask cannot narrow it. if (want_lp) - held.emplace_back(Emitted{piece, npu::logprobs_of(logits, next, opts.sample.top_logprobs)}, piece.size()); + held.emplace_back(Emitted{piece, npu::logprobs_of(logits, e.tok->size(), next, opts.sample.top_logprobs)}, piece.size()); stats.pieces.push_back(piece); // engine#2111: reasoning_tokens is counted over these ++produced; const size_t stop_at = npu::find_stop(pending, opts.stop); diff --git a/app/npu_sampling.h b/app/npu_sampling.h index 065005cb6..75c409deb 100644 --- a/app/npu_sampling.h +++ b/app/npu_sampling.h @@ -198,24 +198,32 @@ struct TokenLogprob { // is why this is computed from the untouched logits rather than inside the sampler's filtered // candidate set. Top entries are the highest raw logits, with the sampler's tie rule (lower id // first), so the same seed and request always report the same list. -inline TokenLogprob logprobs_of(const std::vector& logits, int chosen, int top_n) { +// `vocab` is the tokenizer's bound and MUST be passed whenever `logits` may be wider than it: a +// padded head stores rows past the vocabulary (engine#2026 -- Qwen3: 151,936 rows against 151,669 +// ids), and those rows are not tokens. Normalising over them shifts every reported logprob by the +// padding's mass, which is model-dependent rather than a constant, and invisible to any comparison +// between two paths that share this helper (engine#2272). The bound also keeps padding ids out of +// the `top_logprobs` list, where an id the tokenizer does not have would otherwise appear. +inline TokenLogprob logprobs_of(const std::vector& logits, int vocab, int chosen, int top_n) { TokenLogprob out; out.id = chosen; if (logits.empty()) return out; + const size_t n = token_bound(logits.size(), vocab); + if (n == 0) return out; // log-sum-exp in double: exp() of a float logit and then log() of the sum loses the tail that // top_logprobs is read for. float max_l = logits[0]; - for (const float v : logits) max_l = std::max(max_l, v); + for (size_t i = 0; i < n; i++) max_l = std::max(max_l, logits[i]); if (!std::isfinite(max_l)) return out; // a vocabulary of infinities reports nothing meaningful double sum = 0.0; - for (const float v : logits) sum += std::exp(double(v) - double(max_l)); + for (size_t i = 0; i < n; i++) sum += std::exp(double(logits[i]) - double(max_l)); const double log_z = double(max_l) + std::log(sum); if (!std::isfinite(log_z)) return out; - if (chosen >= 0 && size_t(chosen) < logits.size()) out.logprob = float(double(logits[size_t(chosen)]) - log_z); + if (chosen >= 0 && size_t(chosen) < n) out.logprob = float(double(logits[size_t(chosen)]) - log_z); if (top_n <= 0) return out; std::vector c; - c.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); i++) c.push_back({logits[i], int(i)}); + c.reserve(n); + for (size_t i = 0; i < n; i++) c.push_back({logits[i], int(i)}); auto better = [](const Candidate& a, const Candidate& b) { return a.logit > b.logit || (a.logit == b.logit && a.id < b.id); }; @@ -226,6 +234,12 @@ inline TokenLogprob logprobs_of(const std::vector& logits, int chosen, in return out; } +// For a row that is exactly the vocabulary (no padding ids). A padded row must use the overload +// above with the tokenizer's bound; this form is what a test or a non-padded model wants. +inline TokenLogprob logprobs_of(const std::vector& logits, int chosen, int top_n) { + return logprobs_of(logits, int(logits.size()), chosen, top_n); +} + // The bytes a decoder must hold back before it can be sure no stop sequence straddles // the boundary: the longest stop, minus one. inline size_t stop_holdback(const std::vector& stops) { diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 515695117..4119c8851 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -408,6 +408,25 @@ void test_logprobs() { check(flat.top.empty(), "top_logprobs 0 reports no alternatives"); // Ties go to the lower id, the rule the sampler draws by. + // engine#2272: a padded head stores rows past the vocabulary (engine#2026), and those rows are + // not tokens. Normalising over them shifts every reported logprob by the padding's mass, and the + // padding is given the LARGEST logits here so the shift cannot be mistaken for rounding. The + // expected values are computed from the vocabulary-only softmax below, not through the helper. + { + const size_t vocab = 3, pad = 2; + std::vector padded(vocab + pad, 50.0f); // padding: the largest logits, as a head may store + padded[0] = 0.0f; padded[1] = 0.0f; padded[2] = 0.0f; + double z = 0.0; + for (size_t i = 0; i < vocab; i++) z += std::exp(double(padded[i])); + const double log_z = std::log(z); + const auto bounded = logprobs_of(padded, int(vocab), 1, 64); + check(std::fabs(bounded.logprob - float(-log_z)) < 1e-6f, + "a padded row: the logprob is the vocabulary-only softmax, not the padding's"); + bool padding_reported = false; + for (const auto& t : bounded.top) padding_reported = padding_reported || t.id >= int(vocab); + check(!padding_reported, "a padded row: no id at or past the vocabulary appears in top_logprobs"); + } + const auto tied = logprobs_of(std::vector{1.0f, 1.0f, 1.0f}, 0, 2); check(tied.top.size() == 2 && tied.top[0].id == 0 && tied.top[1].id == 1, "ties keep the lower id first");