From 0134ae24085022ea9236bd6ae418b46eeac69a5b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 30 Aug 2026 22:03:23 +0200 Subject: [PATCH] =?UTF-8?q?fix(eval):=20glm-5.3=20rejects=20thinking.type?= =?UTF-8?q?=3Ddisabled=20=E2=80=94=20send=20reasoning=5Feffort=20alone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Probed the live API (2026-08-31): glm-5.3-flash answers the combined payload with HTTP 400 code 1210 — 'This model always engages in thinking and cannot be disabled; please use low, high, or max'. The both-knobs approach from the previous commit would have 400'd every call and, after retries, silently scored empty predictions. Valid reasoning_effort values are exactly low, high, max; an effort now sends reasoning_effort only, glm-5.2 keeps thinking.type=disabled. --- eval_sadeed_glm.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/eval_sadeed_glm.py b/eval_sadeed_glm.py index 845b9ee..552d281 100644 --- a/eval_sadeed_glm.py +++ b/eval_sadeed_glm.py @@ -12,9 +12,12 @@ unrecognized values silently default to MAX reasoning there — the 2026-08-17 finding was that reasoning mode burns minutes per long paragraph, so a missing parameter turns a ~1h run into days. GLM-5.2 -and earlier disable reasoning via thinking.type=disabled, which -GLM-5.3+ ignores. Passing an effort sends both knobs and warns if the -response still carries reasoning_content (the disable didn't take). +and earlier disable reasoning via thinking.type=disabled; GLM-5.3+ +rejects that parameter outright (verified 2026-08-31: HTTP 400 code +1210, valid reasoning_effort values are exactly low, high, max — +thinking cannot be disabled at all). An effort therefore sends ONLY +reasoning_effort, and any response still carrying reasoning_content +warns that it did not take. Usage: python eval_sadeed_glm.py [model_id] [reasoning_effort] @@ -72,10 +75,14 @@ def call(session: requests.Session, key: str, text: str, tries: int = 5) -> str: "messages": [{"role": "user", "content": PROMPT.format(t=text)}], "temperature": 0, "max_tokens": 8192, - "thinking": {"type": "disabled"}, } if EFFORT: + # glm-5.3+ rejects thinking.type=disabled outright (HTTP 400, + # code 1210: "This model always engages in thinking"); valid + # reasoning_effort values are low, high, max. payload["reasoning_effort"] = EFFORT + else: + payload["thinking"] = {"type": "disabled"} for attempt in range(tries): try: r = session.post(BASE, json=payload, timeout=300)