From 6ebb6f7498f577a860404047bbc8ef95b209fc8e Mon Sep 17 00:00:00 2001 From: mcocdaa <2021137961@qq.com> Date: Sat, 7 Nov 2026 07:41:24 +0100 Subject: [PATCH] fix(core): harden multi-node failover isolation and align 50-tool autotest runner fix(openai): downgrade response_format on HTTP 400 and extend failover timeouts When a node rejects response_format with HTTP 400, automatically downgrade the format capability (json_schema -> json_object -> none), cache it per endpoint/model, and retry on the same node instead of failing over. Strip JsonSchema when ForceJsonMode is disabled. Also treat HTTP 410 and deprecated-model messages as permanent node failures, raise default node isolation to 180s, shorten intermediate failover attempt timeout to 25s, and raise playthrough LLM wait to 95s. Autotest: mark expected domain rejections as DOMAIN_REJECT, add sample args for composite actions, and approve player-gated tool calls. --- .../OpenAI/OpenAIClient.RequestBuilding.cs | 65 ++++++++++- .../Services/Clients/OpenAI/OpenAIClient.cs | 110 +++++++++++++++--- .../UI/DebugActions/BehaviorAutotestRunner.cs | 65 ++++++++++- .../DebugActions/TenDayPlaythroughRunner.cs | 6 +- .../BackupEndpointFailoverContracts.cs | 79 +++++++++++++ 5 files changed, 303 insertions(+), 22 deletions(-) diff --git a/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.RequestBuilding.cs b/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.RequestBuilding.cs index 83b10af..d99fa4e 100644 --- a/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.RequestBuilding.cs +++ b/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.RequestBuilding.cs @@ -7,14 +7,75 @@ namespace RimMind.Infrastructure.Services.Clients.OpenAI { public partial class OpenAIClient { + internal static string BuildFormatCacheKey(string? endpoint, string? model) => + $"{(endpoint ?? string.Empty).Trim().TrimEnd('/')}|{(model ?? string.Empty).Trim()}"; + + private static RimMind.Domain.Llm.LlmRequestEnvelope StripJsonSchema(RimMind.Domain.Llm.LlmRequestEnvelope envelope) + { + if (string.IsNullOrEmpty(envelope.JsonSchema)) + return envelope; + + return new RimMind.Domain.Llm.LlmRequestEnvelope + { + RequestId = envelope.RequestId, + TraceId = envelope.TraceId, + ScenarioId = envelope.ScenarioId, + ModId = envelope.ModId, + Messages = envelope.Messages, + SystemAugmentations = envelope.SystemAugmentations, + JsonSchema = null, + Tools = envelope.Tools, + ToolDispatchMode = envelope.ToolDispatchMode, + Examples = envelope.Examples, + MaxTokens = envelope.MaxTokens, + Temperature = envelope.Temperature, + Priority = envelope.Priority, + ExpireAtTicks = envelope.ExpireAtTicks, + MaxRetryCount = envelope.MaxRetryCount, + IsStreaming = envelope.IsStreaming, + OnStreamChunk = envelope.OnStreamChunk, + Ct = envelope.Ct, + NpcId = envelope.NpcId, + GameStateInfo = envelope.GameStateInfo, + Hints = envelope.Hints, + }; + } + private string BuildEnvelopeRequestJson( RimMind.Domain.Llm.LlmRequestEnvelope envelope, bool stream = false, string? modelNameOverride = null, - string? endpointOverride = null) + string? endpointOverride = null, + string? formatModeOverride = null) { + string effectiveEndpoint = endpointOverride ?? _settings.ApiEndpoint; string rawModel = !string.IsNullOrWhiteSpace(modelNameOverride) ? modelNameOverride! : _settings.ModelName; - string model = NormalizeModelNameForEndpoint(endpointOverride ?? _settings.ApiEndpoint, rawModel); + string model = NormalizeModelNameForEndpoint(effectiveEndpoint, rawModel); + + if (!_settings.ForceJsonMode) + { + return OpenAIRequestSerializer.BuildRequestJson( + StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: false); + } + + string? formatMode = formatModeOverride; + if (string.IsNullOrEmpty(formatMode)) + { + _formatCapabilityCache.TryGetValue(BuildFormatCacheKey(effectiveEndpoint, model), out formatMode); + } + + if (string.Equals(formatMode, "none", StringComparison.OrdinalIgnoreCase)) + { + return OpenAIRequestSerializer.BuildRequestJson( + StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: false); + } + + if (string.Equals(formatMode, "json_object", StringComparison.OrdinalIgnoreCase)) + { + return OpenAIRequestSerializer.BuildRequestJson( + StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: true); + } + return OpenAIRequestSerializer.BuildRequestJson( envelope, model, _settings.MaxTokens, stream, forceJsonMode: _settings.ForceJsonMode); } diff --git a/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.cs b/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.cs index cb528c4..755fcf2 100644 --- a/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.cs +++ b/Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.cs @@ -29,11 +29,11 @@ public partial class OpenAIClient : IAIClient float connectTimeout = 60f, CancellationToken cancellationToken = default); - private const long DefaultNodeIsolationMs = 60_000L; + private const long DefaultNodeIsolationMs = 180_000L; private const long TransientRateLimitIsolationMs = 2_000L; private const long PermanentNodeIsolationMs = 1_800_000L; - private static readonly ConcurrentDictionary _formatCapabilityCache = new(); + private static readonly ConcurrentDictionary _formatCapabilityCache = new(StringComparer.OrdinalIgnoreCase); private static readonly ConcurrentDictionary _endpointIsolatedUntilMs = new(StringComparer.OrdinalIgnoreCase); private static readonly ConcurrentDictionary _endpointExhaustedUntilMs = new(StringComparer.OrdinalIgnoreCase); @@ -44,7 +44,14 @@ internal static void InvalidateFormatCache() _endpointExhaustedUntilMs.Clear(); } - private string BuildCacheKey() => $"{_settings.ApiEndpoint}|{_settings.ModelName}"; + internal static string? GetCachedFormatCapability(string? endpoint, string? model) + { + return _formatCapabilityCache.TryGetValue(BuildFormatCacheKey(endpoint, model), out string? mode) + ? mode + : null; + } + + private string BuildCacheKey() => BuildFormatCacheKey(_settings.ApiEndpoint, _settings.ModelName); private static string BuildNodeKey(BackupEndpointNode node) => $"{(node.ApiEndpoint ?? string.Empty).Trim().TrimEnd('/')}|{(node.ModelName ?? string.Empty).Trim()}|{(string.IsNullOrEmpty(node.ApiKey) ? string.Empty : (node.ApiKey.Length > 8 ? node.ApiKey.Substring(node.ApiKey.Length - 8) : node.ApiKey))}"; @@ -85,7 +92,7 @@ internal static bool IsPermanentNodeQuotaOrAuthError(Exception? ex) { if (ex == null) return false; if (ex is HttpTransport.HttpException httpEx && - (httpEx.StatusCode == 401 || httpEx.StatusCode == 402 || httpEx.StatusCode == 403)) + (httpEx.StatusCode == 401 || httpEx.StatusCode == 402 || httpEx.StatusCode == 403 || httpEx.StatusCode == 410)) { return true; } @@ -97,6 +104,9 @@ internal static bool IsPermanentNodeQuotaOrAuthError(Exception? ex) msg.IndexOf("invalid_api_key", StringComparison.OrdinalIgnoreCase) >= 0 || msg.IndexOf("billing_not_active", StringComparison.OrdinalIgnoreCase) >= 0 || msg.IndexOf("model_not_found", StringComparison.OrdinalIgnoreCase) >= 0 || + msg.IndexOf("ModelDeprecated", StringComparison.OrdinalIgnoreCase) >= 0 || + msg.IndexOf("has been deprecated", StringComparison.OrdinalIgnoreCase) >= 0 || + msg.IndexOf("Model is unavailable", StringComparison.OrdinalIgnoreCase) >= 0 || msg.IndexOf("is not supported by any configured account", StringComparison.OrdinalIgnoreCase) >= 0; } @@ -138,7 +148,7 @@ public OpenAIClient(IOpenAISettings settings, ILogSink? logSink = null, HttpPost } private static float ResolveTimeoutSeconds(bool isLocal, int candidateCount, int attempt) => - isLocal ? 300f : ((candidateCount > 1 && attempt < candidateCount - 1) ? 45f : 60f); + isLocal ? 300f : ((candidateCount > 1 && attempt < candidateCount - 1) ? 25f : 60f); public bool IsConfigured() => _settings.IsConfigured(); @@ -240,6 +250,58 @@ internal List ResolveOrderedCandidateNodes(long nowMs) return healthy; } + private async Task<(string responseText, long httpStatusCode)> PostWithFormatFallbackAsync( + BackupEndpointNode node, + string endpoint, + string effectiveModel, + RimMind.Domain.Llm.LlmRequestEnvelope envelope, + string initialJson, + string? headerName, + string? headerValue, + float connectTimeout) + { + string currentJson = initialJson; + string cacheKey = BuildFormatCacheKey(node.ApiEndpoint, effectiveModel); + + while (true) + { + try + { + var (responseText, httpStatusCode) = await _postAsync( + endpoint, currentJson, $"Bearer {node.ApiKey}", + headerName: headerName, headerValue: headerValue, + connectTimeout: connectTimeout, + cancellationToken: envelope.Ct); + + if (httpStatusCode == 400 && currentJson.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0) + { + throw HttpTransport.HttpException.FromResponse((int)httpStatusCode, responseText, null); + } + + return (responseText, httpStatusCode); + } + catch (HttpTransport.HttpException httpEx) when ( + httpEx.StatusCode == 400 && + currentJson.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0) + { + string nextFormat = currentJson.IndexOf("\"json_schema\"", StringComparison.Ordinal) >= 0 + ? "json_object" + : "none"; + + _formatCapabilityCache[cacheKey] = nextFormat; + _logSink?.LogFromBackground( + $"[RimMind-Core] Node '{node.Name}' ({effectiveModel}) rejected response_format with HTTP 400; downgrading format capability to '{nextFormat}' and retrying on same node."); + + currentJson = BuildEnvelopeRequestJson( + envelope, + stream: false, + modelNameOverride: effectiveModel, + endpointOverride: node.ApiEndpoint, + formatModeOverride: nextFormat); + } + } + } + public async Task> SendAsync(RimMind.Domain.Llm.LlmRequestEnvelope envelope) { if (!IsConfigured()) @@ -280,11 +342,15 @@ internal List ResolveOrderedCandidateNodes(long nowMs) string? headerName = isOpenCode ? "x-opencode-session" : null; string? headerValue = isOpenCode ? ResolveSessionId(envelope) : null; - (string responseText, long httpStatusCode) = await _postAsync( - endpoint, json, $"Bearer {node.ApiKey}", - headerName: headerName, headerValue: headerValue, - connectTimeout: connectTimeout, - cancellationToken: envelope.Ct); + (string responseText, long httpStatusCode) = await PostWithFormatFallbackAsync( + node, + endpoint, + effectiveModel, + envelope, + json, + headerName, + headerValue, + connectTimeout); var parsed = JsonConvert.DeserializeObject(responseText); string content = parsed?.choices?[0]?.message?.content ?? string.Empty; @@ -414,11 +480,15 @@ internal List ResolveOrderedCandidateNodes(long nowMs) { await Task.Delay(1200 * backoffAttempt, envelope.Ct); var sw = Stopwatch.StartNew(); - (string responseText, long httpStatusCode) = await _postAsync( - endpoint, json, $"Bearer {retryNode.ApiKey}", - headerName: headerName, headerValue: headerValue, - connectTimeout: connectTimeout, - cancellationToken: envelope.Ct); + (string responseText, long httpStatusCode) = await PostWithFormatFallbackAsync( + retryNode, + endpoint, + effectiveModel, + envelope, + json, + headerName, + headerValue, + connectTimeout); var parsed = JsonConvert.DeserializeObject(responseText); string content = parsed?.choices?[0]?.message?.content ?? string.Empty; @@ -645,6 +715,16 @@ await SseStreamReader.ReadDataLinesAsync(session.Reader, data => } catch (Exception ex) { + if (ex is HttpTransport.HttpException httpEx && + httpEx.StatusCode == 400 && + json.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0) + { + string nextFormat = json.IndexOf("\"json_schema\"", StringComparison.Ordinal) >= 0 + ? "json_object" + : "none"; + _formatCapabilityCache[BuildFormatCacheKey(node.ApiEndpoint, effectiveModel)] = nextFormat; + } + long cooldownMs = ResolveNodeIsolationCooldownMs(ex, out bool isPermQuota); RecordNodeFailure(node, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), cooldownMs, isPermQuota); diff --git a/Source/Infrastructure/UI/DebugActions/BehaviorAutotestRunner.cs b/Source/Infrastructure/UI/DebugActions/BehaviorAutotestRunner.cs index a3758b2..1532db6 100644 --- a/Source/Infrastructure/UI/DebugActions/BehaviorAutotestRunner.cs +++ b/Source/Infrastructure/UI/DebugActions/BehaviorAutotestRunner.cs @@ -682,9 +682,17 @@ private IEnumerator RunSuiteAllToolsExecution() var execResult = task.Result; if (execResult.IsErr) { - item.Status = "FAIL"; item.IsError = true; item.ErrorMessage = $"Result.Err: {execResult.Error.Code} - {execResult.Error.Message}"; + if (IsExpectedDomainRejection(tool.Definition.Id, execResult.Error.Message)) + { + item.Status = "DOMAIN_REJECT"; + item.Notes = $"Domain rule rejected safely: {execResult.Error.Message}"; + } + else + { + item.Status = "FAIL"; + } } else { @@ -905,6 +913,42 @@ private ToolCallArgs BuildTestArgumentsForTool(IToolHandler tool, Pawn? pawn, Ve dict["patient_id"] = pawnId; dict["reason"] = "Autotest triage"; break; + + case "actions_place_blueprint": + case "actions.place_blueprint": + case "place_blueprint": + dict["building_def"] = "SleepingSpot"; + dict["auto_construct"] = false; + break; + + case "actions_manage_zone": + case "actions.manage_zone": + case "manage_zone": + dict["zone_type"] = "Stockpile"; + dict["width"] = 3; + dict["height"] = 3; + break; + + case "actions_designate_resources": + case "actions.designate_resources": + case "designate_resources": + dict["designation_type"] = "ChopWood"; + dict["max_count"] = 2; + dict["radius"] = 40; + break; + + case "actions_add_worktable_bill": + case "actions.add_worktable_bill": + case "add_worktable_bill": + dict["recipe_def"] = "CookMealSimple"; + break; + + case "actions_plan_room_project": + case "actions.plan_room_project": + case "plan_room_project": + dict["room_type"] = "Shelter"; + dict["size"] = 4; + break; } return new ToolCallArgs @@ -913,7 +957,8 @@ private ToolCallArgs BuildTestArgumentsForTool(IToolHandler tool, Pawn? pawn, Ve ToolName = toolId, ArgumentsJson = JsonConvert.SerializeObject(dict), PawnId = pawnId > 0 ? pawnId : null, - NpcId = pawnId > 0 ? $"NPC-{pawnId}" : null + NpcId = pawnId > 0 ? $"NPC-{pawnId}" : null, + IsPlayerApproved = true }; } @@ -977,6 +1022,22 @@ private static bool IsExpectedDomainRejection(string toolId, string? error) || error.IndexOf("No job available", StringComparison.OrdinalIgnoreCase) >= 0)) return true; + // Composite actions on fresh -quicktest map: resource safety guard or missing worktable + if ((toolId == "actions_plan_room_project" || toolId == "actions_place_blueprint") && + (error.IndexOf("[Safety Guard]", StringComparison.OrdinalIgnoreCase) >= 0 || + error.IndexOf("minimum survival threshold", StringComparison.OrdinalIgnoreCase) >= 0)) + return true; + + if (toolId == "actions_add_worktable_bill" && + (error.IndexOf("No worktable found", StringComparison.OrdinalIgnoreCase) >= 0 || + error.IndexOf("worktable", StringComparison.OrdinalIgnoreCase) >= 0)) + return true; + + if (toolId == "actions_designate_resources" && + (error.IndexOf("No valid", StringComparison.OrdinalIgnoreCase) >= 0 || + error.IndexOf("found within", StringComparison.OrdinalIgnoreCase) >= 0)) + return true; + return false; } diff --git a/Source/Infrastructure/UI/DebugActions/TenDayPlaythroughRunner.cs b/Source/Infrastructure/UI/DebugActions/TenDayPlaythroughRunner.cs index 82100cc..e4a8c91 100644 --- a/Source/Infrastructure/UI/DebugActions/TenDayPlaythroughRunner.cs +++ b/Source/Infrastructure/UI/DebugActions/TenDayPlaythroughRunner.cs @@ -1228,7 +1228,7 @@ private IEnumerator ExecuteMorningThought(Pawn pawn, int day, PlaythroughDayRepo _report.TotalLiveLlmRequests++; - float timeout = 65f; + float timeout = 95f; float elapsed = 0f; while (!completed && elapsed < timeout) { @@ -1458,7 +1458,7 @@ private IEnumerator ExecuteSocialDialogue(Pawn speaker, Pawn listener, int day, _report.TotalLiveLlmRequests++; - float timeout = 65f; + float timeout = 95f; float elapsed = 0f; while (!completed && elapsed < timeout) { @@ -1958,7 +1958,7 @@ private IEnumerator ExecuteAutonomousAgentAction(Pawn agentPawn, int day, Playth _report.TotalLiveLlmRequests++; - float timeout = 65f; + float timeout = 95f; float elapsed = 0f; while (!completed && elapsed < timeout) { diff --git a/Tests/Contracts/BackupEndpointFailoverContracts.cs b/Tests/Contracts/BackupEndpointFailoverContracts.cs index 49a1f75..c60dd26 100644 --- a/Tests/Contracts/BackupEndpointFailoverContracts.cs +++ b/Tests/Contracts/BackupEndpointFailoverContracts.cs @@ -384,6 +384,85 @@ public async Task HybridAIClient_AndPlayer2Dtos_PreserveStreamBoundaries_AndSnak Assert.Single(emitted); } + [Fact] + public async Task SendAsync_DowngradesResponseFormatCapabilityOnHttp400_AndHonorsForceJsonModeFalse() + { + OpenAIClient.InvalidateFormatCache(); + + var schemaEnvelope = new LlmRequestEnvelope + { + RequestId = "req-format-downgrade", + ModId = "RimMind.Storyteller", + JsonSchema = "{\"type\":\"object\",\"properties\":{\"narrative\":{\"type\":\"string\"}}}", + Messages = new List + { + new ChatMessage { Role = "user", Content = "Evaluate colony state." } + } + }; + + // 1. ForceJsonMode = false strips JsonSchema so response_format is omitted completely + var noJsonModeSettings = new FakeOpenAISettings + { + ApiEndpoint = "https://api.deepseek.com/v1", + ModelName = "deepseek-chat", + ForceJsonMode = false + }; + string? capturedDisabledJson = null; + var noJsonClient = new OpenAIClient( + noJsonModeSettings, + postAsyncOverride: (_, json, _, _, _, _, _) => + { + capturedDisabledJson = json; + return Task.FromResult(("{\"choices\":[{\"message\":{\"content\":\"{\\\"narrative\\\":\\\"ok\\\"}\"}}],\"usage\":{\"total_tokens\":10}}", 200L)); + }); + + var disabledRes = await noJsonClient.SendAsync(schemaEnvelope); + Assert.True(disabledRes.IsOk); + Assert.NotNull(capturedDisabledJson); + Assert.DoesNotContain("\"response_format\"", capturedDisabledJson!); + + // 2. ForceJsonMode = true on an endpoint rejecting json_schema AND json_object (HTTP 400) + // automatically downgrades json_schema -> json_object -> none on the same node and caches the capability + OpenAIClient.InvalidateFormatCache(); + var settings = new FakeOpenAISettings + { + ApiEndpoint = "http://localhost:11434/v1", + ModelName = "qwen2.5:7b", + ForceJsonMode = true + }; + var payloads = new List(); + var client = new OpenAIClient( + settings, + postAsyncOverride: (_, json, _, _, _, _, _) => + { + payloads.Add(json); + if (json.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0) + { + throw HttpTransport.HttpException.FromResponse( + 400, + "{\"error\":{\"message\":\"response_format is not supported by this model\"}}", + "Bad Request"); + } + + return Task.FromResult(("{\"choices\":[{\"message\":{\"content\":\"{\\\"narrative\\\":\\\"downgraded-ok\\\"}\"}}],\"usage\":{\"total_tokens\":15}}", 200L)); + }); + + var firstRes = await client.SendAsync(schemaEnvelope); + Assert.True(firstRes.IsOk); + Assert.Equal("{\"narrative\":\"downgraded-ok\"}", firstRes.Value.Content); + Assert.Equal(3, payloads.Count); + Assert.Contains("\"json_schema\"", payloads[0]); + Assert.Contains("\"json_object\"", payloads[1]); + Assert.DoesNotContain("\"response_format\"", payloads[2]); + Assert.Equal("none", OpenAIClient.GetCachedFormatCapability("http://localhost:11434/v1", "qwen2.5:7b")); + + // 3. Subsequent request uses cached capability ("none") immediately in 1 round-trip + var secondRes = await client.SendAsync(schemaEnvelope); + Assert.True(secondRes.IsOk); + Assert.Equal(4, payloads.Count); + Assert.DoesNotContain("\"response_format\"", payloads[3]); + } + private sealed class FaultingAfterFirstLineStream : Stream { private readonly byte[] _firstBytes;