Skip to content

fix(core): harden multi-node failover isolation and align 50-tool autotest runner #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mcocdaa merged 1 commit into from
Nov 7, 2026
Merged

fix(core): harden multi-node failover isolation and align 50-tool autotest runner #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 63 additions & 2 deletions Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.RequestBuilding.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,75 @@ namespace RimMind.Infrastructure.Services.Clients.OpenAI
{
public partial class OpenAIClient
{
internal static string BuildFormatCacheKey(string? endpoint, string? model) =>
$"{(endpoint ?? string.Empty).Trim().TrimEnd('/')}|{(model ?? string.Empty).Trim()}";

private static RimMind.Domain.Llm.LlmRequestEnvelope StripJsonSchema(RimMind.Domain.Llm.LlmRequestEnvelope envelope)
{
if (string.IsNullOrEmpty(envelope.JsonSchema))
return envelope;

return new RimMind.Domain.Llm.LlmRequestEnvelope
{
RequestId = envelope.RequestId,
TraceId = envelope.TraceId,
ScenarioId = envelope.ScenarioId,
ModId = envelope.ModId,
Messages = envelope.Messages,
SystemAugmentations = envelope.SystemAugmentations,
JsonSchema = null,
Tools = envelope.Tools,
ToolDispatchMode = envelope.ToolDispatchMode,
Examples = envelope.Examples,
MaxTokens = envelope.MaxTokens,
Temperature = envelope.Temperature,
Priority = envelope.Priority,
ExpireAtTicks = envelope.ExpireAtTicks,
MaxRetryCount = envelope.MaxRetryCount,
IsStreaming = envelope.IsStreaming,
OnStreamChunk = envelope.OnStreamChunk,
Ct = envelope.Ct,
NpcId = envelope.NpcId,
GameStateInfo = envelope.GameStateInfo,
Hints = envelope.Hints,
};
}

private string BuildEnvelopeRequestJson(
RimMind.Domain.Llm.LlmRequestEnvelope envelope,
bool stream = false,
string? modelNameOverride = null,
string? endpointOverride = null)
string? endpointOverride = null,
string? formatModeOverride = null)
{
string effectiveEndpoint = endpointOverride ?? _settings.ApiEndpoint;
string rawModel = !string.IsNullOrWhiteSpace(modelNameOverride) ? modelNameOverride! : _settings.ModelName;
string model = NormalizeModelNameForEndpoint(endpointOverride ?? _settings.ApiEndpoint, rawModel);
string model = NormalizeModelNameForEndpoint(effectiveEndpoint, rawModel);

if (!_settings.ForceJsonMode)
{
return OpenAIRequestSerializer.BuildRequestJson(
StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: false);
}

string? formatMode = formatModeOverride;
if (string.IsNullOrEmpty(formatMode))
{
_formatCapabilityCache.TryGetValue(BuildFormatCacheKey(effectiveEndpoint, model), out formatMode);
}

if (string.Equals(formatMode, "none", StringComparison.OrdinalIgnoreCase))
{
return OpenAIRequestSerializer.BuildRequestJson(
StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: false);
}

if (string.Equals(formatMode, "json_object", StringComparison.OrdinalIgnoreCase))
{
return OpenAIRequestSerializer.BuildRequestJson(
StripJsonSchema(envelope), model, _settings.MaxTokens, stream, forceJsonMode: true);
}

return OpenAIRequestSerializer.BuildRequestJson(
envelope, model, _settings.MaxTokens, stream, forceJsonMode: _settings.ForceJsonMode);
}
Expand Down
110 changes: 95 additions & 15 deletions Source/Infrastructure/Services/Clients/OpenAI/OpenAIClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ public partial class OpenAIClient : IAIClient
float connectTimeout = 60f,
CancellationToken cancellationToken = default);

private const long DefaultNodeIsolationMs = 60_000L;
private const long DefaultNodeIsolationMs = 180_000L;
private const long TransientRateLimitIsolationMs = 2_000L;
private const long PermanentNodeIsolationMs = 1_800_000L;

private static readonly ConcurrentDictionary<string, string> _formatCapabilityCache = new();
private static readonly ConcurrentDictionary<string, string> _formatCapabilityCache = new(StringComparer.OrdinalIgnoreCase);
private static readonly ConcurrentDictionary<string, long> _endpointIsolatedUntilMs = new(StringComparer.OrdinalIgnoreCase);
private static readonly ConcurrentDictionary<string, long> _endpointExhaustedUntilMs = new(StringComparer.OrdinalIgnoreCase);

Expand All @@ -44,7 +44,14 @@ internal static void InvalidateFormatCache()
_endpointExhaustedUntilMs.Clear();
}

private string BuildCacheKey() => $"{_settings.ApiEndpoint}|{_settings.ModelName}";
internal static string? GetCachedFormatCapability(string? endpoint, string? model)
{
return _formatCapabilityCache.TryGetValue(BuildFormatCacheKey(endpoint, model), out string? mode)
? mode
: null;
}

private string BuildCacheKey() => BuildFormatCacheKey(_settings.ApiEndpoint, _settings.ModelName);

private static string BuildNodeKey(BackupEndpointNode node) =>
$"{(node.ApiEndpoint ?? string.Empty).Trim().TrimEnd('/')}|{(node.ModelName ?? string.Empty).Trim()}|{(string.IsNullOrEmpty(node.ApiKey) ? string.Empty : (node.ApiKey.Length > 8 ? node.ApiKey.Substring(node.ApiKey.Length - 8) : node.ApiKey))}";
Expand Down Expand Up @@ -85,7 +92,7 @@ internal static bool IsPermanentNodeQuotaOrAuthError(Exception? ex)
{
if (ex == null) return false;
if (ex is HttpTransport.HttpException httpEx &&
(httpEx.StatusCode == 401 || httpEx.StatusCode == 402 || httpEx.StatusCode == 403))
(httpEx.StatusCode == 401 || httpEx.StatusCode == 402 || httpEx.StatusCode == 403 || httpEx.StatusCode == 410))
{
return true;
}
Expand All @@ -97,6 +104,9 @@ internal static bool IsPermanentNodeQuotaOrAuthError(Exception? ex)
msg.IndexOf("invalid_api_key", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("billing_not_active", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("model_not_found", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("ModelDeprecated", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("has been deprecated", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("Model is unavailable", StringComparison.OrdinalIgnoreCase) >= 0 ||
msg.IndexOf("is not supported by any configured account", StringComparison.OrdinalIgnoreCase) >= 0;
}

Expand Down Expand Up @@ -138,7 +148,7 @@ public OpenAIClient(IOpenAISettings settings, ILogSink? logSink = null, HttpPost
}

private static float ResolveTimeoutSeconds(bool isLocal, int candidateCount, int attempt) =>
isLocal ? 300f : ((candidateCount > 1 && attempt < candidateCount - 1) ? 45f : 60f);
isLocal ? 300f : ((candidateCount > 1 && attempt < candidateCount - 1) ? 25f : 60f);

public bool IsConfigured() => _settings.IsConfigured();

Expand Down Expand Up @@ -240,6 +250,58 @@ internal List<BackupEndpointNode> ResolveOrderedCandidateNodes(long nowMs)
return healthy;
}

private async Task<(string responseText, long httpStatusCode)> PostWithFormatFallbackAsync(
BackupEndpointNode node,
string endpoint,
string effectiveModel,
RimMind.Domain.Llm.LlmRequestEnvelope envelope,
string initialJson,
string? headerName,
string? headerValue,
float connectTimeout)
{
string currentJson = initialJson;
string cacheKey = BuildFormatCacheKey(node.ApiEndpoint, effectiveModel);

while (true)
{
try
{
var (responseText, httpStatusCode) = await _postAsync(
endpoint, currentJson, $"Bearer {node.ApiKey}",
headerName: headerName, headerValue: headerValue,
connectTimeout: connectTimeout,
cancellationToken: envelope.Ct);

if (httpStatusCode == 400 && currentJson.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0)
{
throw HttpTransport.HttpException.FromResponse((int)httpStatusCode, responseText, null);
}

return (responseText, httpStatusCode);
}
catch (HttpTransport.HttpException httpEx) when (
httpEx.StatusCode == 400 &&
currentJson.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0)
{
string nextFormat = currentJson.IndexOf("\"json_schema\"", StringComparison.Ordinal) >= 0
? "json_object"
: "none";

_formatCapabilityCache[cacheKey] = nextFormat;
_logSink?.LogFromBackground(
$"[RimMind-Core] Node '{node.Name}' ({effectiveModel}) rejected response_format with HTTP 400; downgrading format capability to '{nextFormat}' and retrying on same node.");

currentJson = BuildEnvelopeRequestJson(
envelope,
stream: false,
modelNameOverride: effectiveModel,
endpointOverride: node.ApiEndpoint,
formatModeOverride: nextFormat);
}
}
}

public async Task<Result<RimMind.Domain.Llm.LlmResponse, RimMindError>> SendAsync(RimMind.Domain.Llm.LlmRequestEnvelope envelope)
{
if (!IsConfigured())
Expand Down Expand Up @@ -280,11 +342,15 @@ internal List<BackupEndpointNode> ResolveOrderedCandidateNodes(long nowMs)
string? headerName = isOpenCode ? "x-opencode-session" : null;
string? headerValue = isOpenCode ? ResolveSessionId(envelope) : null;

(string responseText, long httpStatusCode) = await _postAsync(
endpoint, json, $"Bearer {node.ApiKey}",
headerName: headerName, headerValue: headerValue,
connectTimeout: connectTimeout,
cancellationToken: envelope.Ct);
(string responseText, long httpStatusCode) = await PostWithFormatFallbackAsync(
node,
endpoint,
effectiveModel,
envelope,
json,
headerName,
headerValue,
connectTimeout);

var parsed = JsonConvert.DeserializeObject<OpenAIResponseDto>(responseText);
string content = parsed?.choices?[0]?.message?.content ?? string.Empty;
Expand Down Expand Up @@ -414,11 +480,15 @@ internal List<BackupEndpointNode> ResolveOrderedCandidateNodes(long nowMs)
{
await Task.Delay(1200 * backoffAttempt, envelope.Ct);
var sw = Stopwatch.StartNew();
(string responseText, long httpStatusCode) = await _postAsync(
endpoint, json, $"Bearer {retryNode.ApiKey}",
headerName: headerName, headerValue: headerValue,
connectTimeout: connectTimeout,
cancellationToken: envelope.Ct);
(string responseText, long httpStatusCode) = await PostWithFormatFallbackAsync(
retryNode,
endpoint,
effectiveModel,
envelope,
json,
headerName,
headerValue,
connectTimeout);

var parsed = JsonConvert.DeserializeObject<OpenAIResponseDto>(responseText);
string content = parsed?.choices?[0]?.message?.content ?? string.Empty;
Expand Down Expand Up @@ -645,6 +715,16 @@ await SseStreamReader.ReadDataLinesAsync(session.Reader, data =>
}
catch (Exception ex)
{
if (ex is HttpTransport.HttpException httpEx &&
httpEx.StatusCode == 400 &&
json.IndexOf("\"response_format\"", StringComparison.Ordinal) >= 0)
{
string nextFormat = json.IndexOf("\"json_schema\"", StringComparison.Ordinal) >= 0
? "json_object"
: "none";
_formatCapabilityCache[BuildFormatCacheKey(node.ApiEndpoint, effectiveModel)] = nextFormat;
}

long cooldownMs = ResolveNodeIsolationCooldownMs(ex, out bool isPermQuota);
RecordNodeFailure(node, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), cooldownMs, isPermQuota);

Expand Down
65 changes: 63 additions & 2 deletions Source/Infrastructure/UI/DebugActions/BehaviorAutotestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -682,9 +682,17 @@ private IEnumerator RunSuiteAllToolsExecution()
var execResult = task.Result;
if (execResult.IsErr)
{
item.Status = "FAIL";
item.IsError = true;
item.ErrorMessage = $"Result.Err: {execResult.Error.Code} - {execResult.Error.Message}";
if (IsExpectedDomainRejection(tool.Definition.Id, execResult.Error.Message))
{
item.Status = "DOMAIN_REJECT";
item.Notes = $"Domain rule rejected safely: {execResult.Error.Message}";
}
else
{
item.Status = "FAIL";
}
}
else
{
Expand Down Expand Up @@ -905,6 +913,42 @@ private ToolCallArgs BuildTestArgumentsForTool(IToolHandler tool, Pawn? pawn, Ve
dict["patient_id"] = pawnId;
dict["reason"] = "Autotest triage";
break;

case "actions_place_blueprint":
case "actions.place_blueprint":
case "place_blueprint":
dict["building_def"] = "SleepingSpot";
dict["auto_construct"] = false;
break;

case "actions_manage_zone":
case "actions.manage_zone":
case "manage_zone":
dict["zone_type"] = "Stockpile";
dict["width"] = 3;
dict["height"] = 3;
break;

case "actions_designate_resources":
case "actions.designate_resources":
case "designate_resources":
dict["designation_type"] = "ChopWood";
dict["max_count"] = 2;
dict["radius"] = 40;
break;

case "actions_add_worktable_bill":
case "actions.add_worktable_bill":
case "add_worktable_bill":
dict["recipe_def"] = "CookMealSimple";
break;

case "actions_plan_room_project":
case "actions.plan_room_project":
case "plan_room_project":
dict["room_type"] = "Shelter";
dict["size"] = 4;
break;
}

return new ToolCallArgs
Expand All @@ -913,7 +957,8 @@ private ToolCallArgs BuildTestArgumentsForTool(IToolHandler tool, Pawn? pawn, Ve
ToolName = toolId,
ArgumentsJson = JsonConvert.SerializeObject(dict),
PawnId = pawnId > 0 ? pawnId : null,
NpcId = pawnId > 0 ? $"NPC-{pawnId}" : null
NpcId = pawnId > 0 ? $"NPC-{pawnId}" : null,
IsPlayerApproved = true
};
}

Expand Down Expand Up @@ -977,6 +1022,22 @@ private static bool IsExpectedDomainRejection(string toolId, string? error)
|| error.IndexOf("No job available", StringComparison.OrdinalIgnoreCase) >= 0))
return true;

// Composite actions on fresh -quicktest map: resource safety guard or missing worktable
if ((toolId == "actions_plan_room_project" || toolId == "actions_place_blueprint") &&
(error.IndexOf("[Safety Guard]", StringComparison.OrdinalIgnoreCase) >= 0 ||
error.IndexOf("minimum survival threshold", StringComparison.OrdinalIgnoreCase) >= 0))
return true;

if (toolId == "actions_add_worktable_bill" &&
(error.IndexOf("No worktable found", StringComparison.OrdinalIgnoreCase) >= 0 ||
error.IndexOf("worktable", StringComparison.OrdinalIgnoreCase) >= 0))
return true;

if (toolId == "actions_designate_resources" &&
(error.IndexOf("No valid", StringComparison.OrdinalIgnoreCase) >= 0 ||
error.IndexOf("found within", StringComparison.OrdinalIgnoreCase) >= 0))
return true;

return false;
}

Expand Down
6 changes: 3 additions & 3 deletions Source/Infrastructure/UI/DebugActions/TenDayPlaythroughRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1228,7 +1228,7 @@ private IEnumerator ExecuteMorningThought(Pawn pawn, int day, PlaythroughDayRepo

_report.TotalLiveLlmRequests++;

float timeout = 65f;
float timeout = 95f;
float elapsed = 0f;
while (!completed && elapsed < timeout)
{
Expand Down Expand Up @@ -1458,7 +1458,7 @@ private IEnumerator ExecuteSocialDialogue(Pawn speaker, Pawn listener, int day,

_report.TotalLiveLlmRequests++;

float timeout = 65f;
float timeout = 95f;
float elapsed = 0f;
while (!completed && elapsed < timeout)
{
Expand Down Expand Up @@ -1958,7 +1958,7 @@ private IEnumerator ExecuteAutonomousAgentAction(Pawn agentPawn, int day, Playth

_report.TotalLiveLlmRequests++;

float timeout = 65f;
float timeout = 95f;
float elapsed = 0f;
while (!completed && elapsed < timeout)
{
Expand Down
Loading
Loading