From f32c09247a2bd2f078d2c60f92c9ab22f7b739a5 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 27 Aug 2026 17:20:57 -0500 Subject: [PATCH 1/7] Add runtime Exie model settings --- src/Exceptionless.Core/Bootstrapper.cs | 1 + .../Models/SystemSettings.cs | 25 +++ .../ExceptionlessElasticConfiguration.cs | 2 + .../Indexes/SystemSettingsIndex.cs | 38 +++++ .../Interfaces/ISystemSettingsRepository.cs | 6 + .../Repositories/SystemSettingsRepository.cs | 15 ++ .../Api/Endpoints/AdminEndpoints.cs | 10 ++ .../AssistantModelSettingsService.cs | 78 +++++++++ .../Assistant/AssistantService.cs | 8 +- .../src/lib/features/admin/api.svelte.ts | 44 ++++- .../src/lib/features/admin/models.ts | 10 ++ .../src/lib/features/admin/schemas.test.ts | 17 ++ .../src/lib/features/admin/schemas.ts | 5 + .../src/routes/(app)/system/exie/+page.svelte | 153 +++++++++++++++++- .../Models/Admin/UpdateAssistantSettings.cs | 9 ++ src/Exceptionless.Web/Program.cs | 1 + .../Api/Endpoints/AdminEndpointTests.cs | 74 +++++++++ .../Assistant/AssistantServiceTests.cs | 57 ++++++- tests/http/admin.http | 22 +++ 19 files changed, 568 insertions(+), 7 deletions(-) create mode 100644 src/Exceptionless.Core/Models/SystemSettings.cs create mode 100644 src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs create mode 100644 src/Exceptionless.Core/Repositories/Interfaces/ISystemSettingsRepository.cs create mode 100644 src/Exceptionless.Core/Repositories/SystemSettingsRepository.cs create mode 100644 src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.test.ts create mode 100644 src/Exceptionless.Web/Models/Admin/UpdateAssistantSettings.cs diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 805a2f76d4..d3286363d5 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -151,6 +151,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Core/Models/SystemSettings.cs b/src/Exceptionless.Core/Models/SystemSettings.cs new file mode 100644 index 0000000000..f93c8902ac --- /dev/null +++ b/src/Exceptionless.Core/Models/SystemSettings.cs @@ -0,0 +1,25 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models; + +public sealed class SystemSettings : IIdentity, IHaveDates +{ + public const string DefaultId = "000000000000000000000001"; + + [ObjectId] + public string Id { get; set; } = DefaultId; + + [MaxLength(200)] + public string? AssistantModel { get; set; } + + [ObjectId] + public string CreatedByUserId { get; set; } = null!; + + [ObjectId] + public string UpdatedByUserId { get; set; } = null!; + + public DateTime CreatedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } +} diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 7fd742b21b..837632f588 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -47,6 +47,7 @@ ILoggerFactory loggerFactory AddIndex(OAuthTokens = new OAuthTokenIndex(this)); AddIndex(Projects = new ProjectIndex(this)); AddIndex(SavedViews = new SavedViewIndex(this)); + AddIndex(SystemSettings = new SystemSettingsIndex(this)); AddIndex(Tokens = new TokenIndex(this)); AddIndex(Users = new UserIndex(this)); AddIndex(WebHooks = new WebHookIndex(this)); @@ -77,6 +78,7 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) public OAuthTokenIndex OAuthTokens { get; } public ProjectIndex Projects { get; } public SavedViewIndex SavedViews { get; } + public SystemSettingsIndex SystemSettings { get; } public TokenIndex Tokens { get; } public UserIndex Users { get; } public WebHookIndex WebHooks { get; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs new file mode 100644 index 0000000000..d18971c3c3 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs @@ -0,0 +1,38 @@ +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; + +namespace Exceptionless.Core.Repositories.Configuration; + +public sealed class SystemSettingsIndex : VersionedIndex +{ + private readonly ExceptionlessElasticConfiguration _configuration; + + public SystemSettingsIndex(ExceptionlessElasticConfiguration configuration) + : base(configuration, configuration.Options.ScopePrefix + "system-settings", 1) + { + _configuration = configuration; + } + + public override void ConfigureIndexMapping(TypeMappingDescriptor map) + { + map + .Dynamic(DynamicMapping.False) + .Properties(properties => properties + .SetupDefaults() + .Keyword(settings => settings.AssistantModel) + .Keyword(settings => settings.CreatedByUserId) + .Keyword(settings => settings.UpdatedByUserId)); + } + + public override void ConfigureIndex(CreateIndexRequestDescriptor index) + { + base.ConfigureIndex(index); + index.Settings(settings => settings + .NumberOfShards(1) + .NumberOfReplicas(_configuration.Options.NumberOfReplicas) + .Priority(5)); + } +} diff --git a/src/Exceptionless.Core/Repositories/Interfaces/ISystemSettingsRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/ISystemSettingsRepository.cs new file mode 100644 index 0000000000..39eeba1355 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Interfaces/ISystemSettingsRepository.cs @@ -0,0 +1,6 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories; + +namespace Exceptionless.Core.Repositories; + +public interface ISystemSettingsRepository : IRepository; diff --git a/src/Exceptionless.Core/Repositories/SystemSettingsRepository.cs b/src/Exceptionless.Core/Repositories/SystemSettingsRepository.cs new file mode 100644 index 0000000000..ffb13c5a44 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/SystemSettingsRepository.cs @@ -0,0 +1,15 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Validation; +using Foundatio.Repositories; + +namespace Exceptionless.Core.Repositories; + +public sealed class SystemSettingsRepository : RepositoryBase, ISystemSettingsRepository +{ + public SystemSettingsRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options) + : base(configuration.SystemSettings, validator, options) + { + DefaultConsistency = Consistency.Immediate; + } +} diff --git a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs index e6d7f335bc..7a26aad9a0 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs @@ -2,8 +2,12 @@ using Exceptionless.Web.Api.Filters; using Exceptionless.Web.Api.Messages; using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Assistant; +using Exceptionless.Web.Extensions; using Exceptionless.Web.Models.Admin; using Foundatio.Mediator; +using Microsoft.AspNetCore.Mvc; +using HttpResults = Microsoft.AspNetCore.Http.Results; using HttpIResult = Microsoft.AspNetCore.Http.IResult; namespace Exceptionless.Web.Api.Endpoints; @@ -20,6 +24,12 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder group.MapGet("echo", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper) => (await mediator.InvokeAsync>(new GetAdminEcho(httpContext))).ToHttpResult(resultMapper)); + group.MapGet("assistant-settings", async (AssistantModelSettingsService settingsService) + => HttpResults.Ok(await settingsService.GetAsync())); + + group.MapPut("assistant-settings", async (HttpContext httpContext, [FromBody] UpdateAssistantSettings request, AssistantModelSettingsService settingsService) + => HttpResults.Ok(await settingsService.SetModelAsync(request.Model, httpContext.Request.GetUser().Id))); + endpoints.MapGet("api/v2/admin/assistant-usage", GetAssistantUsageAsync) .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) .AddEndpointFilter() diff --git a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs new file mode 100644 index 0000000000..7c930e38a3 --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs @@ -0,0 +1,78 @@ +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Repositories; + +namespace Exceptionless.Web.Assistant; + +public sealed class AssistantModelSettingsService +{ + private readonly AppOptions _appOptions; + private readonly Func> _getSettingsAsync; + private readonly Func _saveSettingsAsync; + private readonly TimeProvider _timeProvider; + + public AssistantModelSettingsService( + ISystemSettingsRepository repository, + AppOptions appOptions, + TimeProvider timeProvider) + : this( + () => repository.GetByIdAsync(SystemSettings.DefaultId, options => options.Cache()), + async settings => await repository.SaveAsync(settings, options => options.Cache().ImmediateConsistency()), + appOptions, + timeProvider) + { + } + + internal AssistantModelSettingsService( + Func> getSettingsAsync, + Func saveSettingsAsync, + AppOptions appOptions, + TimeProvider timeProvider) + { + _getSettingsAsync = getSettingsAsync; + _saveSettingsAsync = saveSettingsAsync; + _appOptions = appOptions; + _timeProvider = timeProvider; + } + + public async Task GetAsync() => CreateResponse(await _getSettingsAsync()); + + public async Task SetModelAsync(string? model, string userId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(userId); + + string? normalizedModel = model?.Trim(); + if (String.IsNullOrWhiteSpace(normalizedModel) + || String.Equals(normalizedModel, _appOptions.AssistantOptions.Model, StringComparison.Ordinal)) + normalizedModel = null; + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + var settings = await _getSettingsAsync() ?? new SystemSettings + { + CreatedByUserId = userId, + CreatedUtc = utcNow + }; + settings.AssistantModel = normalizedModel; + settings.UpdatedByUserId = userId; + settings.UpdatedUtc = utcNow; + + await _saveSettingsAsync(settings); + + return CreateResponse(settings); + } + + private AssistantModelSettings CreateResponse(SystemSettings? settings) + { + string configuredModel = _appOptions.AssistantOptions.Model; + string? modelOverride = settings?.AssistantModel; + bool isOverridden = !String.IsNullOrWhiteSpace(modelOverride); + + return new AssistantModelSettings( + isOverridden ? modelOverride! : configuredModel, + configuredModel, + isOverridden); + } +} + +public sealed record AssistantModelSettings(string Model, string ConfiguredModel, bool IsOverridden); diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index 0a8a57f514..c52de20f52 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -17,6 +17,7 @@ public sealed class AssistantService( ExceptionlessMcpTools tools, AssistantToolContext assistantToolContext, AssistantConversationService assistantConversationService, + AssistantModelSettingsService assistantModelSettingsService, AssistantUsageService assistantUsageService, TimeProvider timeProvider, ILogger logger) @@ -42,6 +43,7 @@ public async IAsyncEnumerable StreamAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { var options = appOptions.AssistantOptions; + string model = (await assistantModelSettingsService.GetAsync()).Model; AssistantConversationState? conversationState = null; if (!String.IsNullOrWhiteSpace(request.OrganizationId) && !String.IsNullOrWhiteSpace(request.ConversationId)) { @@ -112,7 +114,7 @@ public async IAsyncEnumerable StreamAsync( } await using var providerRequest = await assistantUsageService.StartProviderRequestAsync(request.OrganizationId, providerInputCharacters); - using var response = await SendRequestAsync(messages, options, allowTools, request, cancellationToken); + using var response = await SendRequestAsync(messages, options, model, allowTools, request, cancellationToken); providerRequest.MarkAccepted(); await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); using var reader = new StreamReader(stream); @@ -382,7 +384,7 @@ await assistantConversationService.AppendToolResultsAsync( } } - private async Task SendRequestAsync(List messages, AssistantOptions options, bool allowTools, AssistantChatRequest chatRequest, CancellationToken cancellationToken) + private async Task SendRequestAsync(List messages, AssistantOptions options, string model, bool allowTools, AssistantChatRequest chatRequest, CancellationToken cancellationToken) { var client = httpClientFactory.CreateClient(nameof(AssistantService)); using var providerRequest = new HttpRequestMessage(HttpMethod.Post, options.Endpoint); @@ -391,7 +393,7 @@ private async Task SendRequestAsync(List messages, providerRequest.Headers.TryAddWithoutValidation("X-OpenRouter-Title", "Exceptionless"); var payload = new Dictionary { - ["model"] = options.Model, + ["model"] = model, ["messages"] = messages, ["stream"] = true, ["max_tokens"] = AssistantLimits.MaximumOutputTokens, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts index 9e9e67e95c..f8e5144b42 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts @@ -2,6 +2,7 @@ import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; import type { + AdminAssistantSettings, AdminAssistantUsage, AdminStats, ElasticsearchInfo, @@ -9,7 +10,8 @@ import type { MigrationsResponse, OAuthApplication, OAuthApplicationRequest, - PredefinedSavedViewDefinition + PredefinedSavedViewDefinition, + UpdateAssistantSettingsRequest } from './models'; export type RunMaintenanceJobParams = { @@ -20,6 +22,7 @@ export type RunMaintenanceJobParams = { }; export const queryKeys = { + assistantSettings: ['admin', 'assistant-settings'] as const, assistantUsage: (month: string) => ['admin', 'assistant-usage', month] as const, elasticsearch: ['admin', 'elasticsearch'] as const, migrations: ['admin', 'migrations'] as const, @@ -48,6 +51,25 @@ export function deleteOAuthApplicationMutation() { })); } +export function getAdminAssistantSettingsQuery() { + return createQuery(() => ({ + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON('admin/assistant-settings', { + signal + }); + + if (!response.ok) { + throw response.problem; + } + + return response.data!; + }, + queryKey: queryKeys.assistantSettings, + staleTime: 30 * 1000 + })); +} + export function getAdminAssistantUsageQuery(month: () => string) { return createQuery(() => ({ queryFn: async ({ signal }: { signal: AbortSignal }) => { @@ -207,6 +229,26 @@ export function postOAuthApplicationMutation() { })); } +export function putAdminAssistantSettingsMutation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + mutationFn: async (request) => { + const client = useFetchClient(); + const response = await client.putJSON('admin/assistant-settings', request); + + if (!response.ok) { + throw response.problem; + } + + return response.data!; + }, + onSuccess: (settings) => { + queryClient.setQueryData(queryKeys.assistantSettings, settings); + } + })); +} + export function putOAuthApplicationMutation() { const queryClient = useQueryClient(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts index e0c6211a52..80548bb0ef 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts @@ -30,6 +30,12 @@ export type AdminAssistantOrganizationUsage = { turns: number; }; +export type AdminAssistantSettings = { + configured_model: string; + is_overridden: boolean; + model: string; +}; + export type AdminAssistantUsage = { active_organizations: number; completion_tokens: number; @@ -171,6 +177,10 @@ export type ShardMetric = { value: number; }; +export type UpdateAssistantSettingsRequest = { + model: null | string; +}; + export const maintenanceActions: MaintenanceAction[] = [ { category: 'Elasticsearch', diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.test.ts new file mode 100644 index 0000000000..0f0de53e79 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; + +import { AssistantSettingsSchema } from './schemas'; + +describe('assistant settings schema', () => { + it('accepts the GLM 5.3 Flash OpenRouter model ID', () => { + const result = AssistantSettingsSchema.safeParse({ model: 'z-ai/glm-5.3-flash' }); + + expect(result.success).toBe(true); + }); + + it('rejects an empty model ID', () => { + const result = AssistantSettingsSchema.safeParse({ model: ' ' }); + + expect(result.success).toBe(false); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.ts index 9e77203415..84feb8e026 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/schemas.ts @@ -1,5 +1,10 @@ import { array, boolean, date, type infer as Infer, object, string } from 'zod'; +export const AssistantSettingsSchema = object({ + model: string().trim().min(1, 'Enter an OpenRouter model ID.').max(200) +}); +export type AssistantSettingsFormData = Infer; + export const RunMaintenanceJobSchema = object({ confirmText: string().min(1), organizationId: string().optional(), diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte index 495b5ebf7b..c49273ccb9 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte @@ -1,23 +1,34 @@ -
+
+ + +
+ Model Configuration + {#if settings} + + {settings.is_overridden ? 'Runtime override' : 'Deployment default'} + + {/if} +
+ Choose the OpenRouter model used for new Exie conversations and turns. Changes apply without restarting the app. +
+ {#if settingsQuery.isPending} + + + Loading model configuration... + + {:else if settingsQuery.isError} + +

Failed to load the Exie model configuration.

+
+ {:else} +
{ + event.preventDefault(); + void settingsForm.handleSubmit(); + }} + > + + + + {#snippet children(field)} + + OpenRouter model ID + field.handleChange(event.currentTarget.value)} + aria-invalid={ariaInvalid(field)} + autocomplete="off" + placeholder="z-ai/glm-5.3-flash" + /> + + Enter a model slug such as + z-ai/glm-5.3-flash. The + deployment default is {settings?.configured_model}. + + + + {/snippet} + + state.errors}> + {#snippet children(errors)} + + {/snippet} + + + + + {#if settings?.is_overridden} + + {/if} + state.isSubmitting}> + {#snippet children(isSubmitting)} + + {/snippet} + + +
+ {/if} +
+
Monthly Exie usage, provider cost, and plan-limit health across all organizations