Add persistent runtime system configuration

src/Exceptionless.Core/Bootstrapper.cs

@@ -151,6 +151,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
         services.AddSingleton<IUserRepository, UserRepository>();
         services.AddSingleton<IWebHookRepository, WebHookRepository>();
         services.AddSingleton<ISavedViewRepository, SavedViewRepository>();
+        services.AddSingleton<ISystemSettingsRepository, SystemSettingsRepository>();
         services.AddSingleton<ITokenRepository, TokenRepository>();
 
         services.AddSingleton<IGeocodeService, NullGeocodeService>();
@@ -190,6 +191,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
         services.AddSingleton<UserAgentParser>();
         services.AddSingleton<ICoreLastReferenceIdManager, NullCoreLastReferenceIdManager>();
 
+        services.AddSingleton<SystemSettingsService>();
         services.AddSingleton<NotificationService>();
         services.AddSingleton<OrganizationService>();
         services.AddStartupAction<OrganizationService>();

src/Exceptionless.Core/Models/SystemSettings.cs

@@ -0,0 +1,32 @@
+using System.ComponentModel.DataAnnotations;
+using Exceptionless.Core.Attributes;
+using Exceptionless.Core.Messaging.Models;
+using Foundatio.Repositories.Models;
+
+namespace Exceptionless.Core.Models;
+
+public sealed class SystemSettings : IIdentity, IHaveDates
+{
+    public const string DefaultId = "000000000000000000000001";
+
+    [ObjectId]
+    public string Id { get; set; } = DefaultId;
+
+    [MaxLength(200)]
+    public string? AssistantModel { get; set; }
+
+    public bool? AssistantEnabled { get; set; }
+
+    public bool? EventSubmissionEnabled { get; set; }
+
+    public SystemNotification? SystemNotification { get; set; }
+
+    [ObjectId]
+    public string CreatedByUserId { get; set; } = null!;
+
+    [ObjectId]
+    public string UpdatedByUserId { get; set; } = null!;
+
+    public DateTime CreatedUtc { get; set; }
+    public DateTime UpdatedUtc { get; set; }
+}

src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs

@@ -47,6 +47,7 @@ ILoggerFactory loggerFactory
         AddIndex(OAuthTokens = new OAuthTokenIndex(this));
         AddIndex(Projects = new ProjectIndex(this));
         AddIndex(SavedViews = new SavedViewIndex(this));
+        AddIndex(SystemSettings = new SystemSettingsIndex(this));
         AddIndex(Tokens = new TokenIndex(this));
         AddIndex(Users = new UserIndex(this));
         AddIndex(WebHooks = new WebHookIndex(this));
@@ -77,6 +78,7 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder)
     public OAuthTokenIndex OAuthTokens { get; }
     public ProjectIndex Projects { get; }
     public SavedViewIndex SavedViews { get; }
+    public SystemSettingsIndex SystemSettings { get; }
     public TokenIndex Tokens { get; }
     public UserIndex Users { get; }
     public WebHookIndex WebHooks { get; }

src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs

@@ -0,0 +1,45 @@
+using Elastic.Clients.Elasticsearch.IndexManagement;
+using Elastic.Clients.Elasticsearch.Mapping;
+using Exceptionless.Core.Models;
+using Foundatio.Repositories.Elasticsearch.Configuration;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+
+namespace Exceptionless.Core.Repositories.Configuration;
+
+public sealed class SystemSettingsIndex : VersionedIndex<SystemSettings>
+{
+    private readonly ExceptionlessElasticConfiguration _configuration;
+
+    public SystemSettingsIndex(ExceptionlessElasticConfiguration configuration)
+        : base(configuration, configuration.Options.ScopePrefix + "system-settings", 1)
+    {
+        _configuration = configuration;
+    }
+
+    public override void ConfigureIndexMapping(TypeMappingDescriptor<SystemSettings> map)
+    {
+        map
+            .Dynamic(DynamicMapping.False)
+            .Properties(properties => properties
+                .SetupDefaults()
+                .Keyword(settings => settings.AssistantModel)
+                .Boolean(settings => settings.AssistantEnabled)
+                .Boolean(settings => settings.EventSubmissionEnabled)
+                .Object(settings => settings.SystemNotification, notification => notification.Properties(properties => properties
+                    .Date("date")
+                    .Text("message")
+                    .Keyword("level")
+                    .Keyword("target")))
+                .Keyword(settings => settings.CreatedByUserId)
+                .Keyword(settings => settings.UpdatedByUserId));
+    }
+
+    public override void ConfigureIndex(CreateIndexRequestDescriptor index)
+    {
+        base.ConfigureIndex(index);
+        index.Settings(settings => settings
+            .NumberOfShards(1)
+            .NumberOfReplicas(_configuration.Options.NumberOfReplicas)
+            .Priority(5));
+    }
+}

src/Exceptionless.Core/Repositories/Interfaces/ISystemSettingsRepository.cs

@@ -0,0 +1,6 @@
+using Exceptionless.Core.Models;
+using Foundatio.Repositories;
+
+namespace Exceptionless.Core.Repositories;
+
+public interface ISystemSettingsRepository : IRepository<SystemSettings>;

src/Exceptionless.Core/Repositories/SystemSettingsRepository.cs

@@ -0,0 +1,15 @@
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Validation;
+using Foundatio.Repositories;
+
+namespace Exceptionless.Core.Repositories;
+
+public sealed class SystemSettingsRepository : RepositoryBase<SystemSettings>, ISystemSettingsRepository
+{
+    public SystemSettingsRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options)
+        : base(configuration.SystemSettings, validator, options)
+    {
+        DefaultConsistency = Consistency.Immediate;
+    }
+}

src/Exceptionless.Core/Services/NotificationService.cs

@@ -7,29 +7,54 @@
 
 namespace Exceptionless.Core.Services;
 
-public class NotificationService(ICacheClient cacheClient, IMessagePublisher messagePublisher, TimeProvider timeProvider, CacheLockProvider lockProvider)
+public class NotificationService(ICacheClient cacheClient, IMessagePublisher messagePublisher, TimeProvider timeProvider, CacheLockProvider lockProvider, SystemSettingsService systemSettingsService)
 {
     private const string SystemNotificationCacheKey = "system-notification";
+    private const string SystemNotificationCompatibilityCacheKey = "system-notification-compatibility";
     private static readonly TimeSpan OrganizationNotificationLockTimeout = TimeSpan.FromMinutes(90);
 
     public async Task<SystemNotification?> GetSystemNotificationAsync()
     {
-        var result = await cacheClient.GetAsync<SystemNotification>(SystemNotificationCacheKey);
-        return result.HasValue ? result.Value : null;
+        var settings = await systemSettingsService.GetAsync();
+        var durableNotification = settings?.SystemNotification;
+        var legacyNotification = await cacheClient.GetAsync<SystemNotification>(SystemNotificationCacheKey);
+
+        // Reconcile a newer notification written by an older instance during a rolling deployment.
+        if (legacyNotification.HasValue && (durableNotification is null || legacyNotification.Value.Date > durableNotification.Date))
+            return legacyNotification.Value;
+
+        if (durableNotification is null)
+            return legacyNotification.HasValue ? legacyNotification.Value : null;
+
+        // Older instances clear only the legacy key. A separate marker distinguishes that targeted
+        // removal from a full cache restart, where Elasticsearch must remain authoritative.
+        if (!legacyNotification.HasValue)
+        {
+            var compatibilityDate = await cacheClient.GetAsync<DateTime>(SystemNotificationCompatibilityCacheKey);
+            if (compatibilityDate.HasValue && compatibilityDate.Value >= durableNotification.Date)
+                return null;
+        }
+
+        return durableNotification;
     }
 
-    public async Task<SystemNotification> SetSystemNotificationAsync(string message, SystemNotificationLevel level = SystemNotificationLevel.Info, SystemNotificationTarget target = SystemNotificationTarget.Both, bool publish = true)
+    public async Task<SystemNotification> SetSystemNotificationAsync(string message, string userId, SystemNotificationLevel level = SystemNotificationLevel.Info, SystemNotificationTarget target = SystemNotificationTarget.Both, bool publish = true)
     {
         var notification = new SystemNotification { Date = timeProvider.GetUtcNow().UtcDateTime, Message = message, Level = level, Target = target };
+        await systemSettingsService.UpdateAsync(userId, settings => settings.SystemNotification = notification);
+        // Keep older instances in a rolling deployment synchronized until they all read durable settings.
         await cacheClient.SetAsync(SystemNotificationCacheKey, notification);
+        await cacheClient.SetAsync(SystemNotificationCompatibilityCacheKey, notification.Date);
         if (publish)
             await messagePublisher.PublishAsync(notification);
         return notification;
     }
 
-    public async Task ClearSystemNotificationAsync(bool publish = true)
+    public async Task ClearSystemNotificationAsync(string userId, bool publish = true)
     {
+        await systemSettingsService.UpdateAsync(userId, settings => settings.SystemNotification = null);
         await cacheClient.RemoveAsync(SystemNotificationCacheKey);
+        await cacheClient.RemoveAsync(SystemNotificationCompatibilityCacheKey);
         if (publish)
             await messagePublisher.PublishAsync(new SystemNotification { Date = timeProvider.GetUtcNow().UtcDateTime });
     }

src/Exceptionless.Core/Services/SystemSettingsService.cs

@@ -0,0 +1,95 @@
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories;
+using Foundatio.Lock;
+using Foundatio.Repositories;
+
+namespace Exceptionless.Core.Services;
+
+public sealed class SystemSettingsService
+{
+    private readonly AppOptions _appOptions;
+    private readonly Func<Task<SystemSettings?>> _getSettingsAsync;
+    private readonly ILockProvider? _lockProvider;
+    private readonly Func<SystemSettings, Task> _saveSettingsAsync;
+    private readonly TimeProvider _timeProvider;
+
+    public SystemSettingsService(
+        ISystemSettingsRepository repository,
+        ILockProvider lockProvider,
+        AppOptions appOptions,
+        TimeProvider timeProvider)
+        : this(
+            () => repository.GetByIdAsync(SystemSettings.DefaultId, options => options.Cache()),
+            async settings => await repository.SaveAsync(settings, options => options.Cache().ImmediateConsistency()),
+            lockProvider,
+            appOptions,
+            timeProvider)
+    {
+    }
+
+    internal SystemSettingsService(
+        Func<Task<SystemSettings?>> getSettingsAsync,
+        Func<SystemSettings, Task> saveSettingsAsync,
+        AppOptions appOptions,
+        TimeProvider timeProvider)
+        : this(getSettingsAsync, saveSettingsAsync, null, appOptions, timeProvider)
+    {
+    }
+
+    private SystemSettingsService(
+        Func<Task<SystemSettings?>> getSettingsAsync,
+        Func<SystemSettings, Task> saveSettingsAsync,
+        ILockProvider? lockProvider,
+        AppOptions appOptions,
+        TimeProvider timeProvider)
+    {
+        _getSettingsAsync = getSettingsAsync;
+        _saveSettingsAsync = saveSettingsAsync;
+        _lockProvider = lockProvider;
+        _appOptions = appOptions;
+        _timeProvider = timeProvider;
+    }
+
+    public Task<SystemSettings?> GetAsync() => _getSettingsAsync();
+
+    public async Task<SystemSettings> UpdateAsync(string userId, Action<SystemSettings> update)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(userId);
+        ArgumentNullException.ThrowIfNull(update);
+
+        if (_lockProvider is null)
+            return await UpdateCoreAsync(userId, update);
+
+        await using var settingsLock = await _lockProvider.AcquireAsync("system-settings:update", TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15));
+        return await UpdateCoreAsync(userId, update);
+    }
+
+    private async Task<SystemSettings> UpdateCoreAsync(string userId, Action<SystemSettings> update)
+    {
+        var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+        var settings = await _getSettingsAsync() ?? new SystemSettings
+        {
+            CreatedByUserId = userId,
+            CreatedUtc = utcNow
+        };
+
+        update(settings);
+        settings.UpdatedByUserId = userId;
+        settings.UpdatedUtc = utcNow;
+        await _saveSettingsAsync(settings);
+
+        return settings;
+    }
+
+    public async Task<bool> IsAssistantEnabledAsync()
+    {
+        var settings = await _getSettingsAsync();
+        return settings?.AssistantEnabled ?? _appOptions.AssistantOptions.Enabled;
+    }
+
+    public async Task<bool> IsEventSubmissionEnabledAsync()
+    {
+        var settings = await _getSettingsAsync();
+        return settings?.EventSubmissionEnabled ?? !_appOptions.EventSubmissionDisabled;
+    }
+}

src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs

@@ -1,9 +1,15 @@
+using Exceptionless.Core;
 using Exceptionless.Core.Authorization;
+using Exceptionless.Core.Services;
 using Exceptionless.Web.Api.Filters;
 using Exceptionless.Web.Api.Messages;
 using Exceptionless.Web.Api.Results;
+using Exceptionless.Web.Assistant;
+using Exceptionless.Web.Extensions;
 using Exceptionless.Web.Models.Admin;
 using Foundatio.Mediator;
+using Microsoft.AspNetCore.Mvc;
+using HttpResults = Microsoft.AspNetCore.Http.Results;
 using HttpIResult = Microsoft.AspNetCore.Http.IResult;
 
 namespace Exceptionless.Web.Api.Endpoints;
@@ -20,6 +26,59 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder
         group.MapGet("echo", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper<HttpIResult> resultMapper)
             => (await mediator.InvokeAsync<Result<object>>(new GetAdminEcho(httpContext))).ToHttpResult(resultMapper));
 
+        endpoints.MapGet("api/v2/admin/assistant-settings", async (AssistantModelSettingsService settingsService)
+            => HttpResults.Ok(await settingsService.GetAsync()))
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .AddEndpointFilter<AutoValidationEndpointFilter>()
+            .Produces<AssistantModelSettings>(StatusCodes.Status200OK)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden)
+            .WithTags(nameof(AdminEndpoints))
+            .WithSummary("Get Exie assistant settings");
+
+        endpoints.MapPut("api/v2/admin/assistant-settings", async (HttpContext httpContext, [FromBody] UpdateAssistantSettings request, AssistantModelSettingsService settingsService)
+            => HttpResults.Ok(await settingsService.SetModelAsync(request.Model, httpContext.Request.GetUser().Id)))
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .AddEndpointFilter<AutoValidationEndpointFilter>()
+            .Accepts<UpdateAssistantSettings>("application/json", "application/*+json")
+            .Produces<AssistantModelSettings>(StatusCodes.Status200OK)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden)
+            .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+            .WithTags(nameof(AdminEndpoints))
+            .WithSummary("Update Exie assistant settings");
+
+        endpoints.MapPut("api/v2/admin/assistant-settings/enabled", async (HttpContext httpContext, [FromBody] UpdateAssistantEnabledSettings request, AssistantModelSettingsService settingsService)
+            => HttpResults.Ok(await settingsService.SetEnabledAsync(request.Enabled, httpContext.Request.GetUser().Id)))
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .AddEndpointFilter<AutoValidationEndpointFilter>()
+            .Accepts<UpdateAssistantEnabledSettings>("application/json", "application/*+json")
+            .Produces<AssistantModelSettings>(StatusCodes.Status200OK)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden)
+            .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+            .WithTags(nameof(AdminEndpoints))
+            .WithSummary("Update Exie assistant availability");
+
+        endpoints.MapGet("api/v2/admin/event-submission-settings", GetEventSubmissionSettingsAsync)
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .Produces<EventSubmissionSettings>(StatusCodes.Status200OK)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden)
+            .WithTags(nameof(AdminEndpoints))
+            .WithSummary("Get event submission settings");
+
+        endpoints.MapPut("api/v2/admin/event-submission-settings", UpdateEventSubmissionSettingsAsync)
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .AddEndpointFilter<AutoValidationEndpointFilter>()
+            .Accepts<UpdateEventSubmissionSettings>("application/json", "application/*+json")
+            .Produces<EventSubmissionSettings>(StatusCodes.Status200OK)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden)
+            .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+            .WithTags(nameof(AdminEndpoints))
+            .WithSummary("Update event submission settings");
+
         endpoints.MapGet("api/v2/admin/assistant-usage", GetAssistantUsageAsync)
             .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
             .AddEndpointFilter<AutoValidationEndpointFilter>()
@@ -47,4 +106,28 @@ private static async Task<HttpIResult> GetAssistantUsageAsync(
         DateTime? month = null,
         int limit = 100)
         => (await mediator.InvokeAsync<Result<object>>(new GetAdminAssistantUsage(month, limit, httpContext))).ToHttpResult(resultMapper);
+
+    private static async Task<HttpIResult> GetEventSubmissionSettingsAsync(SystemSettingsService settingsService, AppOptions appOptions)
+    {
+        var settings = await settingsService.GetAsync();
+        return HttpResults.Ok(CreateEventSubmissionSettings(settings?.EventSubmissionEnabled, appOptions));
+    }
+
+    private static async Task<HttpIResult> UpdateEventSubmissionSettingsAsync(
+        HttpContext httpContext,
+        [FromBody] UpdateEventSubmissionSettings request,
+        SystemSettingsService settingsService,
+        AppOptions appOptions)
+    {
+        bool configuredEnabled = !appOptions.EventSubmissionDisabled;
+        bool? enabledOverride = request.Enabled == configuredEnabled ? null : request.Enabled;
+        var settings = await settingsService.UpdateAsync(httpContext.Request.GetUser().Id, value => value.EventSubmissionEnabled = enabledOverride);
+        return HttpResults.Ok(CreateEventSubmissionSettings(settings.EventSubmissionEnabled, appOptions));
+    }
+
+    private static EventSubmissionSettings CreateEventSubmissionSettings(bool? enabledOverride, AppOptions appOptions)
+    {
+        bool configuredEnabled = !appOptions.EventSubmissionDisabled;
+        return new EventSubmissionSettings(enabledOverride ?? configuredEnabled, configuredEnabled, enabledOverride.HasValue);
+    }
 }