Add adaptive guided tours to the new UI

src/Exceptionless.Core/Models/Data/ProductTourProgress.cs

@@ -0,0 +1,22 @@
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+
+namespace Exceptionless.Core.Models.Data;
+
+public record ProductTourProgress
+{
+    public ProductTourStatus Status { get; set; }
+    public DateTime UpdatedUtc { get; set; }
+    public int Version { get; set; }
+}
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum ProductTourStatus
+{
+    [JsonStringEnumMemberName("completed")]
+    [EnumMember(Value = "completed")]
+    Completed,
+    [JsonStringEnumMemberName("dismissed")]
+    [EnumMember(Value = "dismissed")]
+    Dismissed
+}

src/Exceptionless.Core/Models/Data/ProductTours.cs

@@ -0,0 +1,88 @@
+using System.Collections.Frozen;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Extensions;
+
+namespace Exceptionless.Core.Models.Data;
+
+public static class ProductTours
+{
+    public const string ConfigureProject = "configure-project";
+    public const string CreateSavedView = "create-saved-view";
+    public const string ExieAnnouncement = "exie-announcement";
+    public const string InvestigateError = "investigate-error";
+    public const string MeetExie = "meet-exie";
+    public const string UiOverview = "ui-overview";
+    public const string Welcome = "welcome";
+
+    public static IReadOnlyDictionary<string, int> Versions { get; } = new Dictionary<string, int>(StringComparer.Ordinal)
+    {
+        [ConfigureProject] = 1,
+        [CreateSavedView] = 1,
+        [ExieAnnouncement] = 1,
+        [InvestigateError] = 1,
+        [MeetExie] = 1,
+        [UiOverview] = 1,
+        [Welcome] = 1
+    }.ToFrozenDictionary(StringComparer.Ordinal);
+
+    public static IReadOnlyCollection<ProductTourTelemetryEvent> TelemetryEvents { get; } = Enum.GetValues<ProductTourTelemetryEvent>();
+    public static IReadOnlyCollection<ProductTourLaunchSource> LaunchSources { get; } = Enum.GetValues<ProductTourLaunchSource>();
+
+    public static bool IsKnown(string name) => Versions.ContainsKey(name);
+
+    public static bool IsValid(string name, int version)
+    {
+        return Versions.TryGetValue(name, out int currentVersion) && version > 0 && version <= currentVersion;
+    }
+
+    public static string CreateTelemetrySource(
+        ProductTourTelemetryEvent telemetryEvent,
+        string tourName,
+        int version,
+        ProductTourLaunchSource launchSource)
+    {
+        return $"product-tour.{GetTelemetryName(telemetryEvent)}.{tourName}.v{version}.{GetLaunchSourceName(launchSource)}";
+    }
+
+    public static string GetTelemetryName(ProductTourTelemetryEvent telemetryEvent) => telemetryEvent.ToString().ToLowerUnderscoredWords('-');
+
+    public static string GetLaunchSourceName(ProductTourLaunchSource launchSource) => launchSource.ToString().ToLowerUnderscoredWords('-');
+}
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum ProductTourTelemetryEvent
+{
+    [JsonStringEnumMemberName("completed")]
+    [EnumMember(Value = "completed")]
+    Completed,
+    [JsonStringEnumMemberName("dismissed")]
+    [EnumMember(Value = "dismissed")]
+    Dismissed,
+    [JsonStringEnumMemberName("shown")]
+    [EnumMember(Value = "shown")]
+    Shown,
+    [JsonStringEnumMemberName("started")]
+    [EnumMember(Value = "started")]
+    Started
+}
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum ProductTourLaunchSource
+{
+    [JsonStringEnumMemberName("automatic")]
+    [EnumMember(Value = "automatic")]
+    Automatic,
+    [JsonStringEnumMemberName("catalog")]
+    [EnumMember(Value = "catalog")]
+    Catalog,
+    [JsonStringEnumMemberName("command-palette")]
+    [EnumMember(Value = "command-palette")]
+    CommandPalette,
+    [JsonStringEnumMemberName("feature-announcement")]
+    [EnumMember(Value = "feature-announcement")]
+    FeatureAnnouncement,
+    [JsonStringEnumMemberName("help-menu")]
+    [EnumMember(Value = "help-menu")]
+    HelpMenu
+}

src/Exceptionless.Core/Models/User.cs

@@ -1,6 +1,7 @@
 using System.Collections.ObjectModel;
 using System.ComponentModel.DataAnnotations;
 using Exceptionless.Core.Attributes;
+using Exceptionless.Core.Models.Data;
 using Foundatio.Repositories.Models;
 
 namespace Exceptionless.Core.Models;
@@ -24,6 +25,7 @@ public record User : IIdentity, IHaveDates, IValidatableObject
     public DateTime PasswordResetTokenExpiration { get; set; }
     public ICollection<OAuthAccount> OAuthAccounts { get; init; } = new Collection<OAuthAccount>();
     public ICollection<UserOrganizationPreference> OrganizationPreferences { get; init; } = new Collection<UserOrganizationPreference>();
+    public IDictionary<string, ProductTourProgress> ProductTours { get; init; } = new Dictionary<string, ProductTourProgress>(StringComparer.Ordinal);
 
     /// <summary>
     /// Gets or sets the users Full Name.

src/Exceptionless.Core/Repositories/EventRepository.cs

@@ -1,10 +1,14 @@
-using Elastic.Clients.Elasticsearch.QueryDsl;
+using Elastic.Clients.Elasticsearch.Aggregations;
+using Elastic.Clients.Elasticsearch.QueryDsl;
 using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Data;
 using Exceptionless.Core.Repositories.Configuration;
 using Exceptionless.Core.Repositories.Queries;
 using Exceptionless.Core.Validation;
 using Exceptionless.DateTimeExtensions;
 using Foundatio.Repositories;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Exceptions;
 using Foundatio.Repositories.Models;
 
 namespace Exceptionless.Core.Repositories;
@@ -82,6 +86,163 @@ public Task<FindResults<PersistentEvent>> GetByReferenceIdAsync(string projectId
         return FindAsync(q => q.Project(projectId).FieldEquals(e => e.ReferenceId, referenceId).SortDescending(e => e.Date), o => o.PageLimit(10));
     }
 
+    public async Task<ProductTourUsageResult> GetProductTourUsageAsync(string projectId, DateTime utcStart, DateTime utcEnd, int recentLimit = 500)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(projectId);
+        ArgumentOutOfRangeException.ThrowIfLessThan(recentLimit, 1);
+        if (utcEnd <= utcStart)
+            throw new ArgumentOutOfRangeException(nameof(utcEnd), "The end date must be later than the start date.");
+
+        var sourcesByTour = ProductTours.Versions.ToDictionary(
+            pair => pair.Key,
+            pair => CreateProductTourSources(pair.Key, pair.Value),
+            StringComparer.Ordinal);
+        var sourcesByName = sourcesByTour.Values
+            .SelectMany(sources => sources)
+            .ToDictionary(source => source.Raw, StringComparer.Ordinal);
+        string[] allSources = sourcesByName.Keys.ToArray();
+
+        var aggregationTask = GetProductTourUsageToursAsync(projectId, utcStart, utcEnd, sourcesByTour, allSources);
+        var recentTask = FindAsync(query => ApplyProductTourUsageFilter(query, projectId, utcStart, utcEnd, allSources)
+            .SortDescending(ev => ev.Date), options => options.PageLimit(recentLimit));
+
+        await Task.WhenAll(aggregationTask, recentTask);
+        var recentEvents = (await recentTask).Documents
+            .Select(ev => ev.Source is not null && sourcesByName.TryGetValue(ev.Source, out var source) ? new ProductTourUsageEvent(ev, source) : null)
+            .OfType<ProductTourUsageEvent>()
+            .ToArray();
+        return new ProductTourUsageResult(await aggregationTask, recentEvents);
+    }
+
+    private async Task<IReadOnlyCollection<ProductTourUsageTour>> GetProductTourUsageToursAsync(
+        string projectId,
+        DateTime utcStart,
+        DateTime utcEnd,
+        IReadOnlyDictionary<string, ProductTourUsageSource[]> sourcesByTour,
+        string[] allSources)
+    {
+        var query = ApplyProductTourUsageFilter(NewQuery(), projectId, utcStart, utcEnd, allSources);
+        var options = ConfigureOptions(null);
+        await OnBeforeQueryAsync(query, options, typeof(PersistentEvent));
+        await RefreshForConsistency(query, options);
+
+        var search = (await CreateSearchDescriptorAsync(query, options))
+            .Size(0)
+            .Aggregations(CreateProductTourAggregations(sourcesByTour));
+        var response = await _client.SearchAsync<PersistentEvent>(search);
+        _logger.LogRequest(response);
+
+        if (!response.IsValidResponse)
+            throw new DocumentException($"Error getting product tour usage: {response.ElasticsearchServerError?.Error?.Reason}", response.ApiCallDetails.OriginalException);
+
+        var tours = new List<ProductTourUsageTour>();
+        foreach ((string tourName, ProductTourUsageSource[] sources) in sourcesByTour)
+        {
+            if (response.Aggregations is null
+                || !response.Aggregations.TryGetValue(tourName, out var aggregate)
+                || aggregate is not FilterAggregate filterAggregate
+                || filterAggregate.Aggregations is null)
+                continue;
+
+            if (!filterAggregate.Aggregations.TryGetValue("sources", out var sourceAggregate) || sourceAggregate is not StringTermsAggregate sourceTerms)
+                continue;
+
+            var buckets = new List<ProductTourUsageBucket>();
+            foreach (var bucket in sourceTerms.Buckets)
+            {
+                if (!bucket.Key.TryGetString(out string? sourceValue))
+                    continue;
+
+                var source = sources.FirstOrDefault(source => String.Equals(source.Raw, sourceValue, StringComparison.Ordinal));
+                if (source is null)
+                    continue;
+
+                long count = bucket.Aggregations is { } bucketAggregations
+                    && bucketAggregations.TryGetValue("count", out var countAggregate)
+                    && countAggregate is SumAggregate sum
+                    ? Convert.ToInt64(sum.Value)
+                    : bucket.DocCount;
+                DateTime? lastUtc = bucket.Aggregations is { } lastAggregations
+                    && lastAggregations.TryGetValue("last", out var lastAggregate)
+                    && lastAggregate is MaxAggregate max
+                    ? max.ValueAsString is null ? null : DateTime.Parse(max.ValueAsString, null, System.Globalization.DateTimeStyles.RoundtripKind)
+                    : null;
+                buckets.Add(new ProductTourUsageBucket(source, count, lastUtc));
+            }
+
+            long uniqueUsers = filterAggregate.Aggregations.TryGetValue("users", out var usersAggregate) && usersAggregate is CardinalityAggregate cardinality
+                ? Convert.ToInt64(cardinality.Value)
+                : 0;
+            if (buckets.Count > 0)
+                tours.Add(new ProductTourUsageTour(tourName, uniqueUsers, buckets));
+        }
+
+        return tours;
+    }
+
+    private IDictionary<string, Aggregation> CreateProductTourAggregations(IReadOnlyDictionary<string, ProductTourUsageSource[]> sourcesByTour)
+    {
+        string sourceField = ElasticIndex.MappingResolver.GetNonAnalyzedFieldName(InferField(ev => ev.Source))!;
+        string userPath = EventIndexExtensions.DataPath<UserInfo>(Event.KnownDataKeys.UserInfo, user => user.Identity);
+        string userField = ElasticIndex.MappingResolver.GetNonAnalyzedFieldName(userPath) ?? userPath;
+        var aggregations = new Dictionary<string, Aggregation>(StringComparer.Ordinal);
+
+        foreach ((string tourName, ProductTourUsageSource[] sources) in sourcesByTour)
+        {
+            aggregations[tourName] = new Aggregation
+            {
+                Filter = new TermsQuery
+                {
+                    Field = sourceField,
+                    Terms = new TermsQueryField(sources.Select(source => (Elastic.Clients.Elasticsearch.FieldValue)source.Raw).ToArray())
+                },
+                Aggregations = new Dictionary<string, Aggregation>
+                {
+                    ["sources"] = new Aggregation
+                    {
+                        Terms = new TermsAggregation { Field = sourceField, Size = sources.Length },
+                        Aggregations = new Dictionary<string, Aggregation>
+                        {
+                            ["count"] = new SumAggregation { Field = InferField(ev => ev.Count), Missing = 1 },
+                            ["last"] = new MaxAggregation { Field = InferField(ev => ev.Date) }
+                        }
+                    },
+                    ["users"] = new CardinalityAggregation { Field = userField }
+                }
+            };
+        }
+
+        return aggregations;
+    }
+
+    private static IRepositoryQuery<PersistentEvent> ApplyProductTourUsageFilter(
+        IRepositoryQuery<PersistentEvent> query,
+        string projectId,
+        DateTime utcStart,
+        DateTime utcEnd,
+        string[] sources)
+    {
+        return query
+            .Project(projectId)
+            .FieldEquals(ev => ev.Type, Event.KnownTypes.FeatureUsage)
+            .FieldEquals(ev => ev.Source, sources)
+            .DateRange(utcStart, utcEnd, (PersistentEvent ev) => ev.Date)
+            .Index(utcStart, utcEnd);
+    }
+
+    private static ProductTourUsageSource[] CreateProductTourSources(string tourName, int currentVersion)
+    {
+        return Enumerable.Range(1, currentVersion)
+            .SelectMany(version => ProductTours.TelemetryEvents.SelectMany(telemetryEvent => ProductTours.LaunchSources.Select(launchSource =>
+                new ProductTourUsageSource(
+                    ProductTours.CreateTelemetrySource(telemetryEvent, tourName, version, launchSource),
+                    telemetryEvent,
+                    tourName,
+                    version,
+                    launchSource))))
+            .ToArray();
+    }
+
     public async Task<PreviousAndNextEventIdResult> GetPreviousAndNextEventIdsAsync(PersistentEvent ev, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null)
     {
         var previous = GetPreviousEventIdAsync(ev, systemFilter, utcStart, utcEnd);

src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs

@@ -13,6 +13,7 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject<Per
     Task<bool> UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true);
     Task<long> RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor<PersistentEvent>? options = null);
     Task<long> RemoveAllByStackIdsAsync(string[] stackIds);
+    Task<ProductTourUsageResult> GetProductTourUsageAsync(string projectId, DateTime utcStart, DateTime utcEnd, int recentLimit = 500);
 }
 
 public static class EventRepositoryExtensions

src/Exceptionless.Core/Repositories/ProductTourUsageResult.cs

@@ -0,0 +1,21 @@
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Data;
+
+namespace Exceptionless.Core.Repositories;
+
+public sealed record ProductTourUsageResult(
+    IReadOnlyCollection<ProductTourUsageTour> Tours,
+    IReadOnlyCollection<ProductTourUsageEvent> RecentEvents);
+
+public sealed record ProductTourUsageTour(string Name, long UniqueUsers, IReadOnlyCollection<ProductTourUsageBucket> Buckets);
+
+public sealed record ProductTourUsageBucket(ProductTourUsageSource Source, long Count, DateTime? LastUtc);
+
+public sealed record ProductTourUsageEvent(PersistentEvent Event, ProductTourUsageSource Source);
+
+public sealed record ProductTourUsageSource(
+    string Raw,
+    ProductTourTelemetryEvent Event,
+    string TourName,
+    int Version,
+    ProductTourLaunchSource LaunchSource);

src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs

@@ -87,6 +87,14 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder
             .Produces(StatusCodes.Status401Unauthorized)
             .Produces(StatusCodes.Status403Forbidden);
 
+        endpoints.MapGet("api/v2/admin/product-tour-usage", GetProductTourUsageAsync)
+            .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy)
+            .AddEndpointFilter<AutoValidationEndpointFilter>()
+            .Produces<AdminProductTourUsageResponse>(StatusCodes.Status200OK)
+            .ProducesProblem(StatusCodes.Status400BadRequest)
+            .Produces(StatusCodes.Status401Unauthorized)
+            .Produces(StatusCodes.Status403Forbidden);
+
         group.MapPost("change-plan", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper<HttpIResult> resultMapper, string organizationId, string planId)
             => (await mediator.InvokeAsync<Result<object>>(new AdminChangePlan(organizationId, planId, httpContext))).ToHttpResult(resultMapper));
 
@@ -130,4 +138,11 @@ private static EventSubmissionSettings CreateEventSubmissionSettings(bool? enabl
         bool configuredEnabled = !appOptions.EventSubmissionDisabled;
         return new EventSubmissionSettings(enabledOverride ?? configuredEnabled, configuredEnabled, enabledOverride.HasValue);
     }
+
+    private static async Task<HttpIResult> GetProductTourUsageAsync(
+        IMediator mediator,
+        IMediatorResultMapper<HttpIResult> resultMapper,
+        DateTime? month = null,
+        int limit = 100)
+        => (await mediator.InvokeAsync<Result<object>>(new GetAdminProductTourUsage(month, limit))).ToHttpResult(resultMapper);
 }

src/Exceptionless.Web/Api/Endpoints/UserEndpoints.cs

@@ -1,4 +1,5 @@
 using Exceptionless.Core.Authorization;
+using Exceptionless.Core.Models.Data;
 using Exceptionless.Core.Extensions;
 using Exceptionless.Web.Api.Filters;
 using Exceptionless.Web.Api.Infrastructure;
@@ -37,6 +38,27 @@ public static IEndpointRouteBuilder MapUserEndpoints(this IEndpointRouteBuilder
             }
         });
 
+        group.MapPut("users/me/product-tours/{tourName:regex(^[a-z0-9]+(?:-[a-z0-9]+)*$):maxlength(64)}", async (string tourName, IMediator mediator, IMediatorResultMapper<HttpIResult> resultMapper, [FromBody] UpdateProductTourProgress progress)
+            => (await mediator.InvokeAsync<Result<ProductTourProgress>>(new UserMessages.UpdateCurrentUserProductTour(tourName, progress))).ToHttpResult(resultMapper))
+        .Accepts<UpdateProductTourProgress>(false, "application/json")
+        .Produces<ProductTourProgress>()
+        .ProducesProblem(StatusCodes.Status400BadRequest)
+        .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+        .ProducesProblem(StatusCodes.Status404NotFound)
+        .WithSummary("Update current user product tour progress")
+        .WithMetadata(new EndpointDocumentation {
+            RequestBodyDescription = "The versioned product tour outcome.",
+            RequestBodyRequired = true,
+            ParameterDescriptions = new() {
+                ["tourName"] = "The stable product tour name.",
+            },
+            ResponseDescriptions = new() {
+                ["400"] = "The request body is missing or malformed.",
+                ["422"] = "The product tour progress is invalid.",
+                ["404"] = "The current user could not be found.",
+            }
+        });
+
         group.MapGet("users/me/oauth-grants", async (IMediator mediator, IMediatorResultMapper<HttpIResult> resultMapper)
             => (await mediator.InvokeAsync<Result<IReadOnlyCollection<ViewOAuthGrant>>>(new UserMessages.GetCurrentUserOAuthGrants())).ToHttpResult(resultMapper))
         .Produces<IReadOnlyCollection<ViewOAuthGrant>>()