Add resizable saved-view columns

src/Exceptionless.Core/Bootstrapper.cs

@@ -55,7 +55,7 @@ namespace Exceptionless.Core;
 
 public class Bootstrapper
 {
-    public static void RegisterServices(IServiceCollection services, AppOptions appOptions)
+    public static void RegisterServices(IServiceCollection services, AppOptions appOptions, bool runDataSeedStartupAction = true)
     {
         // Register System.Text.Json options with Exceptionless defaults (snake_case, null handling)
         services.AddSingleton(_ => new JsonSerializerOptions().ConfigureExceptionlessDefaults());
@@ -81,7 +81,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
 
         services.AddSingleton<DataSeedService>();
         services.AddSingleton<IDataSeed, PredefinedSavedViewsDataSeed>();
-        services.AddStartupAction<DataSeedService>();
+        if (runDataSeedStartupAction)
+            services.AddStartupAction<DataSeedService>();
 
         services.AddStartupAction("Create Sample Data", CreateSampleDataAsync);
 

src/Exceptionless.Core/Jobs/Elastic/MigrationJob.cs

@@ -1,4 +1,5 @@
 using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Seed;
 using Foundatio.Jobs;
 using Foundatio.Repositories.Elasticsearch.Configuration;
 using Foundatio.Repositories.Migrations;
@@ -12,23 +13,27 @@ public class MigrationJob : JobBase
 {
     private readonly MigrationManager _migrationManager;
     private readonly ExceptionlessElasticConfiguration _configuration;
+    private readonly DataSeedService _dataSeedService;
 
     public MigrationJob(
         MigrationManager migrationManager,
         ExceptionlessElasticConfiguration configuration,
+        DataSeedService dataSeedService,
         TimeProvider timeProvider,
         IResiliencePolicyProvider resiliencePolicyProvider,
         ILoggerFactory loggerFactory
     ) : base(timeProvider, resiliencePolicyProvider, loggerFactory)
     {
         _migrationManager = migrationManager;
         _configuration = configuration;
+        _dataSeedService = dataSeedService;
     }
 
     protected override async Task<JobResult> RunInternalAsync(JobContext context)
     {
         await _configuration.ConfigureIndexesAsync(null, false);
         await _migrationManager.RunMigrationsAsync();
+        await _dataSeedService.SeedAsync(context.CancellationToken);
 
         var tasks = _configuration.Indexes.OfType<VersionedIndex>().Select(ReindexIfNecessary);
         await Task.WhenAll(tasks);

src/Exceptionless.Core/Migrations/004_MigrateSavedViewColumns.cs

@@ -0,0 +1,266 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.Core.Search;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Seed;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Migrations;
+using Foundatio.Serializer;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Migrations;
+
+public sealed class MigrateSavedViewColumns : MigrationBase
+{
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        PropertyNameCaseInsensitive = true
+    };
+
+    private readonly ElasticsearchClient _client;
+    private readonly ExceptionlessElasticConfiguration _configuration;
+    private readonly ITextSerializer _serializer;
+
+    public MigrateSavedViewColumns(
+        ExceptionlessElasticConfiguration configuration,
+        ITextSerializer serializer,
+        ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _client = configuration.Client;
+        _configuration = configuration;
+        _serializer = serializer;
+
+        MigrationType = MigrationType.VersionedAndResumable;
+        Version = 4;
+    }
+
+    public override async Task RunAsync(MigrationContext context)
+    {
+        const int pageSize = 500;
+        var pointInTime = await _client.OpenPointInTimeAsync(
+            _configuration.SavedViews.VersionedName,
+            request => request.KeepAlive("2m"),
+            context.CancellationToken);
+        _logger.LogRequest(pointInTime);
+
+        if (!pointInTime.IsValidResponse || String.IsNullOrWhiteSpace(pointInTime.Id))
+            throw new InvalidOperationException("Unable to open a point in time for the saved view migration.");
+
+        string pointInTimeId = pointInTime.Id;
+        FieldValue[]? searchAfter = null;
+        int migrated = 0;
+
+        try
+        {
+            do
+            {
+                var response = await _client.SearchAsync<JsonElement>(request =>
+                {
+                    request
+                        .Pit(pit => pit.Id(pointInTimeId).KeepAlive("2m"))
+                        .SeqNoPrimaryTerm()
+                        .Size(pageSize)
+                        .Sort(sort => sort.Field("_shard_doc"));
+
+                    if (searchAfter is not null)
+                        request.SearchAfter(searchAfter);
+                }, context.CancellationToken);
+                _logger.LogRequest(response);
+
+                if (!response.IsValidResponse)
+                    throw new InvalidOperationException("Unable to read saved views during the column migration.");
+
+                if (!String.IsNullOrWhiteSpace(response.PitId))
+                    pointInTimeId = response.PitId;
+
+                foreach (var hit in response.Hits)
+                {
+                    if (hit.Source.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined)
+                        continue;
+
+                    var source = JsonNode.Parse(hit.Source.GetRawText())?.AsObject();
+                    if (source is null)
+                        continue;
+
+                    if (hit.SeqNo is null || hit.PrimaryTerm is null)
+                        throw new InvalidOperationException($"Unable to read concurrency metadata for saved view '{hit.Id}'.");
+
+                    string? legacyContentHash = GetLegacyContentHash(source);
+                    bool columnsMigrated = TryMigrate(source);
+                    bool contentHashMigrated = UpdatePredefinedContentHash(source, legacyContentHash);
+                    if (!columnsMigrated && !contentHashMigrated)
+                        continue;
+
+                    var indexResponse = await IndexMigratedDocumentAsync(
+                        source,
+                        hit.Id,
+                        hit.SeqNo.Value,
+                        hit.PrimaryTerm.Value,
+                        context.CancellationToken);
+                    _logger.LogRequest(indexResponse);
+
+                    if (!indexResponse.IsValidResponse)
+                    {
+                        string reason = indexResponse.ApiCallDetails.HttpStatusCode == 409
+                            ? " because it changed while the migration was running"
+                            : String.Empty;
+                        throw new InvalidOperationException($"Unable to migrate saved view '{hit.Id}'{reason}. The resumable migration can be run again safely.");
+                    }
+
+                    migrated++;
+                }
+
+                searchAfter = response.Hits.Count == pageSize
+                    ? response.Hits.Last().Sort!.ToArray()
+                    : null;
+
+                await context.Lock.RenewAsync();
+            } while (searchAfter is not null && !context.CancellationToken.IsCancellationRequested);
+        }
+        finally
+        {
+            var closeResponse = await _client.ClosePointInTimeAsync(
+                request => request.Id(pointInTimeId),
+                CancellationToken.None);
+            _logger.LogRequest(closeResponse);
+        }
+
+        _logger.LogInformation("Migrated {SavedViewCount} saved view column configurations", migrated);
+    }
+
+    internal Task<IndexResponse> IndexMigratedDocumentAsync(
+        JsonObject source,
+        string id,
+        long sequenceNumber,
+        long primaryTerm,
+        CancellationToken cancellationToken)
+    {
+        return _client.IndexAsync(
+            source,
+            request => request
+                .Index(_configuration.SavedViews.VersionedName)
+                .Id(id)
+                .IfSeqNo(sequenceNumber)
+                .IfPrimaryTerm(primaryTerm),
+            cancellationToken);
+    }
+
+    internal static bool TryMigrate(JsonObject source)
+    {
+        var columns = source["columns"] as JsonObject;
+        var columnOrder = source["column_order"] as JsonArray;
+        bool hasLegacyColumns = columns?.Any(entry => entry.Value is JsonValue value && value.TryGetValue<bool>(out _)) == true;
+
+        if (!hasLegacyColumns && columnOrder is null)
+            return false;
+
+        var migratedColumns = new JsonObject();
+        if (columns is not null)
+        {
+            foreach (var (columnId, value) in columns)
+            {
+                if (value is JsonValue jsonValue && jsonValue.TryGetValue<bool>(out bool visible))
+                {
+                    migratedColumns[columnId] = new JsonObject
+                    {
+                        ["visible"] = visible
+                    };
+                }
+                else if (value is JsonObject settings)
+                {
+                    migratedColumns[columnId] = settings.DeepClone();
+                }
+            }
+        }
+
+        if (columnOrder is not null)
+        {
+            for (int position = 0; position < columnOrder.Count; position++)
+            {
+                string? columnId = columnOrder[position]?.GetValue<string>();
+                if (String.IsNullOrWhiteSpace(columnId))
+                    continue;
+
+                if (migratedColumns[columnId] is not JsonObject settings)
+                {
+                    settings = new JsonObject();
+                    migratedColumns[columnId] = settings;
+                }
+
+                settings["position"] = position;
+            }
+        }
+
+        source["columns"] = migratedColumns.Count > 0 ? migratedColumns : null;
+        source.Remove("column_order");
+        return true;
+    }
+
+    private bool UpdatePredefinedContentHash(JsonObject source, string? legacyContentHash)
+    {
+        string? predefinedContentHash = source["predefined_content_hash"]?.GetValue<string>();
+        if (String.IsNullOrWhiteSpace(predefinedContentHash))
+            return false;
+
+        if (!String.Equals(predefinedContentHash, legacyContentHash, StringComparison.Ordinal))
+            return false;
+
+        var savedView = _serializer.Deserialize<SavedView>(source.ToJsonString(JsonOptions));
+        if (savedView is null)
+            throw new InvalidOperationException("Unable to deserialize a migrated saved view.");
+
+        source["predefined_content_hash"] = PredefinedSavedViewContentHasher.GetContentHash(savedView);
+        return true;
+    }
+
+    internal static string? GetLegacyContentHash(JsonObject source)
+    {
+        var columns = source["columns"] as JsonObject;
+        if (columns?.Any(entry => entry.Value is not JsonValue value || !value.TryGetValue<bool>(out _)) == true)
+            return null;
+
+        var legacyColumns = columns?.ToDictionary(
+            entry => entry.Key,
+            entry => entry.Value!.GetValue<bool>());
+        var columnOrder = (source["column_order"] as JsonArray)?
+            .Select(value => value?.GetValue<string>())
+            .Where(value => value is not null)
+            .Cast<string>()
+            .ToList();
+
+        var content = new
+        {
+            name = source["name"]?.GetValue<string>(),
+            slug = source["slug"]?.GetValue<string>(),
+            viewType = source["view_type"]?.GetValue<string>(),
+            filter = source["filter"]?.GetValue<string>(),
+            time = source["time"]?.GetValue<string>(),
+            sort = source["sort"]?.GetValue<string>(),
+            filterDefinitions = CanonicalizeFilterDefinitions(source["filter_definitions"]?.GetValue<string>()),
+            Columns = legacyColumns?.OrderBy(column => column.Key, StringComparer.Ordinal),
+            columnOrder,
+            showStats = source["show_stats"]?.GetValue<bool?>(),
+            showChart = source["show_chart"]?.GetValue<bool?>()
+        };
+
+        return JsonSerializer.Serialize(content).ToSHA256();
+    }
+
+    private static string? CanonicalizeFilterDefinitions(string? filterDefinitions)
+    {
+        if (String.IsNullOrWhiteSpace(filterDefinitions))
+            return filterDefinitions;
+
+        try
+        {
+            return JsonNode.Parse(filterDefinitions)?.ToJsonString() ?? filterDefinitions;
+        }
+        catch (JsonException)
+        {
+            return filterDefinitions;
+        }
+    }
+}

src/Exceptionless.Core/Models/SavedView.cs

@@ -45,11 +45,8 @@ public partial record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates
     [MaxLength(MaxFilterDefinitionsLength)]
     public string? FilterDefinitions { get; set; }
 
-    /// <summary>Column visibility state per dashboard table, keyed by column id.</summary>
-    public Dictionary<string, bool>? Columns { get; set; }
-
-    /// <summary>Column display order per dashboard table, excluding utility columns.</summary>
-    public List<string>? ColumnOrder { get; set; }
+    /// <summary>Extensible display settings keyed by dashboard column id.</summary>
+    public Dictionary<string, SavedViewColumnSettings>? Columns { get; set; }
 
     /// <summary>Whether dashboard statistic cards are shown for this view. Null means use the default.</summary>
     public bool? ShowStats { get; set; }

src/Exceptionless.Core/Models/SavedViewColumnSettings.cs

@@ -0,0 +1,22 @@
+namespace Exceptionless.Core.Models;
+
+/// <summary>
+/// Per-column display settings for a saved view. All properties are optional so new settings can
+/// be added without changing the meaning of existing saved views.
+/// </summary>
+public sealed record SavedViewColumnSettings
+{
+    public const int MaxPosition = 49;
+    public const int MaxWidth = 1200;
+    public const int MinWidth = 48;
+
+    /// <summary>Whether the column is visible. Null means use the table default.</summary>
+    public bool? Visible { get; set; }
+
+    /// <summary>Zero-based display position. Null means use the table default order.</summary>
+    public int? Position { get; set; }
+
+    /// <summary>Column width in pixels. Null means use the table default width.</summary>
+    public int? Width { get; set; }
+
+}

src/Exceptionless.Core/Seed/DataSeedService.cs

@@ -1,22 +1,36 @@
 using Foundatio.Extensions.Hosting.Startup;
+using Foundatio.Repositories.Migrations;
 using Microsoft.Extensions.Logging;
 
 namespace Exceptionless.Core.Seed;
 
 public class DataSeedService : IStartupAction
 {
     private readonly IEnumerable<IDataSeed> _seeds;
+    private readonly MigrationManager _migrationManager;
     private readonly ILogger _logger;
 
-    public DataSeedService(IEnumerable<IDataSeed> seeds, ILoggerFactory loggerFactory)
+    public DataSeedService(IEnumerable<IDataSeed> seeds, MigrationManager migrationManager, ILoggerFactory loggerFactory)
     {
         _seeds = seeds;
+        _migrationManager = migrationManager;
         _logger = loggerFactory.CreateLogger<DataSeedService>();
     }
 
-    public Task RunAsync(CancellationToken shutdownToken = default)
+    public async Task RunAsync(CancellationToken shutdownToken = default)
     {
-        return SeedAsync(shutdownToken);
+        if (_migrationManager.Migrations.Count == 0)
+            _migrationManager.AddMigrationsFromLoadedAssemblies();
+
+        var migrationStatus = await _migrationManager.GetMigrationStatus();
+        int pendingVersionedMigrationCount = migrationStatus.PendingMigrations.Count(migration => migration.Migration.MigrationType != MigrationType.Repeatable);
+        if (pendingVersionedMigrationCount > 0)
+        {
+            _logger.LogInformation("Skipping data seeds while {PendingMigrationCount} versioned migration(s) are pending", pendingVersionedMigrationCount);
+            return;
+        }
+
+        await SeedAsync(shutdownToken);
     }
 
     public async Task SeedAsync(CancellationToken cancellationToken = default)