Add resizable saved-view columns

.github/workflows/build.yaml

@@ -558,6 +558,27 @@ jobs:
           sed -i "s/^appVersion:.*$/appVersion: '${VERSION}'/" ./k8s/exceptionless/Chart.yaml
           helm upgrade --set "version=${VERSION}" --reuse-values --values ./k8s/ex-dev-values.yaml ex-dev --namespace $DEV_NAMESPACE ./k8s/exceptionless
 
+      - name: Run Development Migrations
+        if: ${{ needs.version.outputs.is_dev_deploy == 'true' }}
+        run: |
+          migration_job="ex-dev-jobs-migration-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          kubectl create job --from=cronjob/ex-dev-jobs-migration "$migration_job" --namespace "$DEV_NAMESPACE"
+
+          migration_image=$(kubectl get job "$migration_job" --namespace "$DEV_NAMESPACE" \
+            --output jsonpath='{.spec.template.spec.containers[0].image}')
+          if [[ "$migration_image" != *":${VERSION}" ]]; then
+            echo "::error::Migration job uses ${migration_image}; expected version ${VERSION}."
+            exit 1
+          fi
+
+          if ! kubectl wait --for=condition=complete --timeout=1200s "job/${migration_job}" --namespace "$DEV_NAMESPACE"; then
+            echo "::group::Migration job diagnostics"
+            kubectl describe job "$migration_job" --namespace "$DEV_NAMESPACE"
+            kubectl get pods --selector "job-name=${migration_job}" --namespace "$DEV_NAMESPACE" --output wide
+            echo "::endgroup::"
+            exit 1
+          fi
+
       - name: Ensure Development Workloads are Running
         if: ${{ needs.version.outputs.is_dev_deploy == 'true' }}
         run: |
@@ -578,7 +599,6 @@ jobs:
           kubectl patch cronjob/ex-dev-jobs-cleanup-orphaned-data -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
           kubectl patch cronjob/ex-dev-jobs-download-geoip-database -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
           kubectl patch cronjob/ex-dev-jobs-maintain-indexes -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
-          kubectl patch cronjob/ex-dev-jobs-migration -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
 
           wait_for_versioned_pod() {
             component="$1"

src/Exceptionless.Core/Bootstrapper.cs

@@ -50,12 +50,13 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using MaintainIndexesJob = Foundatio.Repositories.Elasticsearch.Jobs.MaintainIndexesJob;
+using MigrationJob = Exceptionless.Core.Jobs.Elastic.MigrationJob;
 
 namespace Exceptionless.Core;
 
 public class Bootstrapper
 {
-    public static void RegisterServices(IServiceCollection services, AppOptions appOptions)
+    public static void RegisterServices(IServiceCollection services, AppOptions appOptions, bool runDataSeedStartupAction = true)
     {
         // Register System.Text.Json options with Exceptionless defaults (snake_case, null handling)
         services.AddSingleton(_ => new JsonSerializerOptions().ConfigureExceptionlessDefaults());
@@ -81,7 +82,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
 
         services.AddSingleton<DataSeedService>();
         services.AddSingleton<IDataSeed, PredefinedSavedViewsDataSeed>();
-        services.AddStartupAction<DataSeedService>();
+        if (runDataSeedStartupAction)
+            services.AddStartupAction<DataSeedService>();
 
         services.AddStartupAction("Create Sample Data", CreateSampleDataAsync);
 
@@ -483,6 +485,7 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log
         services.AddJob<EventPostsJob>(o => o.WaitForStartupActions());
         services.AddJob<EventUserDescriptionsJob>(o => o.WaitForStartupActions());
         services.AddJob<MailMessageJob>(o => o.WaitForStartupActions());
+        services.AddJob<MigrationJob>(o => o.WaitForStartupActions());
         services.AddJob<StackStatusJob>(o => o.WaitForStartupActions());
         services.AddJob<StackEventCountJob>(o => o.WaitForStartupActions());
         services.AddJob<WebHooksJob>(o => o.WaitForStartupActions());

src/Exceptionless.Core/Jobs/Elastic/MigrationJob.cs

@@ -1,4 +1,5 @@
 using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Seed;
 using Foundatio.Jobs;
 using Foundatio.Repositories.Elasticsearch.Configuration;
 using Foundatio.Repositories.Migrations;
@@ -12,23 +13,27 @@ public class MigrationJob : JobBase
 {
     private readonly MigrationManager _migrationManager;
     private readonly ExceptionlessElasticConfiguration _configuration;
+    private readonly DataSeedService _dataSeedService;
 
     public MigrationJob(
         MigrationManager migrationManager,
         ExceptionlessElasticConfiguration configuration,
+        DataSeedService dataSeedService,
         TimeProvider timeProvider,
         IResiliencePolicyProvider resiliencePolicyProvider,
         ILoggerFactory loggerFactory
     ) : base(timeProvider, resiliencePolicyProvider, loggerFactory)
     {
         _migrationManager = migrationManager;
         _configuration = configuration;
+        _dataSeedService = dataSeedService;
     }
 
     protected override async Task<JobResult> RunInternalAsync(JobContext context)
     {
         await _configuration.ConfigureIndexesAsync(null, false);
         await _migrationManager.RunMigrationsAsync();
+        await _dataSeedService.SeedAsync(context.CancellationToken);
 
         var tasks = _configuration.Indexes.OfType<VersionedIndex>().Select(ReindexIfNecessary);
         await Task.WhenAll(tasks);

src/Exceptionless.Core/Migrations/005_MigrateSavedViewColumns.cs

@@ -0,0 +1,280 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.Core.Search;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Seed;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Migrations;
+using Foundatio.Serializer;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Migrations;
+
+public sealed class MigrateSavedViewColumns : MigrationBase
+{
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        PropertyNameCaseInsensitive = true
+    };
+
+    private readonly ElasticsearchClient _client;
+    private readonly ExceptionlessElasticConfiguration _configuration;
+    private readonly ITextSerializer _serializer;
+
+    public MigrateSavedViewColumns(
+        ExceptionlessElasticConfiguration configuration,
+        ITextSerializer serializer,
+        ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _client = configuration.Client;
+        _configuration = configuration;
+        _serializer = serializer;
+
+        MigrationType = MigrationType.VersionedAndResumable;
+        Version = 5;
+    }
+
+    public override async Task RunAsync(MigrationContext context)
+    {
+        const int pageSize = 500;
+        const string pointInTimeKeepAlive = "30m";
+        var pointInTime = await _client.OpenPointInTimeAsync(
+            _configuration.SavedViews.VersionedName,
+            request => request.KeepAlive(pointInTimeKeepAlive),
+            context.CancellationToken);
+        _logger.LogRequest(pointInTime);
+
+        if (!pointInTime.IsValidResponse || String.IsNullOrWhiteSpace(pointInTime.Id))
+            throw new InvalidOperationException("Unable to open a point in time for the saved view migration.");
+
+        string pointInTimeId = pointInTime.Id;
+        FieldValue[]? searchAfter = null;
+        int migrated = 0;
+
+        try
+        {
+            do
+            {
+                context.CancellationToken.ThrowIfCancellationRequested();
+
+                var response = await _client.SearchAsync<JsonElement>(request =>
+                {
+                    request
+                        .Pit(pit => pit.Id(pointInTimeId).KeepAlive(pointInTimeKeepAlive))
+                        .SeqNoPrimaryTerm()
+                        .Size(pageSize)
+                        .Sort(sort => sort.Field("_shard_doc"));
+
+                    if (searchAfter is not null)
+                        request.SearchAfter(searchAfter);
+                }, context.CancellationToken);
+                _logger.LogRequest(response);
+
+                if (!response.IsValidResponse)
+                    throw new InvalidOperationException("Unable to read saved views during the column migration.");
+
+                if (!String.IsNullOrWhiteSpace(response.PitId))
+                    pointInTimeId = response.PitId;
+
+                foreach (var hit in response.Hits)
+                {
+                    if (hit.Source.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined)
+                        continue;
+
+                    var source = JsonNode.Parse(hit.Source.GetRawText())?.AsObject();
+                    if (source is null)
+                        continue;
+
+                    if (hit.SeqNo is null || hit.PrimaryTerm is null)
+                        throw new InvalidOperationException($"Unable to read concurrency metadata for saved view '{hit.Id}'.");
+
+                    string? legacyContentHash = GetLegacyContentHash(source);
+                    bool columnsMigrated = TryMigrate(source);
+                    bool contentHashMigrated = UpdatePredefinedContentHash(source, legacyContentHash);
+                    if (!columnsMigrated && !contentHashMigrated)
+                        continue;
+
+                    var indexResponse = await IndexMigratedDocumentAsync(
+                        source,
+                        hit.Id,
+                        hit.SeqNo.Value,
+                        hit.PrimaryTerm.Value,
+                        context.CancellationToken);
+                    _logger.LogRequest(indexResponse);
+
+                    if (!indexResponse.IsValidResponse)
+                    {
+                        string reason = indexResponse.ApiCallDetails.HttpStatusCode == 409
+                            ? " because it changed while the migration was running"
+                            : String.Empty;
+                        throw new InvalidOperationException($"Unable to migrate saved view '{hit.Id}'{reason}. The resumable migration can be run again safely.");
+                    }
+
+                    migrated++;
+                }
+
+                searchAfter = response.Hits.Count == pageSize
+                    ? response.Hits.Last().Sort!.ToArray()
+                    : null;
+
+                await context.Lock.RenewAsync();
+            } while (searchAfter is not null);
+
+            if (migrated > 0)
+            {
+                var refreshResponse = await _client.Indices.RefreshAsync(
+                    _configuration.SavedViews.VersionedName,
+                    context.CancellationToken);
+                _logger.LogRequest(refreshResponse);
+
+                if (!refreshResponse.IsValidResponse)
+                    throw new InvalidOperationException("Unable to refresh saved views after the column migration.");
+            }
+        }
+        finally
+        {
+            var closeResponse = await _client.ClosePointInTimeAsync(
+                request => request.Id(pointInTimeId),
+                CancellationToken.None);
+            _logger.LogRequest(closeResponse);
+        }
+
+        _logger.LogInformation("Migrated {SavedViewCount} saved view column configurations", migrated);
+    }
+
+    internal Task<IndexResponse> IndexMigratedDocumentAsync(
+        JsonObject source,
+        string id,
+        long sequenceNumber,
+        long primaryTerm,
+        CancellationToken cancellationToken)
+    {
+        return _client.IndexAsync(
+            source,
+            request => request
+                .Index(_configuration.SavedViews.VersionedName)
+                .Id(id)
+                .IfSeqNo(sequenceNumber)
+                .IfPrimaryTerm(primaryTerm),
+            cancellationToken);
+    }
+
+    internal static bool TryMigrate(JsonObject source)
+    {
+        var columns = source["columns"] as JsonObject;
+        var columnOrder = source["column_order"] as JsonArray;
+        bool hasLegacyColumns = columns?.Any(entry => entry.Value is JsonValue value && value.TryGetValue<bool>(out _)) == true;
+
+        if (!hasLegacyColumns && columnOrder is null)
+            return false;
+
+        var migratedColumns = new JsonObject();
+        if (columns is not null)
+        {
+            foreach (var (columnId, value) in columns)
+            {
+                if (value is JsonValue jsonValue && jsonValue.TryGetValue<bool>(out bool visible))
+                {
+                    migratedColumns[columnId] = new JsonObject
+                    {
+                        ["visible"] = visible
+                    };
+                }
+                else if (value is JsonObject settings)
+                {
+                    migratedColumns[columnId] = settings.DeepClone();
+                }
+            }
+        }
+
+        if (columnOrder is not null)
+        {
+            for (int position = 0; position < columnOrder.Count; position++)
+            {
+                string? columnId = columnOrder[position]?.GetValue<string>();
+                if (String.IsNullOrWhiteSpace(columnId))
+                    continue;
+
+                if (migratedColumns[columnId] is not JsonObject settings)
+                {
+                    settings = new JsonObject();
+                    migratedColumns[columnId] = settings;
+                }
+
+                settings["position"] = position;
+            }
+        }
+
+        source["columns"] = migratedColumns.Count > 0 ? migratedColumns : null;
+        source.Remove("column_order");
+        return true;
+    }
+
+    private bool UpdatePredefinedContentHash(JsonObject source, string? legacyContentHash)
+    {
+        string? predefinedContentHash = source["predefined_content_hash"]?.GetValue<string>();
+        if (String.IsNullOrWhiteSpace(predefinedContentHash))
+            return false;
+
+        if (!String.Equals(predefinedContentHash, legacyContentHash, StringComparison.Ordinal))
+            return false;
+
+        var savedView = _serializer.Deserialize<SavedView>(source.ToJsonString(JsonOptions));
+        if (savedView is null)
+            throw new InvalidOperationException("Unable to deserialize a migrated saved view.");
+
+        source["predefined_content_hash"] = PredefinedSavedViewContentHasher.GetContentHash(savedView);
+        return true;
+    }
+
+    internal static string? GetLegacyContentHash(JsonObject source)
+    {
+        var columns = source["columns"] as JsonObject;
+        if (columns?.Any(entry => entry.Value is not JsonValue value || !value.TryGetValue<bool>(out _)) == true)
+            return null;
+
+        var legacyColumns = columns?.ToDictionary(
+            entry => entry.Key,
+            entry => entry.Value!.GetValue<bool>());
+        var columnOrder = (source["column_order"] as JsonArray)?
+            .Select(value => value?.GetValue<string>())
+            .Where(value => value is not null)
+            .Cast<string>()
+            .ToList();
+
+        var content = new
+        {
+            name = source["name"]?.GetValue<string>(),
+            slug = source["slug"]?.GetValue<string>(),
+            viewType = source["view_type"]?.GetValue<string>(),
+            filter = source["filter"]?.GetValue<string>(),
+            time = source["time"]?.GetValue<string>(),
+            sort = source["sort"]?.GetValue<string>(),
+            filterDefinitions = CanonicalizeFilterDefinitions(source["filter_definitions"]?.GetValue<string>()),
+            Columns = legacyColumns?.OrderBy(column => column.Key, StringComparer.Ordinal),
+            columnOrder,
+            showStats = source["show_stats"]?.GetValue<bool?>(),
+            showChart = source["show_chart"]?.GetValue<bool?>()
+        };
+
+        return JsonSerializer.Serialize(content).ToSHA256();
+    }
+
+    private static string? CanonicalizeFilterDefinitions(string? filterDefinitions)
+    {
+        if (String.IsNullOrWhiteSpace(filterDefinitions))
+            return filterDefinitions;
+
+        try
+        {
+            return JsonNode.Parse(filterDefinitions)?.ToJsonString() ?? filterDefinitions;
+        }
+        catch (JsonException)
+        {
+            return filterDefinitions;
+        }
+    }
+}

src/Exceptionless.Core/Models/SavedView.cs

@@ -45,11 +45,8 @@ public partial record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates
     [MaxLength(MaxFilterDefinitionsLength)]
     public string? FilterDefinitions { get; set; }
 
-    /// <summary>Column visibility state per dashboard table, keyed by column id.</summary>
-    public Dictionary<string, bool>? Columns { get; set; }
-
-    /// <summary>Column display order per dashboard table, excluding utility columns.</summary>
-    public List<string>? ColumnOrder { get; set; }
+    /// <summary>Extensible display settings keyed by dashboard column id.</summary>
+    public Dictionary<string, SavedViewColumnSettings>? Columns { get; set; }
 
     /// <summary>Whether dashboard statistic cards are shown for this view. Null means use the default.</summary>
     public bool? ShowStats { get; set; }