Include ref.parent matches in events by-reference navigation

src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs

@@ -99,6 +99,9 @@ public static void SetEventReference(this PersistentEvent ev, string name, strin
     {
         ArgumentException.ThrowIfNullOrEmpty(name);
 
+        if (!name.IsValidFieldName())
+            throw new ArgumentException("Name must contain between 1 and 25 alphanumeric or '-' characters.", nameof(name));
+
         if (!IsValidIdentifier(id) || String.IsNullOrEmpty(id))
             throw new ArgumentException("Id must contain between 8 and 100 alphanumeric or '-' characters.", nameof(id));
 
@@ -108,7 +111,7 @@ public static void SetEventReference(this PersistentEvent ev, string name, strin
 
     public static string? GetSessionId(this PersistentEvent ev)
     {
-        return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference("session");
+        return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference(Event.KnownReferenceNames.Session);
     }
 
     public static void SetSessionId(this PersistentEvent ev, string sessionId)
@@ -119,7 +122,7 @@ public static void SetSessionId(this PersistentEvent ev, string sessionId)
         if (ev.IsSessionStart())
             ev.ReferenceId = sessionId;
         else
-            ev.SetEventReference("session", sessionId);
+            ev.SetEventReference(Event.KnownReferenceNames.Session, sessionId);
     }
 
     public static bool HasSessionEndTime(this PersistentEvent ev)

src/Exceptionless.Core/Extensions/StringExtensions.cs

@@ -131,7 +131,7 @@ public static bool IsNumeric(this string? value)
 
     public static bool IsValidFieldName(this string? value)
     {
-        if (value is null || value.Length > 25)
+        if (String.IsNullOrEmpty(value) || value.Length > 25)
             return false;
 
         return IsValidIdentifier(value);

src/Exceptionless.Core/Migrations/003_BackfillParentReferences.cs

@@ -0,0 +1,66 @@
+using System.Diagnostics;
+using Elastic.Clients.Elasticsearch;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Migrations;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Migrations;
+
+public sealed class BackfillParentReferences : MigrationBase
+{
+    private readonly ElasticsearchClient _client;
+    private readonly ExceptionlessElasticConfiguration _config;
+    private readonly TimeProvider _timeProvider;
+
+    public BackfillParentReferences(ExceptionlessElasticConfiguration configuration, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _config = configuration;
+        _client = configuration.Client;
+        _timeProvider = timeProvider;
+
+        MigrationType = MigrationType.VersionedAndResumable;
+        Version = 3;
+    }
+
+    public override async Task RunAsync(MigrationContext context)
+    {
+        string referenceKey = $"@ref:{Event.KnownReferenceNames.Parent}";
+        string indexKey = $"{Event.KnownReferenceNames.Parent}-r";
+        string script = $"if (ctx._source.data != null && ctx._source.data.containsKey('{referenceKey}') && ctx._source.data['{referenceKey}'] != null) {{ if (ctx._source.idx == null) ctx._source.idx = [:]; ctx._source.idx['{indexKey}'] = ctx._source.data['{referenceKey}']; }} else {{ ctx.op = 'noop'; }}";
+
+        _logger.LogInformation("Backfilling retained event parent references");
+        var stopwatch = Stopwatch.StartNew();
+        var response = await _client.UpdateByQueryAsync<PersistentEvent>(request => request
+            .Indices($"{_config.Events.VersionedName}-*")
+            .Query(query => query.Bool(filter => filter.MustNot(mustNot => mustNot.Exists(exists => exists.Field($"idx.{indexKey}")))))
+            .Script(value => value.Source(script).Lang(ScriptLanguage.Painless))
+            .Conflicts(Conflicts.Proceed)
+            .WaitForCompletion(false));
+        _logger.LogRequest(response, LogLevel.Information);
+
+        if (!response.IsValidResponse || response.Task is null)
+            throw new ApplicationException($"Unable to start parent-reference backfill: {response.DebugInformation}");
+
+        int attempts = 0;
+        while (!context.CancellationToken.IsCancellationRequested)
+        {
+            var taskStatus = await _client.Tasks.GetAsync(response.Task.FullyQualifiedId, context.CancellationToken);
+            if (!taskStatus.IsValidResponse)
+                throw new ApplicationException($"Unable to monitor parent-reference backfill: {taskStatus.DebugInformation}");
+
+            if (taskStatus.Completed)
+            {
+                _logger.LogInformation("Finished parent-reference backfill: Duration={Duration}", stopwatch.Elapsed);
+                return;
+            }
+
+            attempts++;
+            await context.Lock.RenewAsync();
+            await Task.Delay(TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5), _timeProvider, context.CancellationToken);
+        }
+
+        context.CancellationToken.ThrowIfCancellationRequested();
+    }
+}

src/Exceptionless.Core/Models/Event.cs

@@ -159,6 +159,12 @@ public static class KnownTags
         public const string Internal = "Internal";
     }
 
+    public static class KnownReferenceNames
+    {
+        public const string Parent = "parent";
+        public const string Session = "session";
+    }
+
     public static class KnownDataKeys
     {
         public const string Error = "@error";

src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs

@@ -1,4 +1,5 @@
-using Exceptionless.Core.Plugins.EventProcessor;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Plugins.EventProcessor;
 using Microsoft.Extensions.Logging;
 
 namespace Exceptionless.Core.Pipeline;
@@ -11,7 +12,12 @@ public CopySimpleDataToIdxAction(AppOptions options, ILoggerFactory loggerFactor
     public override Task ProcessAsync(EventContext ctx)
     {
         if (!ctx.Organization.HasPremiumFeatures)
+        {
+            if (ctx.Event.GetEventReference(Event.KnownReferenceNames.Parent) is not null)
+                ctx.Event.CopyDataToIndex([$"@ref:{Event.KnownReferenceNames.Parent}"]);
+
             return Task.CompletedTask;
+        }
 
         // TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients.
         ctx.Event.CopyDataToIndex([]);

src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs

@@ -1,4 +1,5 @@
-using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
 using Foundatio.Parsers.LuceneQueries;
 using Foundatio.Parsers.LuceneQueries.Visitors;
 using Microsoft.Extensions.Logging;
@@ -11,6 +12,7 @@ public sealed class PersistentEventQueryValidator : AppQueryValidator
             "date",
             "type",
             EventIndex.Alias.ReferenceId,
+            $"idx.{Event.KnownReferenceNames.Parent}-r",
             "reference_id",
             EventIndex.Alias.OrganizationId,
             "organization_id",
@@ -93,9 +95,11 @@ public PersistentEventQueryValidator(ExceptionlessElasticConfiguration configura
 
     protected override QueryProcessResult ApplyQueryRules(QueryValidationResult result)
     {
+        bool hasInvalidReferenceField = result.ReferencedFields.Any(field => field.StartsWith("ref.", StringComparison.OrdinalIgnoreCase));
         return new QueryProcessResult
         {
-            IsValid = result.IsValid,
+            IsValid = result.IsValid && !hasInvalidReferenceField,
+            Message = hasInvalidReferenceField ? "Invalid reference field name" : null,
             UsesPremiumFeatures = !result.ReferencedFields.All(_freeQueryFields.Contains)
         };
     }

src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs

@@ -73,7 +73,7 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context)
             return null;
 
         string[] parts = field.Split('.');
-        if (parts.Length != 2 || (parts.Length == 2 && parts[1].StartsWith("@")))
+        if (parts.Length != 2 || parts[1].StartsWith("@") || !parts[1].IsValidFieldName())
             return field;
 
         if (String.Equals(parts[0], "data", StringComparison.OrdinalIgnoreCase))

src/Exceptionless.Web/Api/Handlers/EventHandler.cs

@@ -16,19 +16,19 @@
 using Exceptionless.Core.Validation;
 using Exceptionless.DateTimeExtensions;
 using Exceptionless.Web.Api.Infrastructure;
-using Exceptionless.Web.Api.Results;
 using Exceptionless.Web.Api.Messages;
+using Exceptionless.Web.Api.Results;
 using Exceptionless.Web.Extensions;
 using Exceptionless.Web.Models;
 using Exceptionless.Web.Utility;
 using Foundatio.Caching;
 using Foundatio.Mediator;
 using Foundatio.Queues;
-using Foundatio.Serializer;
 using Foundatio.Repositories;
 using Foundatio.Repositories.Elasticsearch.Extensions;
 using Foundatio.Repositories.Extensions;
 using Foundatio.Repositories.Models;
+using Foundatio.Serializer;
 using Microsoft.Net.Http.Headers;
 
 namespace Exceptionless.Web.Api.Handlers;
@@ -230,7 +230,7 @@ public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceId mes
 
         var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider));
         var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true };
-        return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
+        return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
     }
 
     public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceIdAndProject message)
@@ -249,7 +249,7 @@ public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceIdAndP
 
         var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider));
         var sf = new AppFilter(project, organization);
-        return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
+        return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
     }
 
     public async Task<Result<PagedResult<object>>> Handle(GetEventsBySessionId message)

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts

@@ -85,6 +85,16 @@ describe('TagFilter', () => {
     });
 });
 
+describe('ReferenceFilter', () => {
+    it('matches direct and parent references', () => {
+        expect(new ReferenceFilter('ref-123').toFilter()).toBe('(reference:"ref-123" OR ref.parent:"ref-123")');
+    });
+
+    it('quotes the reference in both clauses', () => {
+        expect(new ReferenceFilter('ref 123').toFilter()).toBe('(reference:"ref 123" OR ref.parent:"ref 123")');
+    });
+});
+
 describe('applyTimeFilter', () => {
     it('removes an existing date filter when time is explicitly empty', () => {
         // Arrange

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts

@@ -245,7 +245,8 @@ export class ReferenceFilter implements IFilter {
             return '';
         }
 
-        return `reference:${quoteIfSpecialCharacters(this.value)}`;
+        const reference = quoteIfSpecialCharacters(this.value);
+        return `(reference:${reference} OR ref.parent:${reference})`;
     }
 }
 

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte

@@ -52,6 +52,10 @@
         return refs;
     });
 
+    function isValidReferenceName(name: string): boolean {
+        return /^[\p{L}\p{Nd}-]{1,25}$/u.test(name);
+    }
+
     let level = $derived(event.data?.['@level']?.toLowerCase());
     let location = $derived(getLocation(event));
 
@@ -95,9 +99,17 @@
                 {#if reference.name === 'session'}
                     <Table.Head class="w-40 font-semibold whitespace-nowrap">Session</Table.Head>
                     <Table.Cell class="w-4 pr-0"><EventsFacetedFilter.SessionTrigger changed={filterChanged} value={reference.id} /></Table.Cell>
-                {:else}
+                {:else if reference.name === 'parent'}
                     <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
                     <Table.Cell class="w-4 pr-0"><EventsFacetedFilter.ReferenceTrigger changed={filterChanged} value={reference.id} /></Table.Cell>
+                {:else if isValidReferenceName(reference.name)}
+                    <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
+                    <Table.Cell class="w-4 pr-0"
+                        ><EventsFacetedFilter.StringTrigger changed={filterChanged} term={`ref.${reference.name}`} value={reference.id} /></Table.Cell
+                    >
+                {:else}
+                    <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
+                    <Table.Cell class="w-4 pr-0"></Table.Cell>
                 {/if}
                 <Table.Cell><A href={resolve('/(app)/event/by-ref/[referenceId]', { referenceId: reference.id })}>{reference.id}</A></Table.Cell>
             </Table.Row>

src/Exceptionless.Web/ClientApp/src/routes/(app)/event/by-ref/[referenceId]/+page.svelte

@@ -6,6 +6,7 @@
     import { Button } from '$comp/ui/button';
     import { Spinner } from '$comp/ui/spinner';
     import { getEventsByReferenceQuery } from '$features/events/api.svelte';
+    import { ReferenceFilter } from '$features/events/components/filters';
     import Summary from '$features/events/components/summary/summary.svelte';
 
     const referenceId = $derived(page.params.referenceId || '');
@@ -16,7 +17,7 @@
             }
         }
     });
-    const eventListHref = $derived(`${resolve('/(app)/event')}?filter=${encodeURIComponent(`reference:${referenceId}`)}&limit=20`);
+    const eventListHref = $derived(`${resolve('/(app)/event')}?filter=${encodeURIComponent(new ReferenceFilter(referenceId).toFilter())}&limit=20`);
     let redirectedEventId = $state<string>();
 
     $effect(() => {

tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs

@@ -197,6 +197,33 @@ public async Task GetByReferenceIdAsync_WithExistingReference_ReturnsMatchingEve
         Assert.Equal(referenceId, ev.ReferenceId);
     }
 
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task GetByReferenceIdAsync_WithOnlyParentReference_ReturnsMatchingEvents(bool projectScoped)
+    {
+        // Arrange
+        string referenceId = Guid.NewGuid().ToString("N");
+        await CreateDataAsync(d => d.Event().FreeProject().Reference("parent", referenceId).Message("parent reference route"));
+        await RefreshDataAsync();
+
+        string[] paths = projectScoped
+            ? ["projects", SampleDataService.FREE_PROJECT_ID, "events", "by-ref", referenceId]
+            : ["events", "by-ref", referenceId];
+
+        // Act
+        var events = await SendRequestAsAsync<IReadOnlyCollection<PersistentEvent>>(r => r
+            .AsFreeOrganizationUser()
+            .AppendPaths(paths)
+            .StatusCodeShouldBeOk()
+        );
+
+        // Assert
+        Assert.NotNull(events);
+        var ev = Assert.Single(events);
+        Assert.Equal(referenceId, ev.GetEventReference("parent"));
+    }
+
     [Fact]
     public async Task GetCountByOrganizationAsync_WithExistingEvents_ReturnsOrganizationCount()
     {

tests/Exceptionless.Tests/Migrations/BackfillParentReferencesMigrationTests.cs

@@ -0,0 +1,50 @@
+using Exceptionless.Core.Migrations;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Tests.Utility;
+using Foundatio.Lock;
+using Foundatio.Repositories;
+using Foundatio.Repositories.Migrations;
+using Foundatio.Utility;
+using Xunit;
+
+namespace Exceptionless.Tests.Migrations;
+
+public sealed class BackfillParentReferencesMigrationTests : IntegrationTestsBase
+{
+    private readonly EventData _eventData;
+    private readonly IEventRepository _eventRepository;
+
+    public BackfillParentReferencesMigrationTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory)
+    {
+        _eventData = GetService<EventData>();
+        _eventRepository = GetService<IEventRepository>();
+    }
+
+    protected override void RegisterServices(IServiceCollection services)
+    {
+        services.AddTransient<BackfillParentReferences>();
+        services.AddSingleton<ILock>(EmptyLock.Empty);
+        base.RegisterServices(services);
+    }
+
+    [Fact]
+    public async Task WillBackfillRetainedParentReference()
+    {
+        var ev = _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId, stackId: TestConstants.StackId, generateData: false, occurrenceDate: TimeProvider.GetUtcNow());
+        ev.Data = new() { [$"@ref:{Event.KnownReferenceNames.Parent}"] = "parent-reference" };
+        ev.Idx = null;
+        await _eventRepository.AddAsync(ev, options => options.ImmediateConsistency());
+
+        var before = await _eventRepository.FindAsync(query => query.FieldEquals("idx.parent-r", "parent-reference"));
+        Assert.Empty(before.Documents);
+
+        var migration = GetService<BackfillParentReferences>();
+        var context = new MigrationContext(GetService<ILock>(), _logger, TestCancellationToken);
+        await migration.RunAsync(context);
+        await RefreshDataAsync();
+
+        var after = await _eventRepository.FindAsync(query => query.FieldEquals("idx.parent-r", "parent-reference"));
+        Assert.Equal(ev.Id, Assert.Single(after.Documents).Id);
+    }
+}