From 6f93f3c23f3d5955b8c1c70b0dfae30ea51c56bb Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 27 Jul 2026 23:46:20 -0500 Subject: [PATCH 1/6] Reduce OpenTelemetry ingestion volume --- k8s/signoz.yaml | 31 ++++++++++++++++++++++++++ src/Exceptionless.Web/ApmExtensions.cs | 6 +++++ 2 files changed, 37 insertions(+) diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index f07e6e75d7..a3a126469e 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -33,9 +33,40 @@ otelAgent: endpoint: http://apm.elastic-system.svc:8200 tls: insecure: true + processors: + filter/drop_unused_metrics: + error_mode: ignore + metrics: + metric: + - 'name == "http.client.request.duration"' + - 'name == "http.client.request.time_in_queue"' + - 'name == "http.client.connection.duration"' + - 'name == "http.client.open_connections"' + - 'name == "http.client.active_requests"' + - 'name == "dns.lookup.duration"' + - 'name == "kestrel.connection.duration"' + - 'name == "aspnetcore.authentication.authenticate.duration"' + - 'name == "aspnetcore.routing.match_attempts"' + - 'name == "dotnet.exceptions"' + - 'name == "dotnet.gc.collections"' + - 'name == "dotnet.gc.last_collection.heap.size"' + - 'name == "dotnet.gc.last_collection.heap.fragmentation.size"' + - 'name == "dotnet.process.cpu.time"' + - 'name == "foundatio.lock.wait.time"' + - 'name == "ex.posts.size"' + - 'name == "ex.events.parse.jsoneventparserplugin"' + - 'name == "ex.events.field.count"' + - 'name == "ex.eventpipeline.markascriticalaction"' + filter/drop_cache_spans: + error_mode: ignore + traces: + span: + - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "job-event-posts")' service: pipelines: traces: + processors: [filter/drop_cache_spans, batch] exporters: [otlp, otlp/elastic] metrics: + processors: [filter/drop_unused_metrics, batch] exporters: [otlp, otlp/elastic] diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 999b329740..65de74bd01 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -128,6 +128,12 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddMeter("System.Runtime"); b.AddRuntimeInstrumentation(); b.AddProcessInstrumentation(); + b.AddView( + "http.server.request.duration", + new ExplicitBucketHistogramConfiguration + { + Boundaries = [0.005, 0.025, 0.1, 0.5, 1, 5] + }); if (config.Console) b.AddConsoleExporter((_, metricReaderOptions) => From 2bc894e46ea7d86805dee8be2228e9f73aa9e2d7 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 20:12:18 -0500 Subject: [PATCH 2/6] Drop EntityChanged messaging spans --- k8s/signoz.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index a3a126469e..0139614c14 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -62,6 +62,7 @@ otelAgent: traces: span: - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "job-event-posts")' + - 'name == "Message: EntityChanged"' service: pipelines: traces: From 443f6ac1cdf1a8f6b115b05c620c16eead09c1b2 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 22:22:19 -0500 Subject: [PATCH 3/6] Upgrade SigNoz k8s-infra chart to 0.16.0 --- k8s/ex-prod-tasks.ps1 | 2 +- k8s/ex-setup.ps1 | 2 +- k8s/signoz.yaml | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/k8s/ex-prod-tasks.ps1 b/k8s/ex-prod-tasks.ps1 index d03b7a8b6e..2df5327194 100644 --- a/k8s/ex-prod-tasks.ps1 +++ b/k8s/ex-prod-tasks.ps1 @@ -80,7 +80,7 @@ helm upgrade vpa fairwinds-stable/vpa --namespace vpa -f vpa-values.yaml --reset # upgrade signoz helm repo update -helm upgrade --reset-values signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" --dry-run +helm upgrade --reset-values signoz-collector signoz/k8s-infra --version 0.16.0 -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" --dry-run # upgrade elasticsearch metrics exporter helm upgrade --reset-values es-exporter prometheus-community/prometheus-elasticsearch-exporter ` diff --git a/k8s/ex-setup.ps1 b/k8s/ex-setup.ps1 index 56b4aecd5c..a745401365 100644 --- a/k8s/ex-setup.ps1 +++ b/k8s/ex-setup.ps1 @@ -154,7 +154,7 @@ kubectl apply -f ex-$ENV-redis.yaml -n ex-$ENV # install signoz otel collector helm repo add signoz https://charts.signoz.io -helm install signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" +helm install signoz-collector signoz/k8s-infra --version 0.16.0 -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" # install elasticsearch metrics exporter for signoz (prod only) helm install es-exporter prometheus-community/prometheus-elasticsearch-exporter ` diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index 0139614c14..bddf5f9d5e 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -2,10 +2,12 @@ global: cloud: aks clusterName: ex-k8s-v6 deploymentEnvironment: prod -otelCollectorEndpoint: ingest.us.signoz.cloud:443 +otelCollectorEndpoint: https://ingest.us.signoz.cloud:443 otelInsecure: false presets: otlpExporter: + enabled: false + otlphttpExporter: enabled: true loggingExporter: enabled: false @@ -67,7 +69,7 @@ otelAgent: pipelines: traces: processors: [filter/drop_cache_spans, batch] - exporters: [otlp, otlp/elastic] + exporters: [otlphttp, otlp/elastic] metrics: processors: [filter/drop_unused_metrics, batch] - exporters: [otlp, otlp/elastic] + exporters: [otlphttp, otlp/elastic] From 12111d8cb87248e22b866177569541e62da57b70 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 22:43:58 -0500 Subject: [PATCH 4/6] Fix event-post Redis span filter --- k8s/signoz.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index bddf5f9d5e..19bb4bcf96 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -63,7 +63,7 @@ otelAgent: error_mode: ignore traces: span: - - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "job-event-posts")' + - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "exceptionless-job-event-posts")' - 'name == "Message: EntityChanged"' service: pipelines: From fb99bd943c7530673dc327deb90587bb3bafc916 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 23:52:12 -0500 Subject: [PATCH 5/6] Drop unused metrics at the SDK source --- k8s/signoz.yaml | 2 +- src/Exceptionless.Web/ApmExtensions.cs | 28 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index 19bb4bcf96..bddf5f9d5e 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -63,7 +63,7 @@ otelAgent: error_mode: ignore traces: span: - - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "exceptionless-job-event-posts")' + - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "job-event-posts")' - 'name == "Message: EntityChanged"' service: pipelines: diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 65de74bd01..c8b7aa813e 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -128,6 +128,11 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddMeter("System.Runtime"); b.AddRuntimeInstrumentation(); b.AddProcessInstrumentation(); + foreach (string metricName in _droppedMetricNames) + { + b.AddView(metricName, MetricStreamConfiguration.Drop); + } + b.AddView( "http.server.request.duration", new ExplicitBucketHistogramConfiguration @@ -171,6 +176,29 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) return builder; } + private static readonly string[] _droppedMetricNames = + [ + "http.client.request.duration", + "http.client.request.time_in_queue", + "http.client.connection.duration", + "http.client.open_connections", + "http.client.active_requests", + "dns.lookup.duration", + "kestrel.connection.duration", + "aspnetcore.authentication.authenticate.duration", + "aspnetcore.routing.match_attempts", + "dotnet.exceptions", + "dotnet.gc.collections", + "dotnet.gc.last_collection.heap.size", + "dotnet.gc.last_collection.heap.fragmentation.size", + "dotnet.process.cpu.time", + "foundatio.lock.wait.time", + "ex.posts.size", + "ex.events.parse.jsoneventparserplugin", + "ex.events.field.count", + "ex.eventpipeline.markascriticalaction" + ]; + private static readonly Regex _stackIdListShortener = StackIdListShortenerRegex(); [GeneratedRegex("(\"stack_id\": \\[)([^\\]]*)\\]", RegexOptions.Compiled)] From 4c5f28fa96df65006327fddae656a552c24c69aa Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 29 Jul 2026 00:09:32 -0500 Subject: [PATCH 6/6] Keep metric filtering in Collector config --- k8s/signoz.yaml | 42 ++++++++++++++------------ src/Exceptionless.Web/ApmExtensions.cs | 27 ----------------- 2 files changed, 22 insertions(+), 47 deletions(-) diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml index bddf5f9d5e..9bf97f5445 100644 --- a/k8s/signoz.yaml +++ b/k8s/signoz.yaml @@ -39,26 +39,28 @@ otelAgent: filter/drop_unused_metrics: error_mode: ignore metrics: - metric: - - 'name == "http.client.request.duration"' - - 'name == "http.client.request.time_in_queue"' - - 'name == "http.client.connection.duration"' - - 'name == "http.client.open_connections"' - - 'name == "http.client.active_requests"' - - 'name == "dns.lookup.duration"' - - 'name == "kestrel.connection.duration"' - - 'name == "aspnetcore.authentication.authenticate.duration"' - - 'name == "aspnetcore.routing.match_attempts"' - - 'name == "dotnet.exceptions"' - - 'name == "dotnet.gc.collections"' - - 'name == "dotnet.gc.last_collection.heap.size"' - - 'name == "dotnet.gc.last_collection.heap.fragmentation.size"' - - 'name == "dotnet.process.cpu.time"' - - 'name == "foundatio.lock.wait.time"' - - 'name == "ex.posts.size"' - - 'name == "ex.events.parse.jsoneventparserplugin"' - - 'name == "ex.events.field.count"' - - 'name == "ex.eventpipeline.markascriticalaction"' + exclude: + match_type: strict + metric_names: + - http.client.request.duration + - http.client.request.time_in_queue + - http.client.connection.duration + - http.client.open_connections + - http.client.active_requests + - dns.lookup.duration + - kestrel.connection.duration + - aspnetcore.authentication.authenticate.duration + - aspnetcore.routing.match_attempts + - dotnet.exceptions + - dotnet.gc.collections + - dotnet.gc.last_collection.heap.size + - dotnet.gc.last_collection.heap.fragmentation.size + - dotnet.process.cpu.time + - foundatio.lock.wait.time + - ex.posts.size + - ex.events.parse.jsoneventparserplugin + - ex.events.field.count + - ex.eventpipeline.markascriticalaction filter/drop_cache_spans: error_mode: ignore traces: diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index c8b7aa813e..0f6af7263f 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -128,10 +128,6 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddMeter("System.Runtime"); b.AddRuntimeInstrumentation(); b.AddProcessInstrumentation(); - foreach (string metricName in _droppedMetricNames) - { - b.AddView(metricName, MetricStreamConfiguration.Drop); - } b.AddView( "http.server.request.duration", @@ -176,29 +172,6 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) return builder; } - private static readonly string[] _droppedMetricNames = - [ - "http.client.request.duration", - "http.client.request.time_in_queue", - "http.client.connection.duration", - "http.client.open_connections", - "http.client.active_requests", - "dns.lookup.duration", - "kestrel.connection.duration", - "aspnetcore.authentication.authenticate.duration", - "aspnetcore.routing.match_attempts", - "dotnet.exceptions", - "dotnet.gc.collections", - "dotnet.gc.last_collection.heap.size", - "dotnet.gc.last_collection.heap.fragmentation.size", - "dotnet.process.cpu.time", - "foundatio.lock.wait.time", - "ex.posts.size", - "ex.events.parse.jsoneventparserplugin", - "ex.events.field.count", - "ex.eventpipeline.markascriticalaction" - ]; - private static readonly Regex _stackIdListShortener = StackIdListShortenerRegex(); [GeneratedRegex("(\"stack_id\": \\[)([^\\]]*)\\]", RegexOptions.Compiled)]