Reduce OpenTelemetry ingestion volume

k8s/ex-prod-tasks.ps1

@@ -80,7 +80,7 @@ helm upgrade vpa fairwinds-stable/vpa --namespace vpa -f vpa-values.yaml --reset
 
 # upgrade signoz
 helm repo update
-helm upgrade --reset-values signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" --dry-run
+helm upgrade --reset-values signoz-collector signoz/k8s-infra --version 0.16.0 -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" --dry-run
 
 # upgrade elasticsearch metrics exporter
 helm upgrade --reset-values es-exporter prometheus-community/prometheus-elasticsearch-exporter `

k8s/ex-setup.ps1

@@ -154,7 +154,7 @@ kubectl apply -f ex-$ENV-redis.yaml -n ex-$ENV
 
 # install signoz otel collector
 helm repo add signoz https://charts.signoz.io
-helm install signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY"
+helm install signoz-collector signoz/k8s-infra --version 0.16.0 -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY"
 
 # install elasticsearch metrics exporter for signoz (prod only)
 helm install es-exporter prometheus-community/prometheus-elasticsearch-exporter `

k8s/signoz.yaml

@@ -2,10 +2,12 @@ global:
   cloud: aks
   clusterName: ex-k8s-v6
   deploymentEnvironment: prod
-otelCollectorEndpoint: ingest.us.signoz.cloud:443
+otelCollectorEndpoint: https://ingest.us.signoz.cloud:443
 otelInsecure: false
 presets:
   otlpExporter:
+    enabled: false
+  otlphttpExporter:
     enabled: true
   loggingExporter:
     enabled: false
@@ -33,9 +35,41 @@ otelAgent:
         endpoint: http://apm.elastic-system.svc:8200
         tls:
           insecure: true
+    processors:
+      filter/drop_unused_metrics:
+        error_mode: ignore
+        metrics:
+          metric:
+            - 'name == "http.client.request.duration"'
+            - 'name == "http.client.request.time_in_queue"'
+            - 'name == "http.client.connection.duration"'
+            - 'name == "http.client.open_connections"'
+            - 'name == "http.client.active_requests"'
+            - 'name == "dns.lookup.duration"'
+            - 'name == "kestrel.connection.duration"'
+            - 'name == "aspnetcore.authentication.authenticate.duration"'
+            - 'name == "aspnetcore.routing.match_attempts"'
+            - 'name == "dotnet.exceptions"'
+            - 'name == "dotnet.gc.collections"'
+            - 'name == "dotnet.gc.last_collection.heap.size"'
+            - 'name == "dotnet.gc.last_collection.heap.fragmentation.size"'
+            - 'name == "dotnet.process.cpu.time"'
+            - 'name == "foundatio.lock.wait.time"'
+            - 'name == "ex.posts.size"'
+            - 'name == "ex.events.parse.jsoneventparserplugin"'
+            - 'name == "ex.events.field.count"'
+            - 'name == "ex.eventpipeline.markascriticalaction"'
+      filter/drop_cache_spans:
+        error_mode: ignore
+        traces:
+          span:
+            - 'attributes["db.system"] == "redis" and status.code != STATUS_CODE_ERROR and (resource.attributes["service.name"] == "exceptionless-web" or resource.attributes["service.name"] == "job-event-posts")'
+            - 'name == "Message: EntityChanged"'
     service:
       pipelines:
         traces:
-          exporters: [otlp, otlp/elastic]
+          processors: [filter/drop_cache_spans, batch]
+          exporters: [otlphttp, otlp/elastic]
         metrics:
-          exporters: [otlp, otlp/elastic]
+          processors: [filter/drop_unused_metrics, batch]
+          exporters: [otlphttp, otlp/elastic]

src/Exceptionless.Web/ApmExtensions.cs

@@ -128,6 +128,12 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config)
                 b.AddMeter("System.Runtime");
                 b.AddRuntimeInstrumentation();
                 b.AddProcessInstrumentation();
+                b.AddView(
+                    "http.server.request.duration",
+                    new ExplicitBucketHistogramConfiguration
+                    {
+                        Boundaries = [0.005, 0.025, 0.1, 0.5, 1, 5]
+                    });
 
                 if (config.Console)
                     b.AddConsoleExporter((_, metricReaderOptions) =>