Instana self monitoring integration package

packages/@instana-integration/instana-self-monitoring/events/acceptor/acceptor_high_job_queue_utilizatio.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SRESLO] Acceptor HTTP job queue is filling up [TF]",
+  "description": "The HTTP threads of this instance do not seem to be able to handle the number of incoming requests causing job queue to fill up. This may eventually result in rejected HTTP requests and Acceptor responding with HTTP status code ENHANCE_YOUR_CALM. Potential causes for this are the respective Acceptor instance receiving too much traffic or problems with the Kafka cluster.",
+  "query": "entity.kubernetes.deployment.name:\"acceptor\"",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "window": 60000,
+      "severity": 10,
+      "metricName": "metrics.gauges.org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs-queue-utilization",
+      "aggregation": "avg",
+      "conditionOperator": ">",
+      "conditionValue": 0.5
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/acceptor/acceptor_high_span_error_rate.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SRESLO] Acceptor spans_error_rate rate is too high [TF]",
+  "description": "This indicates problems with either too high load on the individual Acceptor node or problems with the Kafka cluster in this region and leads to gaps in metrics and calls for customers.",
+  "query": "entity.kubernetes.deployment.name:\"acceptor\"",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "window": 60000,
+      "severity": 10,
+      "metricName": "metrics.gauges.KPI.outgoing.spans.error_rate",
+      "aggregation": "avg",
+      "conditionOperator": ">",
+      "conditionValue": 0.05
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/appdata-processor/appdata_processor_failed_to_heal_online_infra_graph.json

@@ -0,0 +1,18 @@
+{
+  "name": "appdata-processor - Failed to heal online infra graph [TF]",
+  "description": "https://www.notion.so/instana/appdata-processor-Failure-to-heal-online-infra-graph-8daaf2073f0143f790607e78abb4a663",
+  "query": "entity.jvm.app.name:appdata-processor",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.meters.com.instana.graph.GraphStateMaintainer.num-rejected-full-updates",
+      "aggregation": "sum",
+      "window": 3600000,
+      "conditionOperator": ">=",
+      "conditionValue": 2,
+      "severity": 5
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/appdata-processor/appdata_processor_failed_to_heal_online_snapshots.json

@@ -0,0 +1,18 @@
+{
+  "name": "appdata-processor - Failed to heal online snapshots [TF]",
+  "description": "https://www.notion.so/instana/appdata-processor-failed-to-heal-online-snapshots-c36dd6b9f2ec4a26ac4717d92c1911ba",
+  "query": "entity.jvm.app.name:appdata-processor",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.meters.com.instana.snapshot.cache.DefaultOnlineSnapshotCache.self-healing-errors",
+      "aggregation": "sum",
+      "window": 3600000,
+      "conditionOperator": ">=",
+      "conditionValue": 2,
+      "severity": 5
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/appdata-processor/appdata_processor_high_span_dropping.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SRETUSLO] appdata-processor is dropping > 25% spans [TF]",
+  "description": "This can lead to missing traces and results in wrong call counts. Possible reasons are an increase in ingress for the customer or that the component is running on an overloaded worker. If the traffic is reasonable (e.g. not spikes through load tests) consider scaling the spans profile for this TU up or scale out appdata-processor. See also https://www.notion.so/instana/appdata-processor-Spans-are-being-dropped-173977adc97d4e1b9b08bbbc7e56b8ec",
+  "query": "entity.jvm.app.name:appdata-processor",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.gauges.KPI.incoming.span_messages.error_rate",
+      "aggregation": "avg",
+      "window": 600000,
+      "conditionOperator": ">",
+      "conditionValue": 0.25,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/appdata-processor/appdata_processor_service_explosion.json

@@ -0,0 +1,18 @@
+{
+  "name": "appdata-processor - service explosion [TF]",
+  "description": "https://www.notion.so/instana/appdata-processor-Service-explosion-6794827fd9484a1cb557c06c7f7ce969",
+  "query": "entity.jvm.app.name:appdata-processor AND NOT entity.kubernetes.pod.label:\"instana.io/tenantUnit=cision-production\"",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.meters.com.instana.spanprocessing.stream.appdatagraph.ServiceExplosionSafeguard.rejected",
+      "aggregation": "sum",
+      "window": 60000,
+      "conditionOperator": ">",
+      "conditionValue": 100,
+      "severity": 5
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/butler/butler_jobs_queue_utilization.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SRESLO] Butler HTTP thread pool is very busy [TF]",
+  "description": "The HTTP threads of this instance do not seem to be able to handle the number of incoming requests causing job queue to fill up. This may eventually result in OOPS page in web-UI. Potential cause for this is butler produce exceeded db access(queries) to PostgreSQL(butler) in rainbow. You might need to check the PostgreSQL in rainbow.",
+  "query": "entity.kubernetes.deployment.name:\"butler\"",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.gauges.org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs-queue-utilization",
+      "aggregation": "avg",
+      "window": 60000,
+      "conditionOperator": ">",
+      "conditionValue": 0.5,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/butler/butler_very_busy_http_thread_pool.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SRESLO] Butler http thread is very busy [TF]",
+  "description": "The HTTP threads of this instance do not seem to be able to handle the number of incoming requests causing job queue to fill up. This may eventually result in OOPS page in web-UI. Potential cause for this is butler produce exceeded db access(queries) to PostgreSQL(butler) in rainbow. You might need to check the PostgreSQL in rainbow.",
+  "query": "entity.kubernetes.deployment.name:\"butler\"",
+  "entityType": "dropwizardApplicationContainer",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "metrics.gauges.org.eclipse.jetty.util.thread.QueuedThreadPool.dw.utilization-max",
+      "aggregation": "avg",
+      "window": 60000,
+      "conditionOperator": ">",
+      "conditionValue": 0.9,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/cassandra/metrics_cassandra_unreachable_node.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] metrics-cassandra node unreachable for > 5m [TF]",
+  "description": "A node in the metrics-cassandra cluster is not reachable since 5 minutes. This usually indicates overall load problems in the cluster or a issue with the underlying cloud VM. Try restarting the process and investigate overall cluster load afterwards.",
+  "query": "(NOT entity.cassandra.cluster.name:\"*state*\") AND (NOT entity.cassandra.cluster.name:\"*profiles*\") AND (NOT entity.cassandra.cluster.name:\"*spans*\")",
+  "entityType": "cassandraCluster",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "unreachableNodeCount",
+      "aggregation": "min",
+      "window": 300000,
+      "conditionOperator": ">",
+      "conditionValue": 0,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/cassandra/profiles_cassandra_unreachable_node.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] profiles-cassandra node unreachable for > 5m [TF]",
+  "description": "A node in the profiles-cassandra cluster is not reachable since 5 minutes. This usually indicates overall load problems in the cluster or a issue with the underlying cloud VM. Try restarting the process and investigate overall cluster load afterwards.",
+  "query": "entity.cassandra.cluster.name:\"*profiles*\"",
+  "entityType": "cassandraCluster",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "unreachableNodeCount",
+      "aggregation": "min",
+      "window": 300000,
+      "conditionOperator": ">",
+      "conditionValue": 0,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/cassandra/spans_cassandra_unreachable_node.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] spans-cassandra node unreachable for > 5m [TF]",
+  "description": "A node in the spans-cassandra cluster is not reachable since 5 minutes. This usually indicates overall load problems in the cluster or a issue with the underlying cloud VM. Try restarting the process and investigate overall cluster load afterwards.",
+  "query": "entity.cassandra.cluster.name:\"*spans*\"",
+  "entityType": "cassandraCluster",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "unreachableNodeCount",
+      "aggregation": "min",
+      "window": 300000,
+      "conditionOperator": ">",
+      "conditionValue": 0,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/cassandra/state_cassandra_unreachable_node.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] state-cassandra node unreachable for > 5m [TF]",
+  "description": "A node in the state-cassandra cluster is not reachable since 5 minutes. This usually indicates overall load problems in the cluster or a issue with the underlying cloud VM. Try restarting the process and investigate overall cluster load afterwards.",
+  "query": "entity.cassandra.cluster.name:\"*state*\"",
+  "entityType": "cassandraCluster",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "unreachableNodeCount",
+      "aggregation": "min",
+      "window": 300000,
+      "conditionOperator": ">",
+      "conditionValue": 0,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/cassandra/state_cassandra_write_latency.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] state-cassandra write latency greater than 1 minute [TF]",
+  "description": "A node in the state-cassandra cluster is having very high write latency. This indicates a major issue which will affect\ndependent components like tag-processor. Try restarting the node.",
+  "query": "entity.cassandra.cluster.name:\"*state*\"",
+  "entityType": "cassandraNode",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "clientrequests.write.mean",
+      "aggregation": "avg",
+      "window": 300000,
+      "conditionOperator": ">",
+      "conditionValue": 60000,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_ap_too_many_parts_per_partition.json

@@ -0,0 +1,23 @@
+{
+  "name": "ClickHouse - AP - Too many parts in one of the partitions of the XXX table [TF]",
+  "description": "https://www.notion.so/instana/ClickHouse-Too-many-parts-in-one-of-the-partitions-of-the-XXX-table-01b7add852d74a499ed0216115ed8804. ##Note: The 'conditionValue' (currently set to 830) is a sample reference for the variable 'active_parts_threshold'. Please adjust this value based on your specific environment.",
+  "query": "entity.type:clickhouse AND entity.zone:ClickHouse",
+  "entityType": "clickHouseDatabase",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "aggregation": "min",
+      "window": 60000,
+      "conditionOperator": ">=",
+      "conditionValue": 830,
+      "severity": 10,
+      "metricPattern": {
+        "prefix": "table_metric",
+        "postfix": "parts",
+        "operator": "any",
+        "placeholder": ""
+      }
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_ap_too_many_rejected_inserts.json

@@ -0,0 +1,18 @@
+{
+  "name": "ClickHouse - AP - Too Many inserts are rejected [TF]",
+  "description": "The data is getting lost. Caused by 'Too many parts' exception due to high number of active data parts for partition. https://www.notion.so/instana/ClickHouse-Too-many-parts-in-one-of-the-partitions-of-the-XXX-table-01b7add852d74a499ed0216115ed8804. ##Note: The 'conditionValue' (currently set to 13500) is a sample reference for the variable 'clickhouse_rejected_inserts_threshold'. Please adjust this value based on your specific environment.",
+  "query": "entity.type:clickhouse AND entity.zone:ClickHouse",
+  "entityType": "clickHouseDatabase",
+  "expirationTime": 3600000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "RejectedInserts",
+      "aggregation": "sum",
+      "window": 3600000,
+      "conditionOperator": ">",
+      "conditionValue": "13500",
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_ap_too_many_simultaneous_queries.json

@@ -0,0 +1,18 @@
+{
+  "name": "ClickHouse - AP - Too many simultaneous queries [TF]",
+  "description": "https://www.notion.so/instana/ClickHouse-Too-many-simultaneous-queries-8adf725a6c4c49088eee518b1af6880e",
+  "query": "entity.type:clickhouse AND entity.zone:ClickHouse",
+  "entityType": "clickHouseDatabase",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "Query",
+      "aggregation": "min",
+      "window": 60000,
+      "conditionOperator": ">=",
+      "conditionValue": 120,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_distribution_queue_filling_up.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] ClickHouse - Distribution queue is filling up [TF]",
+  "description": "https://www.notion.so/instana/Clickhouse-Distribution-queue-is-filling-up-9453591a12df4a5bb9e27",
+  "query": "entity.type:clickhouse",
+  "entityType": "clickHouseDatabase",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "DistributedFilesToInsert",
+      "aggregation": "min",
+      "window": 60000,
+      "conditionOperator": ">",
+      "conditionValue": 1000,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_is_not_running_on_the_expected_hosts.json

@@ -0,0 +1,17 @@
+{
+  "name": "[SREInfraSLO] ClickHouse is not running on the host. [TF]",
+  "description": "The ClickHouse process is not running on the underlying host. This can be caused by issues related to the replication process. Please check ClickHouse logs for any recurring exceptions / errors and if none are present you [start the ClickHouse recovery process](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures).",
+  "query": "entity.zone:\"*clickhouse*\" AND NOT entity.zone:\"ZK*\"",
+  "entityType": "host",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "entity_verification",
+      "severity": 10,
+      "matchingEntityType": "clickHouseDatabase",
+      "matchingOperator": "startsWith",
+      "matchingEntityLabel": "ClickHouse",
+      "offlineDuration": 120000
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_mount_points_full_nvme1n1_used.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] Clickhouse disk fs./dev/nvme1n1.used more than 85% full. [TF]",
+  "description": "This is a critical alert and if no action, the Clickhouse Cluster can go down. A disk attached to a Clickhouse server at fs./dev/nvme1n1.used is greater than 85%. This indicates that the disk is going to reach its maximum capacity. When triggered, please check the Clickhouse cluster capacity and coordinate with the rest of the team to decide to increase the disk or add more shards.",
+  "query": "entity.type:clickhouse AND entity.zone:ClickHouse",
+  "entityType": "host",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "fs./dev/nvme1n1.used",
+      "aggregation": "max",
+      "window": 300000,
+      "conditionOperator": ">=",
+      "conditionValue": 0.85,
+      "severity": 10
+    }
+  ]
+}

packages/@instana-integration/instana-self-monitoring/events/clickhouse/clickhouse_mount_points_full_nvme2n1_used.json

@@ -0,0 +1,18 @@
+{
+  "name": "[SREInfraSLO] Clickhouse disk fs./dev/nvme2n1.used more than 85% full. [TF]",
+  "description": "This is a critical alert and if no action, the Clickhouse Cluster can go down. A disk attached to a Clickhouse server at fs./dev/nvme2n1.used is greater than 85%. This indicates that the disk is going to reach its maximum capacity. When triggered, please check the Clickhouse cluster capacity and coordinate with the rest of the team to decide to increase the disk or add more shards.",
+  "query": "entity.type:clickhouse AND entity.zone:ClickHouse",
+  "entityType": "host",
+  "expirationTime": 300000,
+  "rules": [
+    {
+      "ruleType": "threshold",
+      "metricName": "fs./dev/nvme2n1.used",
+      "aggregation": "max",
+      "window": 300000,
+      "conditionOperator": ">=",
+      "conditionValue": 0.85,
+      "severity": 10
+    }
+  ]
+}