From 6e2de4927a543a7196ba715f84c7e82392c86295 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 24 Jul 2026 16:58:28 +0530 Subject: [PATCH 1/4] [m365_defender] Fix alert and incident pagination The `alert` and `incident` data streams paginated by incrementing `$skip` by `batch_size` on each page while carrying the original `$filter`. Under high volume this stalled catch-up: past a certain `$skip` offset the Graph security API stopped returning further pages, so a backlog was never fully drained within a single interval (reported in #20234). Switch both streams to time-boundary pagination: hold `$skip` at 0 and, on a full page, advance `$filter` to `lastUpdateDateTime gt ` (the last element, since `$orderby` is ascending). A non-full page renders an empty filter value, which stops pagination. The stored cursor is unchanged, so restart behaviour is preserved. Also correct the `alert` `batch_size` default from 2000 to 1000 and clarify the `batch_size` field descriptions. Update the alert and incident system tests (and their mock configs) to exercise multi-page pagination driven by the `$filter` boundary. --- .../deploy/docker/alert-http-mock-config.yml | 13 +++++++++++-- .../docker/incident-http-mock-config.yml | 19 ++++++++++++++----- .../incident-no-alerts-http-mock-config.yml | 18 +++++++++++++----- packages/m365_defender/changelog.yml | 5 +++++ .../alert/agent/stream/httpjson.yml.hbs | 11 +++++++++-- .../data_stream/alert/manifest.yml | 4 ++-- .../incident/agent/stream/httpjson.yml.hbs | 11 +++++++++-- .../data_stream/incident/manifest.yml | 2 +- packages/m365_defender/manifest.yml | 2 +- 9 files changed, 65 insertions(+), 20 deletions(-) diff --git a/packages/m365_defender/_dev/deploy/docker/alert-http-mock-config.yml b/packages/m365_defender/_dev/deploy/docker/alert-http-mock-config.yml index 080c2ce261a..c0fa3f8aca8 100644 --- a/packages/m365_defender/_dev/deploy/docker/alert-http-mock-config.yml +++ b/packages/m365_defender/_dev/deploy/docker/alert-http-mock-config.yml @@ -38,9 +38,11 @@ rules: "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..." } `}} + # Page 1: initial request uses the 'ge' cursor filter (dynamic timestamp). - path: /v1.0/security/alerts_v2 methods: [GET] query_params: + $filter: "{initial:lastUpdateDateTime ge .*}" $skip: 0 $top: 2 request_headers: @@ -190,10 +192,14 @@ rules: ] } `}} + # Page 2: pagination advances the filter to 'gt ' + # while keeping $skip at 0. If the input still used $skip offsets it would send + # $skip=2 here, which no rule matches, and --exit-on-unmatched-rule would fail. - path: /v1.0/security/alerts_v2 methods: [GET] query_params: - $skip: 2 + $filter: "lastUpdateDateTime gt 2023-10-20T09:54:10.4666667Z" + $skip: 0 $top: 2 request_headers: Authorization: @@ -354,10 +360,13 @@ rules: ] } `}} + # Page 3: filter advances to 'gt '. The empty + # page (len < batch_size) stops pagination. - path: /v1.0/security/alerts_v2 methods: [GET] query_params: - $skip: 4 + $filter: "lastUpdateDateTime gt 2023-10-20T09:54:11.4966667Z" + $skip: 0 $top: 2 request_headers: Authorization: diff --git a/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml b/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml index 1a7fc16f71a..e79023a46bf 100644 --- a/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml +++ b/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml @@ -38,9 +38,11 @@ rules: "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..." } `}} + # Page 1: initial request uses the 'ge' cursor filter (dynamic timestamp). - path: /v1.0/security/incidents methods: [GET] query_params: + $filter: "{initial:lastUpdateDateTime ge .*}" $skip: 0 $top: 2 request_headers: @@ -158,7 +160,7 @@ rules: "tenantId": "b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:46.1133333Z", "assignedTo": "KaiC@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -236,10 +238,14 @@ rules: ] } `}} + # Page 2: pagination advances the filter to 'gt ' + # while keeping $skip at 0. If the input still used $skip offsets it would send + # $skip=2 here, which no rule matches, and --exit-on-unmatched-rule would fail. - path: /v1.0/security/incidents methods: [GET] query_params: - $skip: 2 + $filter: "lastUpdateDateTime gt 2021-09-30T09:35:46.1133333Z" + $skip: 0 $top: 2 request_headers: Authorization: @@ -261,7 +267,7 @@ rules: "tenantId": "b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:47.1133333Z", "assignedTo": "KaiC@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -366,7 +372,7 @@ rules: "tenantId": "b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:48.1133333Z", "assignedTo": "KaiC@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -434,10 +440,13 @@ rules: ] } `}} + # Page 3: filter advances to 'gt '. The empty + # page (len < batch_size) stops pagination. - path: /v1.0/security/incidents methods: [GET] query_params: - $skip: 4 + $filter: "lastUpdateDateTime gt 2021-09-30T09:35:48.1133333Z" + $skip: 0 $top: 2 request_headers: Authorization: diff --git a/packages/m365_defender/_dev/deploy/docker/incident-no-alerts-http-mock-config.yml b/packages/m365_defender/_dev/deploy/docker/incident-no-alerts-http-mock-config.yml index 26b951855aa..b103864f667 100644 --- a/packages/m365_defender/_dev/deploy/docker/incident-no-alerts-http-mock-config.yml +++ b/packages/m365_defender/_dev/deploy/docker/incident-no-alerts-http-mock-config.yml @@ -13,9 +13,11 @@ rules: - "application/json" body: |- {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600} + # Page 1: initial request uses the 'ge' cursor filter (dynamic timestamp). - path: /v1.0/security/incidents methods: [GET] query_params: + $filter: "{initial:lastUpdateDateTime ge .*}" $skip: 0 $top: 2 request_headers: @@ -64,7 +66,7 @@ rules: "tenantId": "b3c1b5fc-1234-abcd-5678-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:46.1133333Z", "assignedTo": "admin@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -84,10 +86,13 @@ rules: ] } `}} + # Page 2: pagination advances the filter to 'gt ' + # while keeping $skip at 0. - path: /v1.0/security/incidents methods: [GET] query_params: - $skip: 2 + $filter: "lastUpdateDateTime gt 2021-09-30T09:35:46.1133333Z" + $skip: 0 $top: 2 request_headers: Authorization: @@ -110,7 +115,7 @@ rules: "tenantId": "b3c1b5fc-1234-abcd-5678-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:47.1133333Z", "assignedTo": "example@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -135,7 +140,7 @@ rules: "tenantId": "b3c1b5fc-1234-abcd-5678-10d74f6d6e9c", "displayName": "Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources", "createdDateTime": "2021-08-13T08:43:35.5533333Z", - "lastUpdateDateTime": "2021-09-30T09:35:45.1133333Z", + "lastUpdateDateTime": "2021-09-30T09:35:48.1133333Z", "assignedTo": "example@contoso.onmicrosoft.com", "classification": "truePositive", "determination": "multiStagedAttack", @@ -155,10 +160,13 @@ rules: ] } `}} + # Page 3: filter advances to 'gt '. The empty + # page (len < batch_size) stops pagination. - path: /v1.0/security/incidents methods: [GET] query_params: - $skip: 4 + $filter: "lastUpdateDateTime gt 2021-09-30T09:35:48.1133333Z" + $skip: 0 $top: 2 request_headers: Authorization: diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index 19e31885f75..d78b5633433 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "5.15.1" + changes: + - description: Fix `alert` and `incident` pagination to advance the `$filter` time boundary (`lastUpdateDateTime gt `) with `$skip` held at 0, instead of incrementing `$skip`. This avoids the Graph security API `$skip <= 500` limit (which caused `400` errors and stalled catch-up under high volume) and drains a backlog in a single interval. Also correct the `alert` `batch_size` default from 2000 to 1000, matching the API `$top` cap. + type: bugfix + link: https://github.com/elastic/integrations/pull/20234 - version: "5.15.0" changes: - description: Use new `release` field for agentless deployment mode to establish as beta. diff --git a/packages/m365_defender/data_stream/alert/agent/stream/httpjson.yml.hbs b/packages/m365_defender/data_stream/alert/agent/stream/httpjson.yml.hbs index f5684a1dc96..6cef268afdc 100644 --- a/packages/m365_defender/data_stream/alert/agent/stream/httpjson.yml.hbs +++ b/packages/m365_defender/data_stream/alert/agent/stream/httpjson.yml.hbs @@ -41,14 +41,21 @@ request.transforms: value: 'include-unknown-enum-members' {{/if}} response.pagination: + # The Graph security API caps $skip at 500 and $top at 1000, and alerts_v2 does + # not return @odata.nextLink, so we cannot page with a growing $skip offset. + # Instead, keep $skip at 0 and advance the $filter time boundary: on a full page + # (len == batch_size) move to 'lastUpdateDateTime gt ' (the last element, since $orderby is ascending). A non-full page renders + # an empty value, which stops pagination. + # https://github.com/microsoftgraph/microsoft-graph-docs-contrib/blob/main/api-reference/beta/resources/security-error-codes.md#constraints - set: target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) {{batch_size}})]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' fail_on_template_error: true do_not_log_failure: true - set: target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) {{batch_size}})]][[add (toInt (.last_response.url.params.Get "$skip")) {{batch_size}}]][[end]]' + value: '0' fail_on_template_error: true do_not_log_failure: true response.split: diff --git a/packages/m365_defender/data_stream/alert/manifest.yml b/packages/m365_defender/data_stream/alert/manifest.yml index d96bbb959a4..839a3ebbcb2 100644 --- a/packages/m365_defender/data_stream/alert/manifest.yml +++ b/packages/m365_defender/data_stream/alert/manifest.yml @@ -34,8 +34,8 @@ streams: - name: batch_size type: integer title: Batch Size - description: Batch size for the response of the Alert Security Graph API V2. The maximum supported batch size value is 2000. - default: 2000 + description: Number of alerts to request per page ($top) from the Alert Security Graph API V2. The API caps $top at 1000; larger values are silently clamped, so the maximum effective batch size is 1000. + default: 1000 multi: false required: true show_user: false diff --git a/packages/m365_defender/data_stream/incident/agent/stream/httpjson.yml.hbs b/packages/m365_defender/data_stream/incident/agent/stream/httpjson.yml.hbs index 6d03bf0630d..d525b554f56 100644 --- a/packages/m365_defender/data_stream/incident/agent/stream/httpjson.yml.hbs +++ b/packages/m365_defender/data_stream/incident/agent/stream/httpjson.yml.hbs @@ -41,14 +41,21 @@ request.transforms: value: 'alerts' {{/if}} response.pagination: + # The Graph security API caps $skip at 500 and $top at 1000, and incidents do + # not return @odata.nextLink, so we cannot page with a growing $skip offset. + # Instead, keep $skip at 0 and advance the $filter time boundary: on a full page + # (len == batch_size) move to 'lastUpdateDateTime gt ' (the last element, since $orderby is ascending). A non-full page renders + # an empty value, which stops pagination. + # https://github.com/microsoftgraph/microsoft-graph-docs-contrib/blob/main/api-reference/beta/resources/security-error-codes.md#constraints - set: target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) {{batch_size}})]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' fail_on_template_error: true do_not_log_failure: true - set: target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) {{batch_size}})]][[add (toInt (.last_response.url.params.Get "$skip")) {{batch_size}}]][[end]]' + value: '0' fail_on_template_error: true do_not_log_failure: true response.split: diff --git a/packages/m365_defender/data_stream/incident/manifest.yml b/packages/m365_defender/data_stream/incident/manifest.yml index cd995e44f51..a80d1db27a9 100644 --- a/packages/m365_defender/data_stream/incident/manifest.yml +++ b/packages/m365_defender/data_stream/incident/manifest.yml @@ -33,7 +33,7 @@ streams: - name: batch_size type: integer title: Batch Size - description: Batch size for the response of the Incident Security Graph API V2. The maximum supported batch size value is 50. + description: Number of incidents to request per page ($top) from the Incident Security Graph API V2. The API caps $top at 1000; larger values are silently clamped. A conservative default is used because each incident expands its alerts, which increases per-page memory. default: 50 multi: false required: true diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml index 030be454f21..dcefff3fedd 100644 --- a/packages/m365_defender/manifest.yml +++ b/packages/m365_defender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: m365_defender title: Microsoft Defender XDR -version: "5.15.0" +version: "5.15.1" description: Collect logs from Microsoft Defender XDR with Elastic Agent. categories: - "security" From 2a8cf7d70834b8f8905bbeae181624ed45769c50 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 24 Jul 2026 16:59:43 +0530 Subject: [PATCH 2/4] update pr number --- packages/m365_defender/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index d78b5633433..bd6bf924cb7 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Fix `alert` and `incident` pagination to advance the `$filter` time boundary (`lastUpdateDateTime gt `) with `$skip` held at 0, instead of incrementing `$skip`. This avoids the Graph security API `$skip <= 500` limit (which caused `400` errors and stalled catch-up under high volume) and drains a backlog in a single interval. Also correct the `alert` `batch_size` default from 2000 to 1000, matching the API `$top` cap. type: bugfix - link: https://github.com/elastic/integrations/pull/20234 + link: https://github.com/elastic/integrations/pull/20348 - version: "5.15.0" changes: - description: Use new `release` field for agentless deployment mode to establish as beta. From 8d7e34a063aa1cf716b084f5dcd8d3a37ecbb4f2 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 24 Jul 2026 17:33:07 +0530 Subject: [PATCH 3/4] update policy tests --- .../alert/_dev/test/policy/test-all.expected | 4 ++-- .../alert/_dev/test/policy/test-default.expected | 4 ++-- .../data_stream/alert/sample_event.json | 16 ++++++++-------- .../incident/_dev/test/policy/test-all.expected | 4 ++-- .../_dev/test/policy/test-default.expected | 4 ++-- .../_dev/test/policy/test-no-alerts.expected | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/m365_defender/data_stream/alert/_dev/test/policy/test-all.expected b/packages/m365_defender/data_stream/alert/_dev/test/policy/test-all.expected index 78711d43d1d..075e329c34f 100644 --- a/packages/m365_defender/data_stream/alert/_dev/test/policy/test-all.expected +++ b/packages/m365_defender/data_stream/alert/_dev/test/policy/test-all.expected @@ -136,12 +136,12 @@ inputs: do_not_log_failure: true fail_on_template_error: true target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) 2000)]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' - set: do_not_log_failure: true fail_on_template_error: true target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) 2000)]][[add (toInt (.last_response.url.params.Get "$skip")) 2000]][[end]]' + value: "0" response.split: ignore_empty_value: true target: body.value diff --git a/packages/m365_defender/data_stream/alert/_dev/test/policy/test-default.expected b/packages/m365_defender/data_stream/alert/_dev/test/policy/test-default.expected index 8a91fcc6458..02b230a4bcc 100644 --- a/packages/m365_defender/data_stream/alert/_dev/test/policy/test-default.expected +++ b/packages/m365_defender/data_stream/alert/_dev/test/policy/test-default.expected @@ -44,12 +44,12 @@ inputs: do_not_log_failure: true fail_on_template_error: true target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) 2000)]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' - set: do_not_log_failure: true fail_on_template_error: true target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) 2000)]][[add (toInt (.last_response.url.params.Get "$skip")) 2000]][[end]]' + value: "0" response.split: ignore_empty_value: true target: body.value diff --git a/packages/m365_defender/data_stream/alert/sample_event.json b/packages/m365_defender/data_stream/alert/sample_event.json index 9b479ee170d..d32561ff4df 100644 --- a/packages/m365_defender/data_stream/alert/sample_event.json +++ b/packages/m365_defender/data_stream/alert/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2023-10-20T09:54:07.503Z", "agent": { - "ephemeral_id": "f0b1ddcf-5c43-49cb-be37-acefdf4372d0", - "id": "d6e963cc-04ee-46b8-af02-520891431883", - "name": "elastic-agent-85133", + "ephemeral_id": "a0df1f66-1808-466a-b3b8-8ce68832f22a", + "id": "9c8ca8ce-fef7-4bbf-80b1-c5de3d4ba7ae", + "name": "elastic-agent-93183", "type": "filebeat", - "version": "8.19.4" + "version": "8.19.10" }, "cloud": { "account": { @@ -14,7 +14,7 @@ }, "data_stream": { "dataset": "m365_defender.alert", - "namespace": "57227", + "namespace": "21064", "type": "logs" }, "device": { @@ -26,9 +26,9 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "d6e963cc-04ee-46b8-af02-520891431883", + "id": "9c8ca8ce-fef7-4bbf-80b1-c5de3d4ba7ae", "snapshot": false, - "version": "8.19.4" + "version": "8.19.10" }, "event": { "agent_id_status": "verified", @@ -40,7 +40,7 @@ "duration": 2478000000, "end": "2023-10-20T09:51:41.993Z", "id": "daefa1828b-dd4e-405c-8a3b-aa28596830dd_1", - "ingested": "2026-01-05T12:04:28Z", + "ingested": "2026-07-24T11:51:12Z", "kind": "alert", "original": "{\"actorDisplayName\":null,\"additionalData\":null,\"alertPolicyId\":null,\"alertWebUrl\":\"https://security.microsoft.com/alerts/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"assignedTo\":null,\"category\":\"Execution\",\"classification\":null,\"comments\":[],\"createdDateTime\":\"2023-10-20T09:53:09.8839373Z\",\"description\":\"A suspicious PowerShell activity was observed on the machine. \\nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.\",\"detectionSource\":\"microsoftDefenderForEndpoint\",\"detectorId\":\"7f1c3609-a3ff-40e2-995b-c01770161d68\",\"determination\":null,\"evidence\":[{\"@odata.type\":\"#microsoft.graph.security.deviceEvidence\",\"azureAdDeviceId\":\"f18bd540-d5e4-46e0-8ddd-3d03a59e4e14\",\"createdDateTime\":\"2023-10-20T09:53:10.1933333Z\",\"defenderAvStatus\":\"notSupported\",\"detailedRoles\":[\"PrimaryDevice\"],\"deviceDnsName\":\"clw555test\",\"firstSeenDateTime\":\"2023-10-20T09:50:17.7383987Z\",\"healthStatus\":\"inactive\",\"ipInterfaces\":[\"192.168.5.65\",\"fe80::cfe4:80b:615c:38fb\",\"127.0.0.1\",\"::1\"],\"loggedOnUsers\":[{\"accountName\":\"CDPUserIS-38411\",\"domainName\":\"AzureAD\"}],\"mdeDeviceId\":\"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843\",\"onboardingStatus\":\"onboarded\",\"osBuild\":22621,\"osPlatform\":\"Windows11\",\"rbacGroupId\":0,\"rbacGroupName\":null,\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"riskScore\":\"high\",\"roles\":[],\"tags\":[],\"verdict\":\"unknown\",\"version\":\"22H2\",\"vmMetadata\":null}],\"firstActivityDateTime\":\"2023-10-20T09:51:39.5154802Z\",\"id\":\"daefa1828b-dd4e-405c-8a3b-aa28596830dd_1\",\"incidentId\":\"23\",\"incidentWebUrl\":\"https://security.microsoft.com/incidents/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"lastActivityDateTime\":\"2023-10-20T09:51:41.9939003Z\",\"lastUpdateDateTime\":\"2023-10-20T09:54:07.5033333Z\",\"mitreTechniques\":[\"T1059.001\"],\"productName\":\"Microsoft Defender for Endpoint\",\"providerAlertId\":\"efa1828b-dd4e-405c-8a3b-aa28596830dd_1\",\"recommendedActions\":\"1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\\n2. Search the script for more indicators to investigate - for example IP addresses (potential C\\u0026C servers), target computers etc.\\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.\",\"resolvedDateTime\":null,\"serviceSource\":\"microsoftDefenderForEndpoint\",\"severity\":\"medium\",\"status\":\"new\",\"tenantId\":\"3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"threatDisplayName\":null,\"threatFamilyName\":null,\"title\":\"Suspicious PowerShell command line\"}", "provider": "microsoftDefenderForEndpoint", diff --git a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-all.expected b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-all.expected index c1bfb687c10..726174503a1 100644 --- a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-all.expected +++ b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-all.expected @@ -144,12 +144,12 @@ inputs: do_not_log_failure: true fail_on_template_error: true target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) 2000)]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' - set: do_not_log_failure: true fail_on_template_error: true target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) 2000)]][[add (toInt (.last_response.url.params.Get "$skip")) 2000]][[end]]' + value: "0" response.split: ignore_empty_value: true split: diff --git a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-default.expected b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-default.expected index 7db0ec5fd7c..08f984f034f 100644 --- a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-default.expected +++ b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-default.expected @@ -52,12 +52,12 @@ inputs: do_not_log_failure: true fail_on_template_error: true target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) 50)]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' - set: do_not_log_failure: true fail_on_template_error: true target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) 50)]][[add (toInt (.last_response.url.params.Get "$skip")) 50]][[end]]' + value: "0" response.split: ignore_empty_value: true split: diff --git a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-no-alerts.expected b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-no-alerts.expected index 447bae0f018..f620cff188a 100644 --- a/packages/m365_defender/data_stream/incident/_dev/test/policy/test-no-alerts.expected +++ b/packages/m365_defender/data_stream/incident/_dev/test/policy/test-no-alerts.expected @@ -49,12 +49,12 @@ inputs: do_not_log_failure: true fail_on_template_error: true target: url.params.$filter - value: '[[.last_response.url.params.Get "$filter"]]' + value: '[[if (eq (len .last_response.body.value) 50)]]lastUpdateDateTime gt [[(index .last_response.body.value (add (toInt (len .last_response.body.value)) -1)).lastUpdateDateTime]][[end]]' - set: do_not_log_failure: true fail_on_template_error: true target: url.params.$skip - value: '[[if (eq (len .last_response.body.value) 50)]][[add (toInt (.last_response.url.params.Get "$skip")) 50]][[end]]' + value: "0" response.split: ignore_empty_value: true target: body.value From 22065f0b63cff56e735f74fa979e9f178d135cac Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 24 Jul 2026 17:50:27 +0530 Subject: [PATCH 4/4] reset alert sample event --- .../data_stream/alert/sample_event.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/m365_defender/data_stream/alert/sample_event.json b/packages/m365_defender/data_stream/alert/sample_event.json index d32561ff4df..9b479ee170d 100644 --- a/packages/m365_defender/data_stream/alert/sample_event.json +++ b/packages/m365_defender/data_stream/alert/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2023-10-20T09:54:07.503Z", "agent": { - "ephemeral_id": "a0df1f66-1808-466a-b3b8-8ce68832f22a", - "id": "9c8ca8ce-fef7-4bbf-80b1-c5de3d4ba7ae", - "name": "elastic-agent-93183", + "ephemeral_id": "f0b1ddcf-5c43-49cb-be37-acefdf4372d0", + "id": "d6e963cc-04ee-46b8-af02-520891431883", + "name": "elastic-agent-85133", "type": "filebeat", - "version": "8.19.10" + "version": "8.19.4" }, "cloud": { "account": { @@ -14,7 +14,7 @@ }, "data_stream": { "dataset": "m365_defender.alert", - "namespace": "21064", + "namespace": "57227", "type": "logs" }, "device": { @@ -26,9 +26,9 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "9c8ca8ce-fef7-4bbf-80b1-c5de3d4ba7ae", + "id": "d6e963cc-04ee-46b8-af02-520891431883", "snapshot": false, - "version": "8.19.10" + "version": "8.19.4" }, "event": { "agent_id_status": "verified", @@ -40,7 +40,7 @@ "duration": 2478000000, "end": "2023-10-20T09:51:41.993Z", "id": "daefa1828b-dd4e-405c-8a3b-aa28596830dd_1", - "ingested": "2026-07-24T11:51:12Z", + "ingested": "2026-01-05T12:04:28Z", "kind": "alert", "original": "{\"actorDisplayName\":null,\"additionalData\":null,\"alertPolicyId\":null,\"alertWebUrl\":\"https://security.microsoft.com/alerts/daefa1828b-dd4e-405c-8a3b-aa28596830dd_1?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"assignedTo\":null,\"category\":\"Execution\",\"classification\":null,\"comments\":[],\"createdDateTime\":\"2023-10-20T09:53:09.8839373Z\",\"description\":\"A suspicious PowerShell activity was observed on the machine. \\nThis behavior may indicate that PowerShell was used during installation, exploration, or in some cases in lateral movement activities which are used by attackers to invoke modules, download external payloads, or get more information about the system. Attackers usually use PowerShell to bypass security protection mechanisms by executing their payload in memory without touching the disk and leaving any trace.\",\"detectionSource\":\"microsoftDefenderForEndpoint\",\"detectorId\":\"7f1c3609-a3ff-40e2-995b-c01770161d68\",\"determination\":null,\"evidence\":[{\"@odata.type\":\"#microsoft.graph.security.deviceEvidence\",\"azureAdDeviceId\":\"f18bd540-d5e4-46e0-8ddd-3d03a59e4e14\",\"createdDateTime\":\"2023-10-20T09:53:10.1933333Z\",\"defenderAvStatus\":\"notSupported\",\"detailedRoles\":[\"PrimaryDevice\"],\"deviceDnsName\":\"clw555test\",\"firstSeenDateTime\":\"2023-10-20T09:50:17.7383987Z\",\"healthStatus\":\"inactive\",\"ipInterfaces\":[\"192.168.5.65\",\"fe80::cfe4:80b:615c:38fb\",\"127.0.0.1\",\"::1\"],\"loggedOnUsers\":[{\"accountName\":\"CDPUserIS-38411\",\"domainName\":\"AzureAD\"}],\"mdeDeviceId\":\"505d70d89cfa3428f7aac7d2eb3a64c60fd3d843\",\"onboardingStatus\":\"onboarded\",\"osBuild\":22621,\"osPlatform\":\"Windows11\",\"rbacGroupId\":0,\"rbacGroupName\":null,\"remediationStatus\":\"none\",\"remediationStatusDetails\":null,\"riskScore\":\"high\",\"roles\":[],\"tags\":[],\"verdict\":\"unknown\",\"version\":\"22H2\",\"vmMetadata\":null}],\"firstActivityDateTime\":\"2023-10-20T09:51:39.5154802Z\",\"id\":\"daefa1828b-dd4e-405c-8a3b-aa28596830dd_1\",\"incidentId\":\"23\",\"incidentWebUrl\":\"https://security.microsoft.com/incidents/23?tid=3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"lastActivityDateTime\":\"2023-10-20T09:51:41.9939003Z\",\"lastUpdateDateTime\":\"2023-10-20T09:54:07.5033333Z\",\"mitreTechniques\":[\"T1059.001\"],\"productName\":\"Microsoft Defender for Endpoint\",\"providerAlertId\":\"efa1828b-dd4e-405c-8a3b-aa28596830dd_1\",\"recommendedActions\":\"1. Examine the PowerShell command line to understand what commands were executed. Note: the content may need to be decoded if it is Base64-encoded.\\n2. Search the script for more indicators to investigate - for example IP addresses (potential C\\u0026C servers), target computers etc.\\n3. Explore the timeline of this and other related machines for additional suspect activities around the time of the alert.\\n4. Look for the process that invoked this PowerShell run and their origin. Consider submitting any suspect files in the chain for deep analysis for detailed behavior information.\",\"resolvedDateTime\":null,\"serviceSource\":\"microsoftDefenderForEndpoint\",\"severity\":\"medium\",\"status\":\"new\",\"tenantId\":\"3adb963c-8e61-48e8-a06d-6dbb0dacea39\",\"threatDisplayName\":null,\"threatFamilyName\":null,\"title\":\"Suspicious PowerShell command line\"}", "provider": "microsoftDefenderForEndpoint",