crowdstrike: fix FDR deduplication for CSPM findings

packages/crowdstrike/_dev/build/docs/README.md

@@ -315,6 +315,8 @@ To resolve this, adjust the `Batch Size` setting in the integration to reduce th
 
 The option `Enable Data Deduplication` allows you to avoid consuming duplicate events. By default, this option is set to `false`, and so duplicate events can be ingested. When this option is enabled, a [fingerprint processor](https://www.elastic.co/guide/en/elasticsearch/reference/current/fingerprint-processor.html) is used to calculate a hash from a set of CrowdStrike fields that uniquely identify the event. The hash is assigned to the Elasticsearch [`_id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-id-field.html) field that makes the document unique and prevent duplicates.
 
+The fingerprint includes `@timestamp`, CrowdStrike `id`/`aid`/`cid`, and the FDR object type (`aidmaster`, `userinfo`, or `data`), derived from `log.file.path` or `aws.s3.object.key`. When present, `rule.id` is also included — for Cloud Security (CSPM) findings and for other event types that map a rule id (for example EPP detection summary, FIM rule matched, and Data Protection detection summary). For Cloud Security findings, a resource identifier is included as well so distinct findings that share a timestamp and customer id stay unique.
+
 If duplicate events are ingested, to help find them, the integration's `event.id` field is populated by concatenating a few CrowdStrike fields that uniquely identify the event. These fields are `id`, `aid`, and `cid` from the CrowdStrike event. The fields are separated with pipe `|`.
 For example, if your CrowdStrike event contains `id: 123`, `aid: 456`, and `cid: 789` then the `event.id` would be `123|456|789`.
 

packages/crowdstrike/changelog.yml

@@ -1,4 +1,17 @@
 # newer versions go on top
+- version: "4.4.2"
+  changes:
+    - description: Classify FDR aidmaster/userinfo/data object types from aws.s3.object.key when log.file.path is absent.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/20331
+    - description: >-
+        Run CSPM pipelines before the deduplication fingerprint and include rule.id
+        (when present) plus a CSPM resource id so distinct Cloud Security findings are
+        retained and re-ingested duplicates are dropped. Because rule.id is also set on
+        EPP, FIM, and Data Protection events, enabling deduplication after upgrade
+        produces a one-time _id change for those event types on re-delivery.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/20331
 - version: "4.4.1"
   changes:
     - description: Declare the ANODE anomaly-indicator numeric leaves as `float` in the `falcon`, `fdr`, and `alert` data streams. These values are polymorphic (usually fractional, occasionally whole numbers), so leaving them undeclared let dynamic mapping lock them to `long` on integer-first indices and reject later fractional values to the failure store.

packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log

@@ -126,3 +126,10 @@
 {"aid":"11111111111111111111111111111111","cid":"22222222222222222222222222222222","hostname":"example-XXXXXXXXX","os_version":"Sonoma (14)","product_name":"","product_type_desc":"Workstation","host_hidden_status":"VISIBLE","event_platform":"Mac","scores":{"os":89,"sensor":100,"overall":97,"version":"3.8.1","modified_time":"2024-02-13T22:33:34.077075097Z"},"assessments":{"analytics_and_improvements_mac":"yes","application_firewall_mac":"yes","crendential_dumping_hash_mac":"yes","crendential_dumping_kcpassword_mac":"yes","crowdstrike_full_disk_access":"yes","execution_blocking_custom_blocking_enabled_mac":"yes","execution_blocking_intel_threats_enabled_mac":"yes","execution_blocking_suspicious_processes_enabled_mac":"yes","file_vault_enabled_mac":"yes","gatekeeper_mac":"yes","internet_sharing_mac":"yes","mac_os_version":"yes","ml_adware_detection_mac":"yes","ml_adware_prevention_mac":"yes","ml_cloud_antimalware_detection_mac":"yes","ml_cloud_antimalware_prevention_mac":"yes","ml_sensor_adware_and_pup_detection_mac":"yes","ml_sensor_adware_and_pup_prevention_mac":"yes","ml_sensor_antimalware_detection_mac":"yes","ml_sensor_antimalware_prevention_mac":"yes","quarantine_mac":"yes","real_time_response_enabled_mac":"yes","remote_login_mac":"yes","script_based_execution_monitoring_mac":"yes","sip_enabled_mac":"yes","stealth_mode_mac":"no","system_full_disk_access_mac":"no","unauthorized_remote_access_chopper_mac":"yes","unauthorized_remote_access_empyre_mac":"yes","unauthorized_remote_access_xpcom_mac":"yes"},"event_type":"ZeroTrustHostAssessment","timestamp":"1601546312519"}
 {"AccountType":"Domain User","LastLoggedOnHost":"COMPUTER1","LocalAdminAccess":"No","LogonInfo":"Domain User Logon","LogonTime":"1702546155.197","LogonType":"Interactive","PasswordLastSet":"1699971198.062","User":{"Name":"DOMAIN\\BRADLEYA","ID":"1000"},"UserIsAdmin":"0","UserLogonFlags_decimal":"0","UserSid_readable":"S-1-12-1-3697283754-1083485977-2164330645-2516515886","_time":"1702546168.576","cid":"ffffffff15754bcfb5f9152ec7ac90ad","event_platform":"Win","monthsincereset":"1.0"}
 {"ChangeId":"ca65aa54f7b9453b8ef199a5b2c8e3c4","Host":{"Name":"LINUX-TEST-HOST-01"},"User":{"Name":"testuser","ID":"1000"},"Policy":{"Name":"FileVantage Policy","RuleGroupName":"FileVantage Rule Group","RuleBasePath":"/home/testuser/filevantage/","ID":"8fd42a5c9ac24959a98d9e430837b5e6"},"Prevalence":{"Key":"1:3:DIR:CREATE:/home/testuser/filevantage/suppressed::node:testuser"},"Suppression":{"Suppressed":false},"ContentDiff":{"Exists":false,"SHA256":""},"CustomerIdString":"2cc98db1a47b4c98b913c94d43bfab70","UTCTimestamp":1764581217862,"Nonce":13140498271151144192,"AgentIdString":"2e3d9c94d9c34764860b1f3b444c6d4d","EventUUID":"ca65aa54-f7b9-453b-8ef1-99a5b2c8e3c4","cid":"2cc98db1a47b4c98b913c94d43bfab70","eid":118,"timestamp":"2025-12-01T09:26:57Z","EventType":"Event_ExternalApiEvent","ExternalApiType":"Event_FileIntegrityMonitorRuleMatchedEnriched"}
+{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|global|AWS::Account|123456789012","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc16f84-2de2-4f2e-9f9c-d510f47b75fb","ruleName":"[Custom Test] EBS volume encryption is not enabled by default in all regions","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"123456789012","resourceType":"AWS::Account","legacyResourceId":"123456789012","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}}
+{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|us-east-1|AWS::S3::Bucket|example-bucket-public","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"def27a95-3ef3-5a3f-0a0d-e621a58c86ac","ruleName":"[Custom Test] S3 buckets should block public access","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"example-bucket-public","resourceType":"AWS::S3::Bucket","legacyResourceId":"example-bucket-public","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}}
+{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|us-east-1|AWS::EC2::SecurityGroup|sg-0abc123def4567890","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc38b06-4af4-6b4a-1b1e-f732b69d97bd","ruleName":"[Custom Test] Security groups should not allow unrestricted SSH","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"sg-0abc123def4567890","resourceType":"AWS::EC2::SecurityGroup","legacyResourceId":"sg-0abc123def4567890","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}}
+{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|global|AWS::Account|123456789012","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc16f84-2de2-4f2e-9f9c-d510f47b75fb","ruleName":"[Custom Test] EBS volume encryption is not enabled by default in all regions","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"123456789012","resourceType":"AWS::Account","legacyResourceId":"123456789012","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}}
+{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"8db1ca21-4d8b-4c08-b7bc-a63186cd7740","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:c7c1f904-44bb-4690-9816-c510246c3b6a:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"c7c1f904-44bb-4690-9816-c510246c3b6a","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"}
+{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"9ec2db32-5e9c-5d19-c8cd-b74297de8851","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:d8d2a015-55cc-5701-a927-d621357d4c7b:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"d8d2a015-55cc-5701-a927-d621357d4c7b","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"}
+{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"8db1ca21-4d8b-4c08-b7bc-a63186cd7740","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:c7c1f904-44bb-4690-9816-c510246c3b6a:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"c7c1f904-44bb-4690-9816-c510246c3b6a","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"}

packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/main.tf

@@ -100,14 +100,14 @@ resource "aws_scheduler_schedule" "eventbridge_scheduler_every1minute" {
       cid        = "ffffffff15754bcfb5f9152ec7ac90ac"
       timestamp  = 1625677488615
       fileCount  = 3
-      totalSize  = 120161
+      totalSize  = 132054
       bucket     = aws_s3_bucket.crowdstrike_fdr.id
       pathPrefix = "data/f0714ca5-3689-448d-b5cc-582a6f7a56b1"
       "files" : [
         {
           "path" : aws_s3_object.crowdstrike_data.key,
-          "size" : 115258,
-          "checksum" : "c24b5525ad5d4b3ff92bb3c9c002bdc7"
+          "size" : 127151,
+          "checksum" : "4181bff30762315fd386c4a04467836d"
         },
         {
           "path" : aws_s3_object.crowdstrike_aidmaster.key,

packages/crowdstrike/data_stream/fdr/_dev/test/system/test-default-config.yml

@@ -13,4 +13,7 @@ data_stream:
     preserve_original_event: true
     enable_deduplication: true
 assert:
-  hit_count: 133
+  # Sample has 141 lines (135 data + 5 aidmaster + 1 userinfo).
+  # Three data fingerprint collisions (one pre-existing pair, one CSPM IOM
+  # re-ingest, one CSPM IOA re-ingest); with deduplication enabled hit_count is 138.
+  hit_count: 138