diff --git a/packages/crowdstrike/_dev/build/docs/README.md b/packages/crowdstrike/_dev/build/docs/README.md index fd7c01fb8b7..a207a8c32aa 100644 --- a/packages/crowdstrike/_dev/build/docs/README.md +++ b/packages/crowdstrike/_dev/build/docs/README.md @@ -315,6 +315,8 @@ To resolve this, adjust the `Batch Size` setting in the integration to reduce th The option `Enable Data Deduplication` allows you to avoid consuming duplicate events. By default, this option is set to `false`, and so duplicate events can be ingested. When this option is enabled, a [fingerprint processor](https://www.elastic.co/guide/en/elasticsearch/reference/current/fingerprint-processor.html) is used to calculate a hash from a set of CrowdStrike fields that uniquely identify the event. The hash is assigned to the Elasticsearch [`_id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-id-field.html) field that makes the document unique and prevent duplicates. +The fingerprint includes `@timestamp`, CrowdStrike `id`/`aid`/`cid`, and the FDR object type (`aidmaster`, `userinfo`, or `data`), derived from `log.file.path` or `aws.s3.object.key`. When present, `rule.id` is also included — for Cloud Security (CSPM) findings and for other event types that map a rule id (for example EPP detection summary, FIM rule matched, and Data Protection detection summary). For Cloud Security findings, a resource identifier is included as well so distinct findings that share a timestamp and customer id stay unique. + If duplicate events are ingested, to help find them, the integration's `event.id` field is populated by concatenating a few CrowdStrike fields that uniquely identify the event. These fields are `id`, `aid`, and `cid` from the CrowdStrike event. The fields are separated with pipe `|`. For example, if your CrowdStrike event contains `id: 123`, `aid: 456`, and `cid: 789` then the `event.id` would be `123|456|789`. diff --git a/packages/crowdstrike/changelog.yml b/packages/crowdstrike/changelog.yml index b8f9118f39b..60c958bff74 100644 --- a/packages/crowdstrike/changelog.yml +++ b/packages/crowdstrike/changelog.yml @@ -1,4 +1,17 @@ # newer versions go on top +- version: "4.4.2" + changes: + - description: Classify FDR aidmaster/userinfo/data object types from aws.s3.object.key when log.file.path is absent. + type: bugfix + link: https://github.com/elastic/integrations/pull/20331 + - description: >- + Run CSPM pipelines before the deduplication fingerprint and include rule.id + (when present) plus a CSPM resource id so distinct Cloud Security findings are + retained and re-ingested duplicates are dropped. Because rule.id is also set on + EPP, FIM, and Data Protection events, enabling deduplication after upgrade + produces a one-time _id change for those event types on re-delivery. + type: bugfix + link: https://github.com/elastic/integrations/pull/20331 - version: "4.4.1" changes: - description: Declare the ANODE anomaly-indicator numeric leaves as `float` in the `falcon`, `fdr`, and `alert` data streams. These values are polymorphic (usually fractional, occasionally whole numbers), so leaving them undeclared let dynamic mapping lock them to `long` on integer-first indices and reject later fractional values to the failure store. diff --git a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log index f3d053e28c0..4625d379a33 100644 --- a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log +++ b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/files/fdr-sample.log @@ -126,3 +126,10 @@ {"aid":"11111111111111111111111111111111","cid":"22222222222222222222222222222222","hostname":"example-XXXXXXXXX","os_version":"Sonoma (14)","product_name":"","product_type_desc":"Workstation","host_hidden_status":"VISIBLE","event_platform":"Mac","scores":{"os":89,"sensor":100,"overall":97,"version":"3.8.1","modified_time":"2024-02-13T22:33:34.077075097Z"},"assessments":{"analytics_and_improvements_mac":"yes","application_firewall_mac":"yes","crendential_dumping_hash_mac":"yes","crendential_dumping_kcpassword_mac":"yes","crowdstrike_full_disk_access":"yes","execution_blocking_custom_blocking_enabled_mac":"yes","execution_blocking_intel_threats_enabled_mac":"yes","execution_blocking_suspicious_processes_enabled_mac":"yes","file_vault_enabled_mac":"yes","gatekeeper_mac":"yes","internet_sharing_mac":"yes","mac_os_version":"yes","ml_adware_detection_mac":"yes","ml_adware_prevention_mac":"yes","ml_cloud_antimalware_detection_mac":"yes","ml_cloud_antimalware_prevention_mac":"yes","ml_sensor_adware_and_pup_detection_mac":"yes","ml_sensor_adware_and_pup_prevention_mac":"yes","ml_sensor_antimalware_detection_mac":"yes","ml_sensor_antimalware_prevention_mac":"yes","quarantine_mac":"yes","real_time_response_enabled_mac":"yes","remote_login_mac":"yes","script_based_execution_monitoring_mac":"yes","sip_enabled_mac":"yes","stealth_mode_mac":"no","system_full_disk_access_mac":"no","unauthorized_remote_access_chopper_mac":"yes","unauthorized_remote_access_empyre_mac":"yes","unauthorized_remote_access_xpcom_mac":"yes"},"event_type":"ZeroTrustHostAssessment","timestamp":"1601546312519"} {"AccountType":"Domain User","LastLoggedOnHost":"COMPUTER1","LocalAdminAccess":"No","LogonInfo":"Domain User Logon","LogonTime":"1702546155.197","LogonType":"Interactive","PasswordLastSet":"1699971198.062","User":{"Name":"DOMAIN\\BRADLEYA","ID":"1000"},"UserIsAdmin":"0","UserLogonFlags_decimal":"0","UserSid_readable":"S-1-12-1-3697283754-1083485977-2164330645-2516515886","_time":"1702546168.576","cid":"ffffffff15754bcfb5f9152ec7ac90ad","event_platform":"Win","monthsincereset":"1.0"} {"ChangeId":"ca65aa54f7b9453b8ef199a5b2c8e3c4","Host":{"Name":"LINUX-TEST-HOST-01"},"User":{"Name":"testuser","ID":"1000"},"Policy":{"Name":"FileVantage Policy","RuleGroupName":"FileVantage Rule Group","RuleBasePath":"/home/testuser/filevantage/","ID":"8fd42a5c9ac24959a98d9e430837b5e6"},"Prevalence":{"Key":"1:3:DIR:CREATE:/home/testuser/filevantage/suppressed::node:testuser"},"Suppression":{"Suppressed":false},"ContentDiff":{"Exists":false,"SHA256":""},"CustomerIdString":"2cc98db1a47b4c98b913c94d43bfab70","UTCTimestamp":1764581217862,"Nonce":13140498271151144192,"AgentIdString":"2e3d9c94d9c34764860b1f3b444c6d4d","EventUUID":"ca65aa54-f7b9-453b-8ef1-99a5b2c8e3c4","cid":"2cc98db1a47b4c98b913c94d43bfab70","eid":118,"timestamp":"2025-12-01T09:26:57Z","EventType":"Event_ExternalApiEvent","ExternalApiType":"Event_FileIntegrityMonitorRuleMatchedEnriched"} +{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|global|AWS::Account|123456789012","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc16f84-2de2-4f2e-9f9c-d510f47b75fb","ruleName":"[Custom Test] EBS volume encryption is not enabled by default in all regions","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"123456789012","resourceType":"AWS::Account","legacyResourceId":"123456789012","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}} +{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|us-east-1|AWS::S3::Bucket|example-bucket-public","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"def27a95-3ef3-5a3f-0a0d-e621a58c86ac","ruleName":"[Custom Test] S3 buckets should block public access","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"example-bucket-public","resourceType":"AWS::S3::Bucket","legacyResourceId":"example-bucket-public","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}} +{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|us-east-1|AWS::EC2::SecurityGroup|sg-0abc123def4567890","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc38b06-4af4-6b4a-1b1e-f732b69d97bd","ruleName":"[Custom Test] Security groups should not allow unrestricted SSH","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"sg-0abc123def4567890","resourceType":"AWS::EC2::SecurityGroup","legacyResourceId":"sg-0abc123def4567890","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}} +{"event_simpleName":"CloudSecurityIOMEvaluation","cid":"4092825518eaf67377a6e4492ae44577","crn":"aws|123456789012|global|AWS::Account|123456789012","created":"2025-10-13T03:59:08.974734575Z","firstDetected":"2025-10-08T13:03:26.203492662Z","lastDetected":"2025-10-13T03:59:08.974734575Z","revision":7,"ruleId":"abc16f84-2de2-4f2e-9f9c-d510f47b75fb","ruleName":"[Custom Test] EBS volume encryption is not enabled by default in all regions","legacyPolicyId":100056,"severity":"informational","status":"Unresolved","findings":[{"name":"Encryption Enabled","value":"False"},{"name":"Region","value":"[\"ca-central-1\",\"sa-east-1\",\"eu-central-1\",\"eu-west-1\",\"us-east-1\",\"us-east-2\",\"us-west-2\",\"ap-northeast-2\",\"ap-southeast-1\",\"ap-south-1\",\"us-west-1\",\"eu-west-3\",\"ap-northeast-3\",\"ap-southeast-2\",\"ap-northeast-1\",\"eu-west-2\",\"eu-north-1\"]"}],"url":"https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Volumes:","resource":{"accountId":"123456789012","cloudProvider":"aws","region":"global","captured":"2025-10-13T03:59:08.167485551Z","resourceId":"123456789012","resourceType":"AWS::Account","legacyResourceId":"123456789012","legacyResourceTypeId":116},"compliance":{"frameworks":["Amazon","CIS"],"versions":["11.2024","v1.0.0"],"benchmarkNames":["AWS Well-Architected Framework (Section 2 - Security) 11.2024","AWS Foundational Security Best Practices v1.0.0"],"sections":["SEC 8. How do you protect your data at rest?","EC2"],"requirements":["SEC08-BP03","EC2.7"]},"threat":{"framework":"MITRE ATT&CK","technique":{"id":"T1530","name":"Data from Cloud Storage","reference":"https://attack.mitre.org/techniques/T1530/"},"tactic":{"id":"TA0009","name":"Collection","reference":"https://attack.mitre.org/tactics/TA0009/"}}} +{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"8db1ca21-4d8b-4c08-b7bc-a63186cd7740","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:c7c1f904-44bb-4690-9816-c510246c3b6a:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"c7c1f904-44bb-4690-9816-c510246c3b6a","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"} +{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"9ec2db32-5e9c-5d19-c8cd-b74297de8851","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:d8d2a015-55cc-5701-a927-d621357d4c7b:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"d8d2a015-55cc-5701-a927-d621357d4c7b","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"} +{"mitre_attack_technique":"Data Destruction","cloud_service_friendly":"EC2","aws_account_id":"123456789012","policy_severity":1,"event_type":"AwsApiCall","event_name":"TerminateInstances","mitre_attack_tactic":"Impact","event_source":"ec2.amazonaws.com","account":"123456789012","cloud_region":"us-east-2","event_category":"Management","cloud_provider":"aws","attack_types":["Destruction"],"event_created":"2025-10-02T19:51:16Z","user_identity_principal_id":"AIDAEXAMPLEPRINCIPAL01","event-type":"cspm_policy_249","policy_id":249,"request_id":"8db1ca21-4d8b-4c08-b7bc-a63186cd7740","vertex_type":"ioa","cid":"4092825518eaf67377a6e4492ae44577","vertex_id":"249:c7c1f904-44bb-4690-9816-c510246c3b6a:ioa","user_identity_user_name":"example-user","policy_description":"An IAM user was detected to have manually deleted an EC2 instance.","user_identity_arn":"arn:aws:iam::123456789012:user/example-user","source_ip_address":"89.160.20.112","user_identity_account_id":"123456789012","user_identity_mfa_authenticated":"false","user_agent":"aws-sdk-go/1.44.232","cloudplatform":1,"service":"EC2","event_id":"c7c1f904-44bb-4690-9816-c510246c3b6a","read_only":false,"policy_statement":"EC2 instance manually deleted by IAM user","management_event":true,"user_identity_access_key_id":"AKIAEXAMPLEACCESSKEY01"} diff --git a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/main.tf b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/main.tf index ac47ae84b06..8feed383203 100644 --- a/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/main.tf +++ b/packages/crowdstrike/data_stream/fdr/_dev/deploy/tf/main.tf @@ -100,14 +100,14 @@ resource "aws_scheduler_schedule" "eventbridge_scheduler_every1minute" { cid = "ffffffff15754bcfb5f9152ec7ac90ac" timestamp = 1625677488615 fileCount = 3 - totalSize = 120161 + totalSize = 132054 bucket = aws_s3_bucket.crowdstrike_fdr.id pathPrefix = "data/f0714ca5-3689-448d-b5cc-582a6f7a56b1" "files" : [ { "path" : aws_s3_object.crowdstrike_data.key, - "size" : 115258, - "checksum" : "c24b5525ad5d4b3ff92bb3c9c002bdc7" + "size" : 127151, + "checksum" : "4181bff30762315fd386c4a04467836d" }, { "path" : aws_s3_object.crowdstrike_aidmaster.key, diff --git a/packages/crowdstrike/data_stream/fdr/_dev/test/system/test-default-config.yml b/packages/crowdstrike/data_stream/fdr/_dev/test/system/test-default-config.yml index a3f997679a1..a1e596ea4c5 100644 --- a/packages/crowdstrike/data_stream/fdr/_dev/test/system/test-default-config.yml +++ b/packages/crowdstrike/data_stream/fdr/_dev/test/system/test-default-config.yml @@ -13,4 +13,7 @@ data_stream: preserve_original_event: true enable_deduplication: true assert: - hit_count: 133 + # Sample has 141 lines (135 data + 5 aidmaster + 1 userinfo). + # Three data fingerprint collisions (one pre-existing pair, one CSPM IOM + # re-ingest, one CSPM IOA re-ingest); with deduplication enabled hit_count is 138. + hit_count: 138 diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml index d9c3f02b6cb..444ddfcc754 100644 --- a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml @@ -521,24 +521,100 @@ processors: - _id ignore_missing: true + # Classify FDR object type for the fingerprint discriminator. + # Object-key layout places the type mid-path, e.g. + # c9ec061c00000000-ec39fe72/fdrv2/aidmaster//part-00000.gz + # /data//part-00000.gz + # so match path segments rather than basename alone. + - set: + tag: set__temp_path_d3cfbbfe + field: _temp.path + copy_from: log.file.path + ignore_empty_value: true + - set: + tag: set__temp_path_7459eddc + if: ctx._temp?.path == null + field: _temp.path + copy_from: aws.s3.object.key + ignore_empty_value: true + - set: + tag: set__temp_type_1874950f + if: ctx._temp?.path != null + field: _temp.type + value: data + - set: + tag: set__temp_type_89a2e27b + if: >- + ctx._temp?.path != null && + (ctx._temp.path == 'aidmaster' || + ctx._temp.path.endsWith('/aidmaster') || + ctx._temp.path.contains('/aidmaster/')) + field: _temp.type + value: aidmaster + - set: + tag: set__temp_type_ce5d5927 + if: >- + ctx._temp?.path != null && + (ctx._temp.path == 'userinfo' || + ctx._temp.path.endsWith('/userinfo') || + ctx._temp.path.contains('/userinfo/')) + field: _temp.type + value: userinfo + + # CSPM fields — run before fingerprint so @timestamp and rule/resource identity + # are final for Cloud Security events. + # Can be both string and int, fields are mapped as keyword. + - convert: + tag: convert_crowdstrike_service_to_string_2a8687e8 + field: crowdstrike.service + type: string + ignore_missing: true + - convert: + tag: convert_crowdstrike_cloudplatform_to_string_1f6df19b + field: crowdstrike.cloudplatform + type: string + ignore_missing: true + + - rename: + description: Rename crowdstrike.resource in case concrete value field is mapped as object. + tag: rename_crowdstrike_resource_to_crowdstrike_resource_name_9899a394 + if: ctx.crowdstrike?.resource instanceof String + field: crowdstrike.resource + target_field: crowdstrike.resource_name + - remove: + tag: remove_cloud_4d018ef3 + field: + - cloud + ignore_missing: true + + - pipeline: + tag: pipeline_cspm_iom_62dcd2f9 + if: (ctx.crowdstrike?.disposition != null && ctx.crowdstrike.disposition.equalsIgnoreCase('Failed')) ||(ctx.crowdstrike?.event_simpleName != null && ctx.crowdstrike.event_simpleName.equalsIgnoreCase('CloudSecurityIOMEvaluation')) + name: '{{ IngestPipeline "cspm_iom" }}' + - pipeline: + tag: pipeline_cspm_ioa_8a985b05 + if: ctx.crowdstrike?.vertex_type != null && ctx.crowdstrike.vertex_type.equalsIgnoreCase('ioa') + name: '{{ IngestPipeline "cspm_ioa" }}' + + # Resolve a stable CSPM resource identity for fingerprinting. - script: - tag: script_data_type_89bd92f4 - if: ctx.log?.file?.path != null && ctx.log.file.path != '' + tag: script_cspm_resource_id_for_fingerprint_7c3e9a12 + if: ctx.rule?.id != null source: |- - int lastSlash = ctx.log.file.path.lastIndexOf("/"); - if (lastSlash == -1) { - return; - } ctx._temp = ctx._temp ?: [:]; - ctx._temp.type = ctx.log.file.path.substring(lastSlash + 1); - // aidmaster and userinfo are bucket keys we depend on, the data - // path suffix is tested, but not depended on. So make sure this - // is present for the fingerprint processor. - if (ctx._temp.type != 'aidmaster' && ctx._temp.type != 'userinfo') { - ctx._temp.type = 'data'; + if (ctx.crowdstrike?.ResourceId != null && ctx.crowdstrike.ResourceId != '') { + ctx._temp.cspm_resource_id = ctx.crowdstrike.ResourceId.toString(); + } else if (ctx.crowdstrike?.resource?.resourceId != null && ctx.crowdstrike.resource.resourceId != '') { + ctx._temp.cspm_resource_id = ctx.crowdstrike.resource.resourceId.toString(); + } else if (ctx.crowdstrike?.crn != null && ctx.crowdstrike.crn != '') { + ctx._temp.cspm_resource_id = ctx.crowdstrike.crn.toString(); + } else if (ctx.event?.id != null && ctx.event.id != '') { + // IOA: cspm_ioa renames crowdstrike.event_id -> event.id + ctx._temp.cspm_resource_id = ctx.event.id.toString(); } + - fingerprint: - description: When deduplication is enabled, fingerprint the a set of crowdstrike fields in attempt to prevent the same event from being indexed more than once. + description: When deduplication is enabled, fingerprint a set of crowdstrike fields to prevent the same event from being indexed more than once. CSPM events also include rule.id and resource id so distinct findings that share timestamp/cid stay unique. tag: fingerprint_crowdstrike_fdr_0e5ffd3f if: ctx._conf?.enable_deduplication == true fields: @@ -547,6 +623,8 @@ processors: - crowdstrike.aid - crowdstrike.cid - _temp.type + - rule.id + - _temp.cspm_resource_id target_field: _id ignore_missing: true @@ -616,40 +694,6 @@ processors: field: _temp.isDriver value: true - # CSPM fields - # Can be both string and int, fields are mapped as keyword. - - convert: - tag: convert_crowdstrike_service_to_string_2a8687e8 - field: crowdstrike.service - type: string - ignore_missing: true - - convert: - tag: convert_crowdstrike_cloudplatform_to_string_1f6df19b - field: crowdstrike.cloudplatform - type: string - ignore_missing: true - - - rename: - description: Rename crowdstrike.resource in case concrete value field is mapped as object. - tag: rename_crowdstrike_resource_to_crowdstrike_resource_name_9899a394 - if: ctx.crowdstrike?.resource instanceof String - field: crowdstrike.resource - target_field: crowdstrike.resource_name - - remove: - tag: remove_cloud_4d018ef3 - field: - - cloud - ignore_missing: true - - - pipeline: - tag: pipeline_cspm_iom_62dcd2f9 - if: (ctx.crowdstrike?.disposition != null && ctx.crowdstrike.disposition.equalsIgnoreCase('Failed')) ||(ctx.crowdstrike?.event_simpleName != null && ctx.crowdstrike.event_simpleName.equalsIgnoreCase('CloudSecurityIOMEvaluation')) - name: '{{ IngestPipeline "cspm_iom" }}' - - pipeline: - tag: pipeline_cspm_ioa_8a985b05 - if: ctx.crowdstrike?.vertex_type != null && ctx.crowdstrike.vertex_type.equalsIgnoreCase('ioa') - name: '{{ IngestPipeline "cspm_ioa" }}' - # Event fields. - set: description: Concat the fields used in fingerprint. diff --git a/packages/crowdstrike/docs/README.md b/packages/crowdstrike/docs/README.md index 7609bd345d7..fd14a824815 100644 --- a/packages/crowdstrike/docs/README.md +++ b/packages/crowdstrike/docs/README.md @@ -315,6 +315,8 @@ To resolve this, adjust the `Batch Size` setting in the integration to reduce th The option `Enable Data Deduplication` allows you to avoid consuming duplicate events. By default, this option is set to `false`, and so duplicate events can be ingested. When this option is enabled, a [fingerprint processor](https://www.elastic.co/guide/en/elasticsearch/reference/current/fingerprint-processor.html) is used to calculate a hash from a set of CrowdStrike fields that uniquely identify the event. The hash is assigned to the Elasticsearch [`_id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-id-field.html) field that makes the document unique and prevent duplicates. +The fingerprint includes `@timestamp`, CrowdStrike `id`/`aid`/`cid`, and the FDR object type (`aidmaster`, `userinfo`, or `data`), derived from `log.file.path` or `aws.s3.object.key`. When present, `rule.id` is also included — for Cloud Security (CSPM) findings and for other event types that map a rule id (for example EPP detection summary, FIM rule matched, and Data Protection detection summary). For Cloud Security findings, a resource identifier is included as well so distinct findings that share a timestamp and customer id stay unique. + If duplicate events are ingested, to help find them, the integration's `event.id` field is populated by concatenating a few CrowdStrike fields that uniquely identify the event. These fields are `id`, `aid`, and `cid` from the CrowdStrike event. The fields are separated with pipe `|`. For example, if your CrowdStrike event contains `id: 123`, `aid: 456`, and `cid: 789` then the `event.id` would be `123|456|789`. diff --git a/packages/crowdstrike/manifest.yml b/packages/crowdstrike/manifest.yml index d94feaddba1..00df3a5f6eb 100644 --- a/packages/crowdstrike/manifest.yml +++ b/packages/crowdstrike/manifest.yml @@ -1,6 +1,6 @@ name: crowdstrike title: CrowdStrike -version: "4.4.1" +version: "4.4.2" description: Collect logs from Crowdstrike with Elastic Agent. type: integration format_version: "3.4.0"