diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 1b452d0efdd..7ad70646504 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.37.2" + changes: + - description: Use `properties.createdDateTime` as the event timestamp for SignInLogs when available, falling back to `time`. For sign-in logs, `createdDateTime` is the actual event time, while `time` is a processing timestamp that is typically 1-2 minutes later, so events may now appear slightly earlier than before. + type: bugfix + link: https://github.com/elastic/integrations/pull/20287 - version: "1.37.1" changes: - description: Reduce degraded documents in azure.auditlogs by raising the keyword `ignore_above` limit to `8191` on `result_description`, `result_reason`, `additional_details.value`, and `modified_properties` leaf fields; fix the ingest pipeline to normalize the top-level `resultDescription` field to snake_case so it is correctly mapped; declare the missing `properties.result_description` field mapping as well. diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-alternative-time-and-duration-as-string.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-alternative-time-and-duration-as-string.log-expected.json index 0857261f2c7..430fe2ba789 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-alternative-time-and-duration-as-string.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-alternative-time-and-duration-as-string.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2025-11-14T01:48:53.000Z", + "@timestamp": "2025-11-14T01:46:16.428Z", "azure": { "correlation_id": "11111111-1111-1111-1111-111111111111", "resource": { diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log new file mode 100644 index 00000000000..a4e5f3f0010 --- /dev/null +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log @@ -0,0 +1,2 @@ +{"time": "2026-07-10T20:03:31.2737420Z", "resourceId": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", "operationName": "Sign-in activity", "operationVersion": "1.0", "category": "SignInLogs", "tenantId": "4bbb79f7-5724-4c9e-95f3-de075f6ec090", "resultType": "0", "resultSignature": "None", "durationMs": 0, "callerIpAddress": "0.0.0.0", "correlationId": "aaaa1111-0000-0000-0000-000000000001", "identity": "Test User", "Level": 4, "location": "US", "properties": {"id": "aaaa1111-0000-0000-0000-000000000002", "createdDateTime": "2026-07-10T20:01:51.9248761+00:00", "userDisplayName": "Test User", "userPrincipalName": "testuser@example.com", "userId": "aaaa1111-0000-0000-0000-000000000003", "appId": "aaaa1111-0000-0000-0000-000000000004", "appDisplayName": "Test App", "ipAddress": "0.0.0.0", "status": {"errorCode": 0}, "clientAppUsed": "Browser", "isInteractive": true}} +{"time": "2026-07-10T20:03:31.2737420Z", "resourceId": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", "operationName": "Sign-in activity", "operationVersion": "1.0", "category": "SignInLogs", "tenantId": "4bbb79f7-5724-4c9e-95f3-de075f6ec090", "resultType": "0", "resultSignature": "None", "durationMs": 0, "callerIpAddress": "0.0.0.0", "correlationId": "bbbb2222-0000-0000-0000-000000000001", "identity": "Test User 2", "Level": 4, "location": "US", "properties": {"id": "bbbb2222-0000-0000-0000-000000000002", "userDisplayName": "Test User 2", "userPrincipalName": "testuser2@example.com", "userId": "bbbb2222-0000-0000-0000-000000000003", "appId": "bbbb2222-0000-0000-0000-000000000004", "appDisplayName": "Test App", "ipAddress": "0.0.0.0", "status": {"errorCode": 0}, "clientAppUsed": "Browser", "isInteractive": true}} diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log-expected.json new file mode 100644 index 00000000000..dd70bc7e7d1 --- /dev/null +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-created-datetime-timestamp.log-expected.json @@ -0,0 +1,183 @@ +{ + "expected": [ + { + "@timestamp": "2026-07-10T20:01:51.924Z", + "azure": { + "correlation_id": "aaaa1111-0000-0000-0000-000000000001", + "resource": { + "id": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", + "provider": "Microsoft.aadiam" + }, + "signinlogs": { + "caller_ip_address": "0.0.0.0", + "category": "SignInLogs", + "identity": "Test User", + "operation_name": "Sign-in activity", + "operation_version": "1.0", + "properties": { + "app_display_name": "Test App", + "app_id": "aaaa1111-0000-0000-0000-000000000004", + "client_app_used": "Browser", + "created_at": "2026-07-10T20:01:51.9248761+00:00", + "id": "aaaa1111-0000-0000-0000-000000000002", + "is_interactive": true, + "status": { + "error_code": 0 + }, + "user_display_name": "Test User", + "user_id": "aaaa1111-0000-0000-0000-000000000003", + "user_principal_name": "testuser@example.com" + }, + "result_signature": "None", + "result_type": "0" + }, + "tenant_id": "4bbb79f7-5724-4c9e-95f3-de075f6ec090" + }, + "client": { + "ip": "0.0.0.0" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Sign-in activity", + "category": [ + "authentication" + ], + "duration": 0, + "id": "aaaa1111-0000-0000-0000-000000000002", + "kind": "event", + "original": "{\"time\": \"2026-07-10T20:03:31.2737420Z\", \"resourceId\": \"/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam\", \"operationName\": \"Sign-in activity\", \"operationVersion\": \"1.0\", \"category\": \"SignInLogs\", \"tenantId\": \"4bbb79f7-5724-4c9e-95f3-de075f6ec090\", \"resultType\": \"0\", \"resultSignature\": \"None\", \"durationMs\": 0, \"callerIpAddress\": \"0.0.0.0\", \"correlationId\": \"aaaa1111-0000-0000-0000-000000000001\", \"identity\": \"Test User\", \"Level\": 4, \"location\": \"US\", \"properties\": {\"id\": \"aaaa1111-0000-0000-0000-000000000002\", \"createdDateTime\": \"2026-07-10T20:01:51.9248761+00:00\", \"userDisplayName\": \"Test User\", \"userPrincipalName\": \"testuser@example.com\", \"userId\": \"aaaa1111-0000-0000-0000-000000000003\", \"appId\": \"aaaa1111-0000-0000-0000-000000000004\", \"appDisplayName\": \"Test App\", \"ipAddress\": \"0.0.0.0\", \"status\": {\"errorCode\": 0}, \"clientAppUsed\": \"Browser\", \"isInteractive\": true}}", + "outcome": "success", + "type": [ + "info" + ] + }, + "geo": { + "country_iso_code": "US" + }, + "log": { + "level": "4" + }, + "related": { + "entity": [ + "aaaa1111-0000-0000-0000-000000000004", + "aaaa1111-0000-0000-0000-000000000003" + ], + "ip": [ + "0.0.0.0" + ], + "user": [ + "aaaa1111-0000-0000-0000-000000000003", + "testuser@example.com", + "Test User" + ] + }, + "source": { + "address": "0.0.0.0", + "ip": "0.0.0.0" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "email": "testuser@example.com", + "full_name": "Test User", + "id": "aaaa1111-0000-0000-0000-000000000003", + "name": "testuser@example.com" + } + }, + { + "@timestamp": "2026-07-10T20:03:31.273Z", + "azure": { + "correlation_id": "bbbb2222-0000-0000-0000-000000000001", + "resource": { + "id": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", + "provider": "Microsoft.aadiam" + }, + "signinlogs": { + "caller_ip_address": "0.0.0.0", + "category": "SignInLogs", + "identity": "Test User 2", + "operation_name": "Sign-in activity", + "operation_version": "1.0", + "properties": { + "app_display_name": "Test App", + "app_id": "bbbb2222-0000-0000-0000-000000000004", + "client_app_used": "Browser", + "id": "bbbb2222-0000-0000-0000-000000000002", + "is_interactive": true, + "status": { + "error_code": 0 + }, + "user_display_name": "Test User 2", + "user_id": "bbbb2222-0000-0000-0000-000000000003", + "user_principal_name": "testuser2@example.com" + }, + "result_signature": "None", + "result_type": "0" + }, + "tenant_id": "4bbb79f7-5724-4c9e-95f3-de075f6ec090" + }, + "client": { + "ip": "0.0.0.0" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Sign-in activity", + "category": [ + "authentication" + ], + "duration": 0, + "id": "bbbb2222-0000-0000-0000-000000000002", + "kind": "event", + "original": "{\"time\": \"2026-07-10T20:03:31.2737420Z\", \"resourceId\": \"/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam\", \"operationName\": \"Sign-in activity\", \"operationVersion\": \"1.0\", \"category\": \"SignInLogs\", \"tenantId\": \"4bbb79f7-5724-4c9e-95f3-de075f6ec090\", \"resultType\": \"0\", \"resultSignature\": \"None\", \"durationMs\": 0, \"callerIpAddress\": \"0.0.0.0\", \"correlationId\": \"bbbb2222-0000-0000-0000-000000000001\", \"identity\": \"Test User 2\", \"Level\": 4, \"location\": \"US\", \"properties\": {\"id\": \"bbbb2222-0000-0000-0000-000000000002\", \"userDisplayName\": \"Test User 2\", \"userPrincipalName\": \"testuser2@example.com\", \"userId\": \"bbbb2222-0000-0000-0000-000000000003\", \"appId\": \"bbbb2222-0000-0000-0000-000000000004\", \"appDisplayName\": \"Test App\", \"ipAddress\": \"0.0.0.0\", \"status\": {\"errorCode\": 0}, \"clientAppUsed\": \"Browser\", \"isInteractive\": true}}", + "outcome": "success", + "type": [ + "info" + ] + }, + "geo": { + "country_iso_code": "US" + }, + "log": { + "level": "4" + }, + "related": { + "entity": [ + "bbbb2222-0000-0000-0000-000000000004", + "bbbb2222-0000-0000-0000-000000000003" + ], + "ip": [ + "0.0.0.0" + ], + "user": [ + "bbbb2222-0000-0000-0000-000000000003", + "testuser2@example.com", + "Test User 2" + ] + }, + "source": { + "address": "0.0.0.0", + "ip": "0.0.0.0" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "email": "testuser2@example.com", + "full_name": "Test User 2", + "id": "bbbb2222-0000-0000-0000-000000000003", + "name": "testuser2@example.com" + } + } + ] +} diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index 1ec204018b3..96ed7024314 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -51,19 +51,30 @@ processors: description: Drop non-SignInLogs. if: ctx.azure?.signinlogs?.category == null || !ctx.azure.signinlogs.category.endsWith('SignInLogs') # Unfortunately, some Azure services generate logs with - # `created_date_time` field instead of `time` field. + # `created_date_time` at **the root level** instead of `time` + # field. + # + # We need to rename `azure.signinlogs.created_date_time` + # as `azure.signinlogs.time` to apply the correct timestamp + # processing. # - # We need to rename `created_date_time` as `time` to - # apply the correct timestamp processing. - # # Refs; - # - https://github.com/elastic/integrations/issues/15083 + # - https://github.com/elastic/integrations/issues/15083 - rename: - field: azure.signinlogs.created_date_time - target_field: azure.signinlogs.time - ignore_missing: true - if: ctx.azure?.signinlogs?.time == null - description: 'Fallback to handle special cases for log category: use `created_date_time` as `time` if `time` is not already set.' + field: azure.signinlogs.created_date_time + target_field: azure.signinlogs.time + ignore_missing: true + if: ctx.azure?.signinlogs?.time == null + description: 'Fallback to handle special cases for log category: use `created_date_time` as `time` if `time` is not already set.' + # For SignInLogs, `.properties.createdDateTime` is the actual + # sign-in event time; the top-level `time` is a processing + # timestamp that arrives slightly later. + - set: + field: azure.signinlogs.time + copy_from: azure.signinlogs.properties.created_date_time + override: true + ignore_empty_value: true + description: 'Override `time` with the actual sign-in event time when available.' - date: field: azure.signinlogs.time # Order is prioritized by observed frequency for this log @@ -71,7 +82,7 @@ processors: # We expect the first format to be used in 99% of cases. # However, we include the other formats seen in other # categories as a necessary fallback. - formats: + formats: - ISO8601 - "M/d/yyyy h:mm:ss a XXX" - "M/d/yyyy h:mm:ss a" @@ -205,10 +216,10 @@ processors: } else if (details != null) { ctx['message'] = details; }" - - remove: + - remove: field: azure.signinlogs.properties.status.failure_reason ignore_missing: true - - remove: + - remove: field: azure.signinlogs.properties.status.additional_details ignore_missing: true - rename: diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index ef84192596d..3f9e5b38457 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.37.1" +version: "1.37.2" description: This Elastic integration collects logs from Azure type: integration icons: