Skip to content

Commit c5b49d5

Browse files
chrisberkhoutchrisberkhout
authored
[azure,o365,m365_defender] ECS mapping improvements (#14085)
Various suggested ECS field mapping updates to improve correlation of ingested events with other security telemetry, for the integrations azure, o365 and m365_defender.
1 parent 31e8b30 commit c5b49d5

27 files changed

+437
-66
lines changed

‎packages/azure/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
- version: "1.26.0"
2+
changes:
3+
- description: Standardize user fields for identity_protection and signinlogs data stream.
4+
type: enhancement
5+
link: https://github.com/elastic/integrations/pull/14085
16
- version: "1.25.0"
27
changes:
38
- description: Add Grok processor for `AzureFirewallThreatIntelLog` in `azure.firewall_logs`.

‎packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@
3939
"duration": 0,
4040
"kind": "event",
4141
"original": "{\"time\":\"8/22/2022 6:11:52 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"neu\",\"properties\":{\"id\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-08-22T18:11:52.702Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"medium\",\"isGuest\":false,\"isDeleted\":false,\"isProcessing\":false}}"
42+
},
43+
"user": {
44+
"domain": "mauriziobrancaoutlook.onmicrosoft.com",
45+
"email": "[email protected]",
46+
"full_name": "Joe Danger",
47+
"name": "joe.danger"
4248
}
4349
},
4450
{
@@ -80,7 +86,13 @@
8086
"duration": 0,
8187
"kind": "event",
8288
"original": "{\"time\":\"9/9/2022 9:59:27 AM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"identity\":\"joel miller\",\"Level\":4,\"location\":\"weu\",\"properties\":{\"id\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"userDisplayName\":\"Joel Miller\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-09-09T9:59:27.958Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"high\",\"isGuest\":false,\"isDeleted\":true,\"isProcessing\":false}}"
89+
},
90+
"user": {
91+
"domain": "mauriziobrancaoutlook.onmicrosoft.com",
92+
"email": "[email protected]",
93+
"full_name": "Joel Miller",
94+
"name": "joel.miller"
8395
}
8496
}
8597
]
86-
}
98+
}

‎packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,13 @@
8080
}
8181
},
8282
"ip": "67.43.156.42"
83+
},
84+
"user": {
85+
"domain": "mauriziobrancaoutlook.onmicrosoft.com",
86+
"email": "[email protected]",
87+
"full_name": "Joe Danger",
88+
"id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
89+
"name": "joe.danger"
8390
}
8491
},
8592
{
@@ -162,7 +169,14 @@
162169
}
163170
},
164171
"ip": "67.43.156.42"
172+
},
173+
"user": {
174+
"domain": "mauriziobrancaoutlook.onmicrosoft.com",
175+
"email": "[email protected]",
176+
"full_name": "Joel Miller",
177+
"id": "e3b2b242-4ccb-4cf1-9b8b-004cf034a458",
178+
"name": "joel.miller"
165179
}
166180
}
167181
]
168-
}
182+
}

‎packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,16 +199,40 @@ processors:
199199
target_field: azure.identityprotection.properties.user_id
200200
description: "Unique ID of the user."
201201
ignore_missing: true
202+
- remove:
203+
description: Drop user_id field if value is null.
204+
if: ctx?.azure?.identityprotection?.properties?.user_id == null
205+
field: azure.identityprotection.properties.user_id
206+
ignore_missing: true
207+
- set:
208+
field: user.id
209+
copy_from: azure.identityprotection.properties.user_id
210+
ignore_empty_value: true
202211
- rename:
203212
field: azure.identityprotection.properties.userDisplayName
204213
target_field: azure.identityprotection.properties.user_display_name
205214
description: "The user principal name (UPN) of the user."
206215
ignore_missing: true
216+
- set:
217+
field: user.full_name
218+
copy_from: azure.identityprotection.properties.user_display_name
219+
ignore_empty_value: true
207220
- rename:
208221
field: azure.identityprotection.properties.userPrincipalName
209222
target_field: azure.identityprotection.properties.user_principal_name
210223
description: "The user principal name (UPN) of the user."
211224
ignore_missing: true
225+
- set:
226+
field: user.email
227+
copy_from: azure.identityprotection.properties.user_principal_name
228+
if: ctx.azure?.identityprotection?.properties?.user_principal_name?.contains('@') == true
229+
- grok:
230+
field: azure.identityprotection.properties.user_principal_name
231+
patterns:
232+
- '%{USERNAME:user.name}@%{HOSTNAME:user.domain}'
233+
- '%{GREEDYDATA:user.name}'
234+
ignore_missing: true
235+
ignore_failure: true
212236
- rename:
213237
field: azure.identityprotection.properties.userType
214238
target_field: azure.identityprotection.properties.user_type

‎packages/azure/data_stream/identity_protection/sample_event.json‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"detection_timing_type": "realtime",
2121
"id": "ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47",
2222
"ip_address": "67.43.156.42",
23+
"last_updated_datetime": "2022-08-22T18:07:16.894Z",
2324
"location": {
2425
"city": "Dresden",
2526
"countryOrRegion": "DE",
@@ -33,15 +34,14 @@
3334
"request_id": "e1b6d9d7-5fc0-4638-ae1a-e0abceb92200",
3435
"risk_detail": "none",
3536
"risk_event_type": "anonymizedIPAddress",
36-
"risk_last_updated_datetime": "2022-08-22T18:07:16.894Z",
3737
"risk_level": "high",
3838
"risk_state": "atRisk",
3939
"risk_type": "anonymizedIPAddress",
4040
"source": "IdentityProtection",
4141
"token_issuer_type": "AzureAD",
4242
"user_display_name": "Joe Danger",
4343
"user_id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
44-
"user_principal_name": "joe.danger@contoso.onmicrosoft.com",
44+
"user_principal_name": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com",
4545
"user_type": "member"
4646
},
4747
"result_signature": "None"
@@ -61,7 +61,8 @@
6161
"event": {
6262
"action": "User Risk Detection",
6363
"duration": 0,
64-
"kind": "event"
64+
"kind": "event",
65+
"original": "{\"time\":\"8/22/2022 6:07:16 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"de\",\"properties\":{\"id\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"requestId\":\"e1b6d9d7-5fc0-4638-ae1a-e0abceb92200\",\"correlationId\":\"266133c2-fabb-492f-9ebf-bdf12317b817\",\"riskType\":\"anonymizedIPAddress\",\"riskEventType\":\"anonymizedIPAddress\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"realtime\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Dresden\",\"state\":\"Sachsen\",\"countryOrRegion\":\"DE\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":51.0714,\"longitude\":13.7399}},\"activityDateTime\":\"2022-08-22T18:05:06.133Z\",\"detectedDateTime\":\"2022-08-22T18:05:06.133Z\",\"lastUpdatedDateTime\":\"2022-08-22T18:07:16.894Z\",\"userId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}"
6566
},
6667
"source": {
6768
"as": {
@@ -77,5 +78,12 @@
7778
}
7879
},
7980
"ip": "67.43.156.42"
81+
},
82+
"user": {
83+
"domain": "mauriziobrancaoutlook.onmicrosoft.com",
84+
"email": "[email protected]",
85+
"full_name": "Joe Danger",
86+
"id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
87+
"name": "joe.danger"
8088
}
81-
}
89+
}

0 commit comments

Comments
 (0)