[azure,o365,m365_defender] ECS mapping improvements

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.26.0"
+  changes:
+    - description: Standardize user fields for identity_protection and signinlogs data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/14085
 - version: "1.25.0"
   changes:
     - description: Add Grok processor for `AzureFirewallThreatIntelLog` in `azure.firewall_logs`.

packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json

@@ -39,6 +39,12 @@
                 "duration": 0,
                 "kind": "event",
                 "original": "{\"time\":\"8/22/2022 6:11:52 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"neu\",\"properties\":{\"id\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-08-22T18:11:52.702Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"medium\",\"isGuest\":false,\"isDeleted\":false,\"isProcessing\":false}}"
+            },
+            "user": {
+                "domain": "mauriziobrancaoutlook.onmicrosoft.com",
+                "email": "[email protected]",
+                "full_name": "Joe Danger",
+                "name": "joe.danger"
             }
         },
         {
@@ -80,7 +86,13 @@
                 "duration": 0,
                 "kind": "event",
                 "original": "{\"time\":\"9/9/2022 9:59:27 AM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"identity\":\"joel miller\",\"Level\":4,\"location\":\"weu\",\"properties\":{\"id\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"userDisplayName\":\"Joel Miller\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-09-09T9:59:27.958Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"high\",\"isGuest\":false,\"isDeleted\":true,\"isProcessing\":false}}"
+            },
+            "user": {
+                "domain": "mauriziobrancaoutlook.onmicrosoft.com",
+                "email": "[email protected]",
+                "full_name": "Joel Miller",
+                "name": "joel.miller"
             }
         }
     ]
-}
+}

packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json

@@ -80,6 +80,13 @@
                     }
                 },
                 "ip": "67.43.156.42"
+            },
+            "user": {
+                "domain": "mauriziobrancaoutlook.onmicrosoft.com",
+                "email": "[email protected]",
+                "full_name": "Joe Danger",
+                "id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
+                "name": "joe.danger"
             }
         },
         {
@@ -162,7 +169,14 @@
                     }
                 },
                 "ip": "67.43.156.42"
+            },
+            "user": {
+                "domain": "mauriziobrancaoutlook.onmicrosoft.com",
+                "email": "[email protected]",
+                "full_name": "Joel Miller",
+                "id": "e3b2b242-4ccb-4cf1-9b8b-004cf034a458",
+                "name": "joel.miller"
             }
         }
     ]
-}
+}

packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml

@@ -199,16 +199,40 @@ processors:
      target_field: azure.identityprotection.properties.user_id
      description: "Unique ID of the user."
      ignore_missing: true
+  - remove:
+     description: Drop user_id field if value is null.
+     if: ctx?.azure?.identityprotection?.properties?.user_id == null
+     field: azure.identityprotection.properties.user_id
+     ignore_missing: true
+  - set:
+     field: user.id
+     copy_from: azure.identityprotection.properties.user_id
+     ignore_empty_value: true
   - rename:
      field: azure.identityprotection.properties.userDisplayName
      target_field: azure.identityprotection.properties.user_display_name
      description: "The user principal name (UPN) of the user."
      ignore_missing: true
+  - set:
+     field: user.full_name
+     copy_from: azure.identityprotection.properties.user_display_name
+     ignore_empty_value: true
   - rename:
      field: azure.identityprotection.properties.userPrincipalName
      target_field: azure.identityprotection.properties.user_principal_name
      description: "The user principal name (UPN) of the user."
      ignore_missing: true
+  - set:
+     field: user.email
+     copy_from: azure.identityprotection.properties.user_principal_name
+     if: ctx.azure?.identityprotection?.properties?.user_principal_name?.contains('@') == true
+  - grok:
+     field: azure.identityprotection.properties.user_principal_name
+     patterns:
+       - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}'
+       - '%{GREEDYDATA:user.name}'
+     ignore_missing: true
+     ignore_failure: true
   - rename:
      field: azure.identityprotection.properties.userType
      target_field: azure.identityprotection.properties.user_type

packages/azure/data_stream/identity_protection/sample_event.json

@@ -20,6 +20,7 @@
                 "detection_timing_type": "realtime",
                 "id": "ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47",
                 "ip_address": "67.43.156.42",
+                "last_updated_datetime": "2022-08-22T18:07:16.894Z",
                 "location": {
                     "city": "Dresden",
                     "countryOrRegion": "DE",
@@ -33,15 +34,14 @@
                 "request_id": "e1b6d9d7-5fc0-4638-ae1a-e0abceb92200",
                 "risk_detail": "none",
                 "risk_event_type": "anonymizedIPAddress",
-                "risk_last_updated_datetime": "2022-08-22T18:07:16.894Z",
                 "risk_level": "high",
                 "risk_state": "atRisk",
                 "risk_type": "anonymizedIPAddress",
                 "source": "IdentityProtection",
                 "token_issuer_type": "AzureAD",
                 "user_display_name": "Joe Danger",
                 "user_id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
-                "user_principal_name": "joe.danger@contoso.onmicrosoft.com",
+                "user_principal_name": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com",
                 "user_type": "member"
             },
             "result_signature": "None"
@@ -61,7 +61,8 @@
     "event": {
         "action": "User Risk Detection",
         "duration": 0,
-        "kind": "event"
+        "kind": "event",
+        "original": "{\"time\":\"8/22/2022 6:07:16 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"de\",\"properties\":{\"id\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"requestId\":\"e1b6d9d7-5fc0-4638-ae1a-e0abceb92200\",\"correlationId\":\"266133c2-fabb-492f-9ebf-bdf12317b817\",\"riskType\":\"anonymizedIPAddress\",\"riskEventType\":\"anonymizedIPAddress\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"realtime\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Dresden\",\"state\":\"Sachsen\",\"countryOrRegion\":\"DE\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":51.0714,\"longitude\":13.7399}},\"activityDateTime\":\"2022-08-22T18:05:06.133Z\",\"detectedDateTime\":\"2022-08-22T18:05:06.133Z\",\"lastUpdatedDateTime\":\"2022-08-22T18:07:16.894Z\",\"userId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}"
     },
     "source": {
         "as": {
@@ -77,5 +78,12 @@
             }
         },
         "ip": "67.43.156.42"
+    },
+    "user": {
+        "domain": "mauriziobrancaoutlook.onmicrosoft.com",
+        "email": "[email protected]",
+        "full_name": "Joe Danger",
+        "id": "51e26eae-d07b-44e5-bb0b-249f49569a8c",
+        "name": "joe.danger"
     }
-}
+}