diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 95e340fa7ee..f1841c2fa9e 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.26.0" + changes: + - description: Standardize user fields for identity_protection and signinlogs data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/14085 - version: "1.25.0" changes: - description: Add Grok processor for `AzureFirewallThreatIntelLog` in `azure.firewall_logs`. diff --git a/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json b/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json index 5306d5e65f8..e3733ed9094 100644 --- a/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json +++ b/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json @@ -39,6 +39,12 @@ "duration": 0, "kind": "event", "original": "{\"time\":\"8/22/2022 6:11:52 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"neu\",\"properties\":{\"id\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"joe.danger@mauriziobrancaoutlook.onmicrosoft.com\",\"riskLastUpdatedDateTime\":\"2022-08-22T18:11:52.702Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"medium\",\"isGuest\":false,\"isDeleted\":false,\"isProcessing\":false}}" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joe Danger", + "name": "joe.danger" } }, { @@ -80,7 +86,13 @@ "duration": 0, "kind": "event", "original": "{\"time\":\"9/9/2022 9:59:27 AM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"identity\":\"joel miller\",\"Level\":4,\"location\":\"weu\",\"properties\":{\"id\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"userDisplayName\":\"Joel Miller\",\"userPrincipalName\":\"joel.miller@mauriziobrancaoutlook.onmicrosoft.com\",\"riskLastUpdatedDateTime\":\"2022-09-09T9:59:27.958Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"high\",\"isGuest\":false,\"isDeleted\":true,\"isProcessing\":false}}" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joel.miller@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joel Miller", + "name": "joel.miller" } } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json b/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json index bb870884e9b..47ac02a34ea 100644 --- a/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json +++ b/packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json @@ -80,6 +80,13 @@ } }, "ip": "67.43.156.42" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joe Danger", + "id": "51e26eae-d07b-44e5-bb0b-249f49569a8c", + "name": "joe.danger" } }, { @@ -162,7 +169,14 @@ } }, "ip": "67.43.156.42" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joel.miller@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joel Miller", + "id": "e3b2b242-4ccb-4cf1-9b8b-004cf034a458", + "name": "joel.miller" } } ] -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml index be29c954932..b4ad8ff1e18 100644 --- a/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml @@ -199,16 +199,40 @@ processors: target_field: azure.identityprotection.properties.user_id description: "Unique ID of the user." ignore_missing: true + - remove: + description: Drop user_id field if value is null. + if: ctx?.azure?.identityprotection?.properties?.user_id == null + field: azure.identityprotection.properties.user_id + ignore_missing: true + - set: + field: user.id + copy_from: azure.identityprotection.properties.user_id + ignore_empty_value: true - rename: field: azure.identityprotection.properties.userDisplayName target_field: azure.identityprotection.properties.user_display_name description: "The user principal name (UPN) of the user." ignore_missing: true + - set: + field: user.full_name + copy_from: azure.identityprotection.properties.user_display_name + ignore_empty_value: true - rename: field: azure.identityprotection.properties.userPrincipalName target_field: azure.identityprotection.properties.user_principal_name description: "The user principal name (UPN) of the user." ignore_missing: true + - set: + field: user.email + copy_from: azure.identityprotection.properties.user_principal_name + if: ctx.azure?.identityprotection?.properties?.user_principal_name?.contains('@') == true + - grok: + field: azure.identityprotection.properties.user_principal_name + patterns: + - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' + - '%{GREEDYDATA:user.name}' + ignore_missing: true + ignore_failure: true - rename: field: azure.identityprotection.properties.userType target_field: azure.identityprotection.properties.user_type diff --git a/packages/azure/data_stream/identity_protection/sample_event.json b/packages/azure/data_stream/identity_protection/sample_event.json index 67eec49f614..8414b1a33d9 100644 --- a/packages/azure/data_stream/identity_protection/sample_event.json +++ b/packages/azure/data_stream/identity_protection/sample_event.json @@ -20,6 +20,7 @@ "detection_timing_type": "realtime", "id": "ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47", "ip_address": "67.43.156.42", + "last_updated_datetime": "2022-08-22T18:07:16.894Z", "location": { "city": "Dresden", "countryOrRegion": "DE", @@ -33,7 +34,6 @@ "request_id": "e1b6d9d7-5fc0-4638-ae1a-e0abceb92200", "risk_detail": "none", "risk_event_type": "anonymizedIPAddress", - "risk_last_updated_datetime": "2022-08-22T18:07:16.894Z", "risk_level": "high", "risk_state": "atRisk", "risk_type": "anonymizedIPAddress", @@ -41,7 +41,7 @@ "token_issuer_type": "AzureAD", "user_display_name": "Joe Danger", "user_id": "51e26eae-d07b-44e5-bb0b-249f49569a8c", - "user_principal_name": "joe.danger@contoso.onmicrosoft.com", + "user_principal_name": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", "user_type": "member" }, "result_signature": "None" @@ -61,7 +61,8 @@ "event": { "action": "User Risk Detection", "duration": 0, - "kind": "event" + "kind": "event", + "original": "{\"time\":\"8/22/2022 6:07:16 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"de\",\"properties\":{\"id\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"requestId\":\"e1b6d9d7-5fc0-4638-ae1a-e0abceb92200\",\"correlationId\":\"266133c2-fabb-492f-9ebf-bdf12317b817\",\"riskType\":\"anonymizedIPAddress\",\"riskEventType\":\"anonymizedIPAddress\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"realtime\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Dresden\",\"state\":\"Sachsen\",\"countryOrRegion\":\"DE\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":51.0714,\"longitude\":13.7399}},\"activityDateTime\":\"2022-08-22T18:05:06.133Z\",\"detectedDateTime\":\"2022-08-22T18:05:06.133Z\",\"lastUpdatedDateTime\":\"2022-08-22T18:07:16.894Z\",\"userId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"joe.danger@mauriziobrancaoutlook.onmicrosoft.com\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}" }, "source": { "as": { @@ -77,5 +78,12 @@ } }, "ip": "67.43.156.42" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joe Danger", + "id": "51e26eae-d07b-44e5-bb0b-249f49569a8c", + "name": "joe.danger" } -} \ No newline at end of file +} diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity-sample.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity-sample.log-expected.json index 3a483db9e13..0fb119cbd9c 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity-sample.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity-sample.log-expected.json @@ -80,6 +80,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -164,6 +167,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -248,6 +254,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -332,6 +341,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -416,6 +428,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -500,6 +515,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -584,6 +602,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -668,6 +689,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -752,6 +776,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -836,6 +863,9 @@ "f7c675ad-1c13-4d16-9824-460f095dab88" ] }, + "service": { + "name": "testmigrate" + }, "tags": [ "preserve_original_event" ] @@ -920,6 +950,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -1004,6 +1037,9 @@ "f7c675ad-1c13-4d16-9824-460f095dab88" ] }, + "service": { + "name": "testmigrate" + }, "tags": [ "preserve_original_event" ] @@ -1088,6 +1124,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -1172,6 +1211,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -1256,6 +1298,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -1340,6 +1385,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -1424,6 +1472,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -1508,6 +1559,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -1592,6 +1646,9 @@ "50706135-8a01-4e7a-8912-d03e4a1e7b03" ] }, + "service": { + "name": "omsagent-testmigrate" + }, "tags": [ "preserve_original_event" ] @@ -1676,6 +1733,9 @@ "f7c675ad-1c13-4d16-9824-460f095dab88" ] }, + "service": { + "name": "testmigrate" + }, "tags": [ "preserve_original_event" ] @@ -1760,6 +1820,9 @@ "9d370547-cb61-4753-be23-68534909af90" ] }, + "service": { + "name": "testplatformlogslube" + }, "tags": [ "preserve_original_event" ] @@ -1844,6 +1907,9 @@ "f7c675ad-1c13-4d16-9824-460f095dab88" ] }, + "service": { + "name": "testmigrate" + }, "tags": [ "preserve_original_event" ] @@ -1928,6 +1994,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2012,6 +2081,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -2096,6 +2168,9 @@ "5338db98-8266-400c-ba61-d8efab370100" ] }, + "service": { + "name": "aplatofrmlogstesting" + }, "tags": [ "preserve_original_event" ] @@ -2180,6 +2255,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2264,6 +2342,9 @@ "9e32432b-bc44-4e0b-b624-996109b24ed7" ] }, + "service": { + "name": "omsagent-test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2348,6 +2429,9 @@ "a189002f-5801-4cb4-a1f8-ab78907ec4f9" ] }, + "service": { + "name": "vakunchaloggeneration" + }, "tags": [ "preserve_original_event" ] @@ -2432,6 +2516,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2516,6 +2603,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2600,6 +2690,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2684,6 +2777,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] @@ -2768,6 +2864,9 @@ "2a652c71-4d7b-40e6-b12d-f45ff732d79c" ] }, + "service": { + "name": "test-vidhi-aks" + }, "tags": [ "preserve_original_event" ] diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log-expected.json index 091bc9de429..b6ef6ed68fc 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-managed-identity.log-expected.json @@ -74,6 +74,9 @@ "22222222-864d-4e00-9882-ff649530f186" ] }, + "service": { + "name": "ASC provisioning Dependency agent for Linux" + }, "tags": [ "preserve_original_event" ] diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-sample.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-sample.log-expected.json index 49627ec2f2b..85aa8ac69b8 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-sample.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-sample.log-expected.json @@ -124,6 +124,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -253,6 +254,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -382,6 +384,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -511,6 +514,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -640,6 +644,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -769,6 +774,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -898,6 +904,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1027,6 +1034,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1156,6 +1164,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1285,6 +1294,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1414,6 +1424,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1543,6 +1554,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1672,6 +1684,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1801,6 +1814,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -1930,6 +1944,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-signin.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-signin.log-expected.json index c5588164846..17ee519a774 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-signin.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user-signin.log-expected.json @@ -276,6 +276,7 @@ ], "user": { "domain": "cyberfortgroup.com", + "email": "nikhita.sethi@cyberfortgroup.com", "full_name": "Nikhita Sethi", "id": "da495378-1cbd-450f-997c-5393402e41f8", "name": "nikhita.sethi" diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user.log-expected.json index 24d410c723b..fbe5094c72c 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-non-interactive-user.log-expected.json @@ -230,6 +230,7 @@ ], "user": { "domain": "company.de", + "email": "hello.world@company.de", "full_name": "Hello World", "id": "22222222-473d-4f4e-a526-ff54e71afe84", "name": "hello.world" diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal-signinlogs-sample.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal-signinlogs-sample.log-expected.json index 7688f4ea079..886217473fd 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal-signinlogs-sample.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal-signinlogs-sample.log-expected.json @@ -92,6 +92,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -198,6 +201,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -304,6 +310,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -410,6 +419,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -516,6 +528,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -622,6 +637,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { @@ -728,6 +746,9 @@ "1.128.3.4" ] }, + "service": { + "name": "Terraform-Datadog-CLI" + }, "source": { "address": "1.128.3.4", "as": { diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal.log-expected.json index c8faa05aab1..3ba02fdb23b 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-service-principal.log-expected.json @@ -86,6 +86,9 @@ "81.2.69.144" ] }, + "service": { + "name": "ConfigMgrSvc_22222222-dfb4-4070-ad95-cf1e68280bb0" + }, "source": { "address": "81.2.69.144", "geo": { diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json index 5d12ff997cd..458584dab88 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json @@ -112,6 +112,7 @@ ], "user": { "domain": "elastic.co", + "email": "test@elastic.co", "full_name": "Test LTest", "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "name": "test" diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-sample.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-sample.log-expected.json index 02d28a06005..ad4d0514089 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-sample.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-sample.log-expected.json @@ -135,6 +135,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" @@ -288,6 +289,7 @@ ], "user": { "domain": "outlook.com", + "email": "mpliftrelastic20210901@outlook.com", "full_name": "elastic testing", "id": "2ce85a15-8640-465d-b916-d2eac620a717", "name": "mpliftrelastic20210901" diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index 964141c74ff..18d38ed3726 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -223,6 +223,10 @@ processors: field: event.id value: '{{{azure.signinlogs.properties.id}}}' ignore_empty_value: true + - set: + field: user.email + copy_from: azure.signinlogs.properties.user_principal_name + if: ctx.azure?.signinlogs?.properties?.user_principal_name?.contains('@') == true - grok: field: azure.signinlogs.properties.user_principal_name patterns: @@ -283,7 +287,6 @@ processors: - user_agent: field: user_agent.original ignore_missing: true - - append: field: related.entity value: '{{{ azure.signinlogs.properties.app_id }}}' @@ -294,6 +297,10 @@ processors: value: '{{{ azure.signinlogs.properties.resource_id }}}' allow_duplicates: false if: ctx.azure?.signinlogs?.properties?.resource_id != null && ctx.azure.signinlogs.properties.resource_id != '' + - set: + field: service.name + copy_from: azure.signinlogs.properties.service_principal_name + ignore_empty_value: true - append: field: related.entity value: '{{{ azure.signinlogs.properties.service_principal_id }}}' diff --git a/packages/azure/data_stream/signinlogs/sample_event.json b/packages/azure/data_stream/signinlogs/sample_event.json index 39ba8d5e3e4..8ff3811ebbd 100644 --- a/packages/azure/data_stream/signinlogs/sample_event.json +++ b/packages/azure/data_stream/signinlogs/sample_event.json @@ -7,6 +7,7 @@ "provider": "Microsoft.aadiam" }, "signinlogs": { + "caller_ip_address": "81.2.69.144", "category": "SignInLogs", "identity": "Test LTest", "operation_name": "Sign-in activity", @@ -20,7 +21,6 @@ "created_at": "2019-10-18T04:45:48.0729893-05:00", "device_detail": { "browser": "Chrome 77.0.3865", - "device_id": "", "operating_system": "MacOs" }, "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", @@ -31,11 +31,9 @@ "risk_level_aggregated": "none", "risk_level_during_signin": "none", "risk_state": "none", - "service_principal_id": "", "status": { "error_code": 50140 }, - "token_issuer_name": "", "token_issuer_type": "AzureAD", "user_display_name": "Test LTest", "user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", @@ -48,7 +46,7 @@ "tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53" }, "client": { - "ip": "1.1.1.1" + "ip": "81.2.69.144" }, "cloud": { "provider": "azure" @@ -63,9 +61,8 @@ ], "duration": 0, "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "ingested": "2021-09-14T17:20:47.736433526Z", "kind": "event", - "original": "{\"Level\":\"4\",\"callerIpAddress\":\"1.1.1.1\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"1.1.1.1\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", + "original": "{\"Level\":\"4\",\"callerIpAddress\":\"81.2.69.144\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.2.69.144\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "outcome": "failure", "type": [ "info" @@ -74,47 +71,48 @@ "geo": { "city_name": "Champs-Sur-Marne", "country_iso_code": "FR", - "country_name": "Seine-Et-Marne", "location": { "lat": 48.12341234, "lon": 2.12341234 - } + }, + "region_name": "Seine-Et-Marne" }, "log": { "level": "4" }, "message": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", "related": { + "entity": [ + "8a4de8b5-095c-47d0-a96f-a75130c61d53" + ], "ip": [ - "1.1.1.1" + "81.2.69.144" ] }, "source": { - "address": "1.1.1.1", - "as": { - "number": 13335, - "organization": { - "name": "Cloudflare, Inc." - } - }, + "address": "81.2.69.144", "geo": { - "continent_name": "Oceania", - "country_iso_code": "AU", - "country_name": "Australia", + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", "location": { - "lat": -33.494, - "lon": 143.2104 - } + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" }, - "ip": "1.1.1.1" + "ip": "81.2.69.144" }, "tags": [ "preserve_original_event" ], "user": { "domain": "elastic.co", + "email": "test@elastic.co", "full_name": "Test LTest", "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "name": "test" } -} \ No newline at end of file +} diff --git a/packages/azure/docs/adlogs.md b/packages/azure/docs/adlogs.md index b77413f62a0..1febc60247d 100644 --- a/packages/azure/docs/adlogs.md +++ b/packages/azure/docs/adlogs.md @@ -96,6 +96,7 @@ An example event for `signinlogs` looks as following: "provider": "Microsoft.aadiam" }, "signinlogs": { + "caller_ip_address": "81.2.69.144", "category": "SignInLogs", "identity": "Test LTest", "operation_name": "Sign-in activity", @@ -109,7 +110,6 @@ An example event for `signinlogs` looks as following: "created_at": "2019-10-18T04:45:48.0729893-05:00", "device_detail": { "browser": "Chrome 77.0.3865", - "device_id": "", "operating_system": "MacOs" }, "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", @@ -120,11 +120,9 @@ An example event for `signinlogs` looks as following: "risk_level_aggregated": "none", "risk_level_during_signin": "none", "risk_state": "none", - "service_principal_id": "", "status": { "error_code": 50140 }, - "token_issuer_name": "", "token_issuer_type": "AzureAD", "user_display_name": "Test LTest", "user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", @@ -137,7 +135,7 @@ An example event for `signinlogs` looks as following: "tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53" }, "client": { - "ip": "1.1.1.1" + "ip": "81.2.69.144" }, "cloud": { "provider": "azure" @@ -152,9 +150,8 @@ An example event for `signinlogs` looks as following: ], "duration": 0, "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "ingested": "2021-09-14T17:20:47.736433526Z", "kind": "event", - "original": "{\"Level\":\"4\",\"callerIpAddress\":\"1.1.1.1\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"1.1.1.1\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", + "original": "{\"Level\":\"4\",\"callerIpAddress\":\"81.2.69.144\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.2.69.144\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "outcome": "failure", "type": [ "info" @@ -163,45 +160,46 @@ An example event for `signinlogs` looks as following: "geo": { "city_name": "Champs-Sur-Marne", "country_iso_code": "FR", - "country_name": "Seine-Et-Marne", "location": { "lat": 48.12341234, "lon": 2.12341234 - } + }, + "region_name": "Seine-Et-Marne" }, "log": { "level": "4" }, "message": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", "related": { + "entity": [ + "8a4de8b5-095c-47d0-a96f-a75130c61d53" + ], "ip": [ - "1.1.1.1" + "81.2.69.144" ] }, "source": { - "address": "1.1.1.1", - "as": { - "number": 13335, - "organization": { - "name": "Cloudflare, Inc." - } - }, + "address": "81.2.69.144", "geo": { - "continent_name": "Oceania", - "country_iso_code": "AU", - "country_name": "Australia", + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", "location": { - "lat": -33.494, - "lon": 143.2104 - } + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" }, - "ip": "1.1.1.1" + "ip": "81.2.69.144" }, "tags": [ "preserve_original_event" ], "user": { "domain": "elastic.co", + "email": "test@elastic.co", "full_name": "Test LTest", "id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "name": "test" @@ -334,6 +332,7 @@ An example event for `identity_protection` looks as following: "detection_timing_type": "realtime", "id": "ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47", "ip_address": "67.43.156.42", + "last_updated_datetime": "2022-08-22T18:07:16.894Z", "location": { "city": "Dresden", "countryOrRegion": "DE", @@ -347,7 +346,6 @@ An example event for `identity_protection` looks as following: "request_id": "e1b6d9d7-5fc0-4638-ae1a-e0abceb92200", "risk_detail": "none", "risk_event_type": "anonymizedIPAddress", - "risk_last_updated_datetime": "2022-08-22T18:07:16.894Z", "risk_level": "high", "risk_state": "atRisk", "risk_type": "anonymizedIPAddress", @@ -355,7 +353,7 @@ An example event for `identity_protection` looks as following: "token_issuer_type": "AzureAD", "user_display_name": "Joe Danger", "user_id": "51e26eae-d07b-44e5-bb0b-249f49569a8c", - "user_principal_name": "joe.danger@contoso.onmicrosoft.com", + "user_principal_name": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", "user_type": "member" }, "result_signature": "None" @@ -375,7 +373,8 @@ An example event for `identity_protection` looks as following: "event": { "action": "User Risk Detection", "duration": 0, - "kind": "event" + "kind": "event", + "original": "{\"time\":\"8/22/2022 6:07:16 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"de\",\"properties\":{\"id\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"requestId\":\"e1b6d9d7-5fc0-4638-ae1a-e0abceb92200\",\"correlationId\":\"266133c2-fabb-492f-9ebf-bdf12317b817\",\"riskType\":\"anonymizedIPAddress\",\"riskEventType\":\"anonymizedIPAddress\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"realtime\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Dresden\",\"state\":\"Sachsen\",\"countryOrRegion\":\"DE\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":51.0714,\"longitude\":13.7399}},\"activityDateTime\":\"2022-08-22T18:05:06.133Z\",\"detectedDateTime\":\"2022-08-22T18:05:06.133Z\",\"lastUpdatedDateTime\":\"2022-08-22T18:07:16.894Z\",\"userId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"joe.danger@mauriziobrancaoutlook.onmicrosoft.com\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}" }, "source": { "as": { @@ -391,6 +390,13 @@ An example event for `identity_protection` looks as following: } }, "ip": "67.43.156.42" + }, + "user": { + "domain": "mauriziobrancaoutlook.onmicrosoft.com", + "email": "joe.danger@mauriziobrancaoutlook.onmicrosoft.com", + "full_name": "Joe Danger", + "id": "51e26eae-d07b-44e5-bb0b-249f49569a8c", + "name": "joe.danger" } } ``` diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 293485bd15e..c2d22df0efb 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.25.0" +version: "1.26.0" description: This Elastic integration collects logs from Azure type: integration icons: diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index 5fef84d79f3..801e7635dfd 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.9.0" + changes: + - description: ECS mapping improvements. + type: enhancement + link: https://github.com/elastic/integrations/pull/14085 - version: "3.8.0" changes: - description: Add vulnerability data stream. diff --git a/packages/m365_defender/data_stream/incident/_dev/test/pipeline/test-incident.log-expected.json b/packages/m365_defender/data_stream/incident/_dev/test/pipeline/test-incident.log-expected.json index ca41238c255..da1d6460931 100644 --- a/packages/m365_defender/data_stream/incident/_dev/test/pipeline/test-incident.log-expected.json +++ b/packages/m365_defender/data_stream/incident/_dev/test/pipeline/test-incident.log-expected.json @@ -778,6 +778,7 @@ "\"Team\" ", "f149b355-542f-4216-bce3-5347cf02a4aa", "user@example.com", + "Jens Luffe", "user", "S-1-5-21-3978388234-3821721435-422805698-27297" ] @@ -1938,6 +1939,7 @@ "transport@net.cl", "Igor Hansen", "6d4c2f48-1d1e-4c8f-8666-914b2085332c", + "Dennis Uber", "user3", "S-1-5-21-3978388234-3821721435-422805698-30771" ] @@ -2110,6 +2112,7 @@ "James <2000@hotmail.com>", "12145719-f684-456e-b8ba-f2f7c67ada56", "user4@example.com", + "Eamon Hess", "user4", "S-1-5-21-3978388234-3821721435-422805698-3556" ] diff --git a/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml index 6b202e74a88..d80cda86072 100644 --- a/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml +++ b/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml @@ -1550,6 +1550,15 @@ processors: field: _ingest._value.userAccount.displayName target_field: _ingest._value.user_account.display_name ignore_missing: true + - foreach: + field: json.alerts.evidence + if: ctx.json?.alerts?.evidence instanceof List + processor: + append: + field: related.user + value: '{{{_ingest._value.user_account.display_name}}}' + allow_duplicates: false + ignore_failure: true - foreach: field: json.alerts.evidence if: ctx.json?.alerts?.evidence instanceof List diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml index 0d07ca87030..f84e34bf77a 100644 --- a/packages/m365_defender/manifest.yml +++ b/packages/m365_defender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: m365_defender title: Microsoft M365 Defender -version: "3.8.0" +version: "3.9.0" description: Collect logs from Microsoft M365 Defender with Elastic Agent. categories: - "security" diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index e7efe21df64..fb2270e7092 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.18.0" + changes: + - description: ECS mapping improvements. + type: enhancement + link: https://github.com/elastic/integrations/pull/14085 - version: "2.17.1" changes: - description: Validate organization field type before accessing subfields. diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 050f67ca4b5..3a35b70ed2b 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -34,6 +34,70 @@ processors: - o365audit.Id target_field: "_id" ignore_missing: true + + # Miscellaneous + + - set: + field: file.extension + copy_from: o365audit.FileExtension + ignore_empty_value: true + + - set: + field: file.hash.sha1 + copy_from: o365audit.Sha1 + ignore_empty_value: true + - set: + field: file.hash.sha256 + copy_from: o365audit.Sha256 + ignore_empty_value: true + + - set: + field: file.path + copy_from: o365audit.FilePath + ignore_empty_value: true + - set: + field: file.path + copy_from: o365audit.TargetFilePath + ignore_empty_value: true + override: false + + - set: + field: file.size + copy_from: o365audit.FileSizeBytes + ignore_empty_value: true + - set: + field: file.size + copy_from: o365audit.FileSize + ignore_empty_value: true + override: false + - gsub: + field: file.size + pattern: '\D+' + replacement: '' + if: ctx.file?.size instanceof String + ignore_missing: true + - convert: + tag: convert_file_size_to_long + field: file.size + type: long + ignore_missing: true + on_failure: + - remove: + field: file.size + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + - set: + field: process.name + copy_from: o365audit.Application + ignore_empty_value: true + + - set: + field: url.domain + copy_from: o365audit.OriginatingDomain + ignore_empty_value: true + # General Schema - date: field: o365audit.CreationTime @@ -263,6 +327,37 @@ processors: - '^-?Identity\s"?%{DATA:o365audit.NetworkMessageId}"?$' ignore_missing: true ignore_failure: true + + - script: + lang: painless + tag: collect_email_to_addresses_from_parameters + source: > + void splitTrimAdd(Set acc, String str) { + if (str != null && str != '') { + String[] parts = str.splitOnToken(';'); + for (int i = 0; i < parts.length; i++) { + acc.add(parts[i].trim()); + } + } + } + + def addressSet = new HashSet(ctx.email?.to?.address ?: []); + + splitTrimAdd(addressSet, ctx.o365audit?.Parameters?.ForwardAsAttachmentTo); + splitTrimAdd(addressSet, ctx.o365audit?.Parameters?.ForwardTo); + splitTrimAdd(addressSet, ctx.o365audit?.Parameters?.RedirectTo); + + if (!addressSet.isEmpty()) { + ctx.email = ctx.email ?: [:]; + ctx.email.to = ctx.email.to ?: [:]; + ctx.email.to.address = addressSet.asList(); + } + + - set: + field: email.from.address + copy_from: o365audit.Parameters.From + ignore_empty_value: true + - script: if: ctx.o365audit?.Platform != null lang: painless @@ -1108,8 +1203,11 @@ processors: def orgid = ctx.organization.id; if (conftenants instanceof Map && conftenants.containsKey(orgid)) { ctx.organization.name = conftenants[orgid]; - ctx.host.name = conftenants[orgid]; } + - set: + field: host.name + copy_from: o365audit.DeviceName + if: ctx.o365audit?.DeviceName != null && ctx.host?.name == null - set: field: host.name copy_from: organization.name @@ -1194,12 +1292,16 @@ processors: if: ctx.o365audit?.P1Sender != null && ctx.o365audit.P1Sender != '' tag: append_email_sender_address_1 allow_duplicates: false - - set: - field: email.to.address - copy_from: o365audit.Recipients + - foreach: + field: o365audit.Recipients if: ctx.o365audit?.Recipients instanceof List && ctx.o365audit.Recipients.length > 0 - tag: set_email_to_address_1 - ignore_empty_value: true + tag: foreach_recipient + processor: + append: + field: email.to.address + value: "{{{_ingest._value}}}" + tag: append_recipient_to_email_to_address + allow_duplicates: false - append: field: related.ip value: "{{{o365audit.SenderIp}}}" diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml index 4e3edc0e088..190e846b129 100644 --- a/packages/o365/data_stream/audit/fields/fields.yml +++ b/packages/o365/data_stream/audit/fields/fields.yml @@ -51,6 +51,8 @@ object_type_mapping_type: '*' - name: AppId type: keyword + - name: Application + type: keyword - name: ApplicationDisplayName type: keyword - name: ApplicationId @@ -247,6 +249,8 @@ type: keyword - name: DetectionType type: keyword + - name: DeviceName + type: keyword - name: Directionality type: keyword - name: EffectiveOrganization @@ -294,6 +298,10 @@ object_type_mapping_type: '*' - name: ExternalAccess type: boolean + - name: FileExtension + type: keyword + - name: FileSize + type: keyword - name: FileSizeBytes type: long - name: FilteringDate @@ -415,6 +423,8 @@ type: keyword - name: OriginalDeliveryLocation type: keyword + - name: OriginatingDomain + type: keyword - name: OriginatingServer type: keyword - name: P1Sender @@ -433,6 +443,8 @@ type: keyword - name: Parameters.Enabled type: keyword + - name: Parameters.From + type: keyword - name: Parameters.ForwardAsAttachmentTo type: keyword - name: Parameters.ForwardTo @@ -496,6 +508,10 @@ type: keyword - name: Severity type: keyword + - name: Sha1 + type: keyword + - name: Sha256 + type: keyword - name: SharePointMetaData.* type: object # This object may contain date formatted fields, but we do not ensure validity, so leave as keyword. @@ -561,6 +577,8 @@ type: keyword - name: TargetContextId type: keyword + - name: TargetFilePath + type: keyword - name: TargetUserOrGroupName type: keyword - name: TargetUserOrGroupType diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md index f90ad9c8c6b..be06c2149c5 100644 --- a/packages/o365/docs/README.md +++ b/packages/o365/docs/README.md @@ -219,6 +219,7 @@ An example event for `audit` looks as following: | o365.audit.AlertType | | keyword | | o365.audit.AppAccessContext.\* | | object | | o365.audit.AppId | | keyword | +| o365.audit.Application | | keyword | | o365.audit.ApplicationDisplayName | | keyword | | o365.audit.ApplicationId | | keyword | | o365.audit.Approver | | keyword | @@ -312,6 +313,7 @@ An example event for `audit` looks as following: | o365.audit.Description | | match_only_text | | o365.audit.DetectionMethod | | keyword | | o365.audit.DetectionType | | keyword | +| o365.audit.DeviceName | | keyword | | o365.audit.Directionality | | keyword | | o365.audit.EffectiveOrganization | | keyword | | o365.audit.EndTimeUtc | | date | @@ -331,6 +333,8 @@ An example event for `audit` looks as following: | o365.audit.ExtendedProperties.\* | | object | | o365.audit.ExtendedProperties.RequestType | | keyword | | o365.audit.ExternalAccess | | boolean | +| o365.audit.FileExtension | | keyword | +| o365.audit.FileSize | | keyword | | o365.audit.FileSizeBytes | | long | | o365.audit.FilteringDate | | date | | o365.audit.GroupName | | keyword | @@ -384,6 +388,7 @@ An example event for `audit` looks as following: | o365.audit.OrganizationId | | keyword | | o365.audit.OrganizationName | | keyword | | o365.audit.OriginalDeliveryLocation | | keyword | +| o365.audit.OriginatingDomain | | keyword | | o365.audit.OriginatingServer | | keyword | | o365.audit.P1Sender | | keyword | | o365.audit.P1SenderDomain | | keyword | @@ -397,6 +402,7 @@ An example event for `audit` looks as following: | o365.audit.Parameters.Enabled | | keyword | | o365.audit.Parameters.ForwardAsAttachmentTo | | keyword | | o365.audit.Parameters.ForwardTo | | keyword | +| o365.audit.Parameters.From | | keyword | | o365.audit.Parameters.RedirectTo | | keyword | | o365.audit.PhishConfidenceLevel | | keyword | | o365.audit.Platform | | keyword | @@ -420,6 +426,8 @@ An example event for `audit` looks as following: | o365.audit.SensitiveInfoDetectionIsIncluded | | boolean | | o365.audit.SessionId | | keyword | | o365.audit.Severity | | keyword | +| o365.audit.Sha1 | | keyword | +| o365.audit.Sha256 | | keyword | | o365.audit.SharePointMetaData.\* | | object | | o365.audit.Site | | keyword | | o365.audit.SiteUrl | | keyword | @@ -448,6 +456,7 @@ An example event for `audit` looks as following: | o365.audit.Target.ID | | keyword | | o365.audit.Target.Type | | keyword | | o365.audit.TargetContextId | | keyword | +| o365.audit.TargetFilePath | | keyword | | o365.audit.TargetUserOrGroupName | | keyword | | o365.audit.TargetUserOrGroupType | | keyword | | o365.audit.TeamGuid | | keyword | diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index 280c249fc86..d24c1630f39 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "2.17.1" +version: "2.18.0" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.2.3"