diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 45f442ae6e9..aa73ed210a0 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.6.5" + changes: + - description: Fix the processing of duplicated QueryTime in Data field. + type: bugfix + link: https://github.com/elastic/integrations/pull/11499 - version: "2.6.4" changes: - description: Remove in-program template snippets and format CEL code. diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json new file mode 100644 index 00000000000..84ecfddb6db --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json @@ -0,0 +1,43 @@ +{ + "events": [ + { + "event": { + "original": "{\"AlertEntityId\":\"asr@testsiem.onmicrosoft.com\",\"AlertId\":\"5ba6e029-8b6e-13bd-b800-08d7b180173c\",\"AlertLinks\":[{\"AlertLinkHref\":\"http://example.net/alert\"},{\"AlertLinkHref\":\"http://example.net/info\"}],\"AlertType\":\"System\",\"Category\":\"AccessGovernance\",\"Comments\":\"New alert\",\"CreationTime\":\"2020-02-14T19:00:00\",\"Data\":\"{\\\"etype\\\":\\\"User\\\",\\\"eid\\\":\\\"asr@testsiem.onmicrosoft.com\\\",\\\"tid\\\":\\\"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd\\\",\\\"ts\\\":\\\"2020-02-14T18:54:45.0000000Z\\\",\\\"te\\\":\\\"2020-02-14T18:54:45.0000000Z\\\",\\\"op\\\":\\\"GrantAdminPermission\\\",\\\"tdc\\\":\\\"1\\\",\\\"suid\\\":\\\"asr@testsiem.onmicrosoft.com\\\",\\\"ut\\\":\\\"Admin\\\",\\\"lon\\\":\\\"GrantAdminPermission\\\",\\\"Entities\\\":[{\\\"QueryTime\\\":\\\"2024-09-30T06:16:41.7320497Z\\\",\\\"QueryTime\":\\\"9/30/2024 6:16:41 AM\\\"}]}\",\"EntityType\":\"User\",\"Id\":\"448854d7-81f6-4a06-d31a-08d7b1c1fb2f\",\"Name\":\"Elevation of Exchange admin privilege\",\"ObjectId\":\"asr@testsiem.onmicrosoft.com\",\"Operation\":\"AlertEntityGenerated\",\"OrganizationId\":\"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd\",\"PolicyId\":\"17d51759-88e1-40c1-8df3-20bcf2e43057\",\"RecordType\":64,\"ResultStatus\":\"Succeeded\",\"Severity\":\"Low\",\"Source\":\"Office 365 Security \\u0026 Compliance\",\"Status\":\"Active\",\"UserId\":\"SecurityComplianceAlerts\",\"UserKey\":\"SecurityComplianceAlerts\",\"UserType\":4,\"Version\":1,\"Workload\":\"SecurityComplianceCenter\"}" + }, + "o365audit": { + "AlertEntityId": "asr@testsiem.onmicrosoft.com", + "AlertId": "5ba6e029-8b6e-13bd-b800-08d7b180173c", + "AlertLinks": [ + { + "AlertLinkHref": "http://example.net/alert" + }, + { + "AlertLinkHref": "http://example.net/info" + } + ], + "AlertType": "System", + "Category": "AccessGovernance", + "Comments": "New alert", + "CreationTime": "2020-02-14T19:00:00", + "Data": "{\"etype\":\"User\",\"eid\":\"asr@testsiem.onmicrosoft.com\",\"tid\":\"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd\",\"ts\":\"2020-02-14T18:54:45.0000000Z\",\"te\":\"2020-02-14T18:54:45.0000000Z\",\"op\":\"GrantAdminPermission\",\"tdc\":\"1\",\"suid\":\"asr@testsiem.onmicrosoft.com\",\"ut\":\"Admin\",\"lon\":\"GrantAdminPermission\",\"Entities\":[{\"QueryTime\":\"2024-09-30T06:16:41.7320497Z\",\"QueryTime\":\"9/30/2024 6:16:41 AM\"}]}", + "EntityType": "User", + "Id": "448854d7-81f6-4a06-d31a-08d7b1c1fb2f", + "Name": "Elevation of Exchange admin privilege", + "ObjectId": "asr@testsiem.onmicrosoft.com", + "Operation": "AlertEntityGenerated", + "OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "PolicyId": "17d51759-88e1-40c1-8df3-20bcf2e43057", + "RecordType": 64, + "ResultStatus": "Succeeded", + "Severity": "Low", + "Source": "Office 365 Security \u0026 Compliance", + "Status": "Active", + "UserId": "SecurityComplianceAlerts", + "UserKey": "SecurityComplianceAlerts", + "UserType": 4, + "Version": 1, + "Workload": "SecurityComplianceCenter" + } + } + ] +} diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json new file mode 100644 index 00000000000..0e9a27d2713 --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-data-duplicated-querytime-events.json-expected.json @@ -0,0 +1,100 @@ +{ + "expected": [ + { + "@timestamp": "2020-02-14T19:00:00.000Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "AlertEntityGenerated", + "category": [ + "web" + ], + "code": "AirInvestigation", + "id": "448854d7-81f6-4a06-d31a-08d7b1c1fb2f", + "kind": "event", + "original": "{\"AlertEntityId\":\"asr@testsiem.onmicrosoft.com\",\"AlertId\":\"5ba6e029-8b6e-13bd-b800-08d7b180173c\",\"AlertLinks\":[{\"AlertLinkHref\":\"http://example.net/alert\"},{\"AlertLinkHref\":\"http://example.net/info\"}],\"AlertType\":\"System\",\"Category\":\"AccessGovernance\",\"Comments\":\"New alert\",\"CreationTime\":\"2020-02-14T19:00:00\",\"Data\":\"{\\\"etype\\\":\\\"User\\\",\\\"eid\\\":\\\"asr@testsiem.onmicrosoft.com\\\",\\\"tid\\\":\\\"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd\\\",\\\"ts\\\":\\\"2020-02-14T18:54:45.0000000Z\\\",\\\"te\\\":\\\"2020-02-14T18:54:45.0000000Z\\\",\\\"op\\\":\\\"GrantAdminPermission\\\",\\\"tdc\\\":\\\"1\\\",\\\"suid\\\":\\\"asr@testsiem.onmicrosoft.com\\\",\\\"ut\\\":\\\"Admin\\\",\\\"lon\\\":\\\"GrantAdminPermission\\\",\\\"Entities\\\":[{\\\"QueryTime\\\":\\\"2024-09-30T06:16:41.7320497Z\\\",\\\"QueryTime\":\\\"9/30/2024 6:16:41 AM\\\"}]}\",\"EntityType\":\"User\",\"Id\":\"448854d7-81f6-4a06-d31a-08d7b1c1fb2f\",\"Name\":\"Elevation of Exchange admin privilege\",\"ObjectId\":\"asr@testsiem.onmicrosoft.com\",\"Operation\":\"AlertEntityGenerated\",\"OrganizationId\":\"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd\",\"PolicyId\":\"17d51759-88e1-40c1-8df3-20bcf2e43057\",\"RecordType\":64,\"ResultStatus\":\"Succeeded\",\"Severity\":\"Low\",\"Source\":\"Office 365 Security \\u0026 Compliance\",\"Status\":\"Active\",\"UserId\":\"SecurityComplianceAlerts\",\"UserKey\":\"SecurityComplianceAlerts\",\"UserType\":4,\"Version\":1,\"Workload\":\"SecurityComplianceCenter\"}", + "outcome": "success", + "provider": "SecurityComplianceCenter", + "type": [ + "info" + ] + }, + "host": { + "id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "name": "mytenant.onmicrosoft.com" + }, + "o365": { + "audit": { + "AlertEntityId": "asr@testsiem.onmicrosoft.com", + "AlertId": "5ba6e029-8b6e-13bd-b800-08d7b180173c", + "AlertLinks": [ + "http://example.net/alert", + "http://example.net/info" + ], + "AlertType": "System", + "Category": "AccessGovernance", + "Comments": "New alert", + "CreationTime": "2020-02-14T19:00:00", + "Data": { + "eid": "asr@testsiem.onmicrosoft.com", + "etype": "User", + "flattened": { + "Entities": [ + { + "QueryTime": "2024-09-30T06:16:41.7320497Z" + } + ], + "eid": "asr@testsiem.onmicrosoft.com", + "etype": "User", + "lon": "GrantAdminPermission", + "op": "GrantAdminPermission", + "suid": "asr@testsiem.onmicrosoft.com", + "tdc": "1", + "te": "2020-02-14T18:54:45.0000000Z", + "tid": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "ts": "2020-02-14T18:54:45.0000000Z", + "ut": "Admin" + }, + "lon": "GrantAdminPermission", + "op": "GrantAdminPermission", + "suid": "asr@testsiem.onmicrosoft.com", + "tdc": "1", + "te": "2020-02-14T18:54:45.000Z", + "tid": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "ts": "2020-02-14T18:54:45.000Z", + "ut": "Admin" + }, + "EntityType": "User", + "Name": "Elevation of Exchange admin privilege", + "ObjectId": "asr@testsiem.onmicrosoft.com", + "PolicyId": "17d51759-88e1-40c1-8df3-20bcf2e43057", + "RecordType": "64", + "ResultStatus": "Succeeded", + "Severity": "Low", + "Source": "Office 365 Security & Compliance", + "Status": "Active", + "UserId": "SecurityComplianceAlerts", + "UserKey": "SecurityComplianceAlerts", + "UserType": "4", + "Version": "1" + } + }, + "organization": { + "id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "name": "mytenant.onmicrosoft.com" + }, + "related": { + "user": [ + "asr@testsiem.onmicrosoft.com" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "SecurityComplianceAlerts" + } + } + ] +} \ No newline at end of file diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index eb5cbb24675..e46409def1b 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1154,9 +1154,20 @@ processors: field: o365audit.YammerNetworkId type: string ignore_missing: true + - gsub: + field: o365audit.Data + pattern: ',\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\"|\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\",' + replacement: "" + if: ctx.o365audit?.containsKey('Data') == true && ctx.o365audit?.RecordType == '64' + tag: gsub_remove_duplicated_querytime - json: field: o365audit.Data if: ctx.o365audit?.containsKey('Data') == true + on_failure: + - remove: + field: o365audit.Data + ignore_missing: true + description: remove_malformed_data - rename: field: o365audit.Data target_field: o365audit.Data.flattened diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index 4b839574f2b..088db0be9f7 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "2.6.4" +version: "2.6.5" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.0.2"