Skip to content

Commit 69c17ee

Browse files
efd6efd6
committed
o365: improve handling of o365.audit.OperationProperties
In some cases, this field may be a string. This results in a mapping failure. So in cases where the field is a string, conditionally parse out the JSON.
1 parent 618de99 commit 69c17ee

File tree

7 files changed

+192
-1
lines changed

7 files changed

+192
-1
lines changed

‎packages/o365/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "2.3.2"
3+
changes:
4+
- description: Improve handling of o365.audit.OperationProperties.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/9493
27
- version: "2.3.1"
38
changes:
49
- description: Check contentExpiration timestamp before requesting contents.

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"events": [
3+
{
4+
"event": {
5+
"original": "{\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[{\\\"Name\\\":\\\"SystemArtifactType\\\",\\\"Value\\\":\\\"None\\\"}]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\"}"
6+
},
7+
"o365audit": {
8+
"Activity": "CreateArtifact",
9+
"WorkspaceName": "obszar_robaczy",
10+
"OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
11+
"Operation": "CreateArtifact",
12+
"Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
13+
"CreationTime": "2024-01-30T14:23:40",
14+
"Timestamp": "2024-01-30T14:22:50",
15+
"UserId": "[email protected]",
16+
"ClientIP": "81.2.69.144",
17+
"RecordType": 20,
18+
"ResultStatus": "InProgress",
19+
"ObjectDisplayName": "test_lakehouse",
20+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
21+
"Experience": "Lakehouse",
22+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
23+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
24+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
25+
"Workload": "PowerBI",
26+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
27+
"OperationProperties": "[{\"Name\":\"SystemArtifactType\",\"Value\":\"None\"}]",
28+
"ObjectType": "Lakehouse",
29+
"UserType": 0,
30+
"UserKey": "xxxxxxxx"
31+
}
32+
}
33+
]
34+
}

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-stringly-json-events.json-expected.json‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2024-01-30T14:23:40.000Z",
5+
"client": {
6+
"address": "81.2.69.144",
7+
"ip": "81.2.69.144"
8+
},
9+
"ecs": {
10+
"version": "8.11.0"
11+
},
12+
"event": {
13+
"action": "CreateArtifact",
14+
"category": [
15+
"web"
16+
],
17+
"code": "PowerBIAudit",
18+
"id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
19+
"kind": "event",
20+
"original": "{\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[{\\\"Name\\\":\\\"SystemArtifactType\\\",\\\"Value\\\":\\\"None\\\"}]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\"}",
21+
"outcome": "success",
22+
"provider": "PowerBI",
23+
"type": [
24+
"info"
25+
]
26+
},
27+
"host": {
28+
"id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
29+
"name": "domain.pl"
30+
},
31+
"network": {
32+
"type": "ipv4"
33+
},
34+
"o365": {
35+
"audit": {
36+
"Activity": "CreateArtifact",
37+
"CreationTime": "2024-01-30T14:23:40",
38+
"Experience": "Lakehouse",
39+
"ObjectDisplayName": "test_lakehouse",
40+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
41+
"ObjectType": "Lakehouse",
42+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
43+
"OperationProperties": [
44+
{
45+
"Name": "SystemArtifactType",
46+
"Value": "None"
47+
}
48+
],
49+
"RecordType": "20",
50+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
51+
"ResultStatus": "InProgress",
52+
"Timestamp": "2024-01-30T14:22:50",
53+
"UserId": "[email protected]",
54+
"UserKey": "xxxxxxxx",
55+
"UserType": "0",
56+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
57+
"WorkspaceName": "obszar_robaczy"
58+
}
59+
},
60+
"organization": {
61+
"id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81"
62+
},
63+
"related": {
64+
"ip": [
65+
"81.2.69.144"
66+
],
67+
"user": [
68+
"username"
69+
]
70+
},
71+
"source": {
72+
"geo": {
73+
"city_name": "London",
74+
"continent_name": "Europe",
75+
"country_iso_code": "GB",
76+
"country_name": "United Kingdom",
77+
"location": {
78+
"lat": 51.5142,
79+
"lon": -0.0931
80+
},
81+
"region_iso_code": "GB-ENG",
82+
"region_name": "England"
83+
},
84+
"ip": "81.2.69.144"
85+
},
86+
"tags": [
87+
"preserve_original_event"
88+
],
89+
"user": {
90+
"domain": "domain.pl",
91+
"email": "[email protected]",
92+
"id": "[email protected]",
93+
"name": "username"
94+
},
95+
"user_agent": {
96+
"device": {
97+
"name": "Other"
98+
},
99+
"name": "Chrome",
100+
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
101+
"os": {
102+
"full": "Windows 10",
103+
"name": "Windows",
104+
"version": "10"
105+
},
106+
"version": "120.0.0.0"
107+
}
108+
}
109+
]
110+
}

‎packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,16 @@ processors:
8080
field: o365audit.OrganizationId
8181
target_field: organization.id
8282
ignore_missing: true
83+
- json:
84+
tag: json-extract-stringly-OperationProperties
85+
field: o365audit.OperationProperties
86+
if: ctx.o365audit?.OperationProperties instanceof String
87+
on_failure:
88+
- remove:
89+
field: o365audit.OperationProperties
90+
- append:
91+
field: error.message
92+
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
8393
- rename:
8494
field: o365audit.UserAgent
8595
target_field: user_agent.original

‎packages/o365/data_stream/audit/fields/fields.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
- name: o365.audit
22
type: group
33
fields:
4+
- name: Activity
5+
type: keyword
46
- name: Actor
57
type: group
68
fields:
@@ -205,6 +207,8 @@
205207
# not expressible here; object_type_mapping_type cannot be 'boolean'.
206208
object_type: keyword
207209
object_type_mapping_type: '*'
210+
- name: Experience
211+
type: keyword
208212
- name: ExtendedProperties.*
209213
type: object
210214
object_type: keyword
@@ -269,10 +273,20 @@
269273
type: keyword
270274
- name: NewValue
271275
type: keyword
276+
- name: ObjectDisplayName
277+
type: keyword
272278
- name: ObjectId
273279
type: keyword
280+
- name: ObjectType
281+
type: keyword
274282
- name: Operation
275283
type: keyword
284+
- name: OperationId
285+
type: keyword
286+
- name: OperationProperties
287+
type: object
288+
object_type: keyword
289+
object_type_mapping_type: '*'
276290
- name: OrganizationId
277291
type: keyword
278292
- name: OrganizationName
@@ -293,6 +307,8 @@
293307
type: keyword
294308
- name: ResultStatus
295309
type: keyword
310+
- name: RequestId
311+
type: keyword
296312
- name: SensitiveInfoDetectionIsIncluded
297313
type: boolean
298314
- name: SharePointMetaData.*
@@ -337,6 +353,8 @@
337353
type: keyword
338354
- name: TeamGuid
339355
type: keyword
356+
- name: Timestamp
357+
type: keyword
340358
- name: UniqueSharingId
341359
type: keyword
342360
- name: UserAgent
@@ -353,5 +371,9 @@
353371
type: keyword
354372
- name: Workload
355373
type: keyword
374+
- name: WorkspaceId
375+
type: keyword
376+
- name: WorkspaceName
377+
type: keyword
356378
- name: YammerNetworkId
357379
type: keyword

‎packages/o365/docs/README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ An example event for `audit` looks as following:
245245
| log.offset | Offset of the entry in the log file. | long |
246246
| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
247247
| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword |
248+
| o365.audit.Activity | | keyword |
248249
| o365.audit.Actor.ID | | keyword |
249250
| o365.audit.Actor.Type | | keyword |
250251
| o365.audit.ActorContextId | | keyword |
@@ -338,6 +339,7 @@ An example event for `audit` looks as following:
338339
| o365.audit.EventSource | | keyword |
339340
| o365.audit.ExceptionInfo.\* | | object |
340341
| o365.audit.ExchangeMetaData.\* | | object |
342+
| o365.audit.Experience | | keyword |
341343
| o365.audit.ExtendedProperties.\* | | object |
342344
| o365.audit.ExternalAccess | | boolean |
343345
| o365.audit.FileSizeBytes | | long |
@@ -366,8 +368,12 @@ An example event for `audit` looks as following:
366368
| o365.audit.ModifiedProperties.\*.\* | | object |
367369
| o365.audit.Name | | keyword |
368370
| o365.audit.NewValue | | keyword |
371+
| o365.audit.ObjectDisplayName | | keyword |
369372
| o365.audit.ObjectId | | keyword |
373+
| o365.audit.ObjectType | | keyword |
370374
| o365.audit.Operation | | keyword |
375+
| o365.audit.OperationId | | keyword |
376+
| o365.audit.OperationProperties | | object |
371377
| o365.audit.OrganizationId | | keyword |
372378
| o365.audit.OrganizationName | | keyword |
373379
| o365.audit.OriginatingServer | | keyword |
@@ -376,6 +382,7 @@ An example event for `audit` looks as following:
376382
| o365.audit.PolicyDetails | | flattened |
377383
| o365.audit.PolicyId | | keyword |
378384
| o365.audit.RecordType | | keyword |
385+
| o365.audit.RequestId | | keyword |
379386
| o365.audit.ResultStatus | | keyword |
380387
| o365.audit.SensitiveInfoDetectionIsIncluded | | boolean |
381388
| o365.audit.SessionId | | keyword |
@@ -396,6 +403,7 @@ An example event for `audit` looks as following:
396403
| o365.audit.TargetUserOrGroupType | | keyword |
397404
| o365.audit.TeamGuid | | keyword |
398405
| o365.audit.TeamName | | keyword |
406+
| o365.audit.Timestamp | | keyword |
399407
| o365.audit.UniqueSharingId | | keyword |
400408
| o365.audit.UserAgent | | keyword |
401409
| o365.audit.UserId | | keyword |
@@ -404,6 +412,8 @@ An example event for `audit` looks as following:
404412
| o365.audit.Version | | keyword |
405413
| o365.audit.WebId | | keyword |
406414
| o365.audit.Workload | | keyword |
415+
| o365.audit.WorkspaceId | | keyword |
416+
| o365.audit.WorkspaceName | | keyword |
407417
| o365.audit.YammerNetworkId | | keyword |
408418
| organization.id | Unique identifier for the organization. | keyword |
409419
| organization.name | Organization name. | keyword |

‎packages/o365/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: o365
22
title: Microsoft 365
3-
version: "2.3.1"
3+
version: "2.3.2"
44
description: Collect logs from Microsoft 365 with Elastic Agent.
55
type: integration
66
format_version: "3.0.2"

0 commit comments

Comments
 (0)