Skip to content

Commit ebb54c3

Browse files
efd6efd6
committed
o365: allow user configuration of API request batch size
API responses may be very large, so allow users to specify the length of the response interval. Set the default to a reasonably short value to favour a slow flow rate over an OOM for the worse case.
1 parent 2afadab commit ebb54c3

File tree

4 files changed

+47
-33
lines changed

4 files changed

+47
-33
lines changed

‎packages/o365/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.27.0"
3+
changes:
4+
- description: Allow user configuration of API request batch size.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/8413
27
- version: "1.26.0"
38
changes:
49
- description: Preserve 'event.original' from 'o365audit' field.

‎packages/o365/data_stream/audit/agent/stream/cel.yml.hbs‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -77,40 +77,42 @@ program: |
7777
(has(start_subs_resp_body.error) && has(start_subs_resp_body.error.code) && start_subs_resp_body.error.code == "AF20024")
7878
) ?
7979
// When start-subscription API returns success or if already started subscription,
80-
request("GET",
81-
(
82-
state.want_more &&
83-
has(state.cursor) && has(state.cursor.content_types_state_as_list) && (state.cursor.content_types_state_as_list.filter(e, e.content_type == content_type)[0].next_page != "")
84-
) ?
85-
// if NextPageUri exists
86-
state.cursor.content_types_state_as_list.filter(e, e.content_type == content_type)[0].next_page
87-
: (has (state.cursor) && has(state.cursor.content_types_state_as_list)) ?
88-
// if NextPageUri does not exist, but content_type_state_created_at exists in state
89-
state.cursor.content_types_state_as_list.filter(e,
90-
e.content_type == content_type
91-
).as(content_type_state,
92-
content_type_state[0].content_created_at.as(content_type_state_created_at,
93-
// if saved time inside state is more than 7 days old, then change it to 7 days.
94-
content_type_state_created_at.parse_time(time_layout.RFC3339).as(state_created_at,
95-
// The 168h API age limit is expressed as 167h55m to
96-
// prevent API call delay from causing a call to fail.
97-
state_created_at < (now - duration("167h55m")) ?
98-
now - duration("167h55m")
99-
:
100-
state_created_at
101-
).as(state_created_at_calc,
102-
state.base.list_contents_url + content_type + "&PublisherIdentifier={{azure_tenant_id}}"
103-
+ "&startTime=" + string(state_created_at_calc + duration("1s"))
104-
+ "&endTime=" + string((state_created_at_calc + duration("24h")).as(calc_end_time, calc_end_time <= now ? calc_end_time : now))
80+
duration("{{batch_interval}}").as(batch_interval, (batch_interval > duration("24h") ? duration("24h") : batch_interval).as(batch_interval,
81+
request("GET",
82+
(
83+
state.want_more &&
84+
has(state.cursor) && has(state.cursor.content_types_state_as_list) && (state.cursor.content_types_state_as_list.filter(e, e.content_type == content_type)[0].next_page != "")
85+
) ?
86+
// if NextPageUri exists
87+
state.cursor.content_types_state_as_list.filter(e, e.content_type == content_type)[0].next_page
88+
: (has (state.cursor) && has(state.cursor.content_types_state_as_list)) ?
89+
// if NextPageUri does not exist, but content_type_state_created_at exists in state
90+
state.cursor.content_types_state_as_list.filter(e,
91+
e.content_type == content_type
92+
).as(content_type_state,
93+
content_type_state[0].content_created_at.as(content_type_state_created_at,
94+
// if saved time inside state is more than 7 days old, then change it to 7 days.
95+
content_type_state_created_at.parse_time(time_layout.RFC3339).as(state_created_at,
96+
// The 168h API age limit is expressed as 167h55m to
97+
// prevent API call delay from causing a call to fail.
98+
state_created_at < (now - duration("167h55m")) ?
99+
now - duration("167h55m")
100+
:
101+
state_created_at
102+
).as(state_created_at_calc,
103+
state.base.list_contents_url + content_type + "&PublisherIdentifier={{azure_tenant_id}}"
104+
+ "&startTime=" + string(state_created_at_calc + duration("1s"))
105+
+ "&endTime=" + string((state_created_at_calc + batch_interval).as(calc_end_time, calc_end_time <= now ? calc_end_time : now))
106+
)
105107
)
106108
)
107-
)
108-
:
109-
// initial run when no cursor state exists i.e., polling from initial_interval
110-
state.base.list_contents_url + content_type + "&PublisherIdentifier={{azure_tenant_id}}"
111-
+ "&startTime=" + string(now - duration(state.base.list_contents_start_time))
112-
+ "&endTime=" + string((now - duration(state.base.list_contents_start_time) + duration("24h")).as(calc_end_time, calc_end_time <= now ? calc_end_time : now))
113-
).do_request().as(list_contents_resp,
109+
:
110+
// initial run when no cursor state exists i.e., polling from initial_interval
111+
state.base.list_contents_url + content_type + "&PublisherIdentifier={{azure_tenant_id}}"
112+
+ "&startTime=" + string(now - duration(state.base.list_contents_start_time))
113+
+ "&endTime=" + string((now - duration(state.base.list_contents_start_time) + batch_interval).as(calc_end_time, calc_end_time <= now ? calc_end_time : now))
114+
)
115+
)).do_request().as(list_contents_resp,
114116
bytes(list_contents_resp.Body).decode_json().as(list_contents_resp_body,
115117
(
116118
type(list_contents_resp_body) != map && size(list_contents_resp_body) > 0 &&

‎packages/o365/data_stream/audit/manifest.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,13 @@ streams:
6161
show_user: true
6262
required: true
6363
default: 167h55m
64+
- name: batch_interval
65+
type: text
66+
title: Batch Interval
67+
description: Interval for each API request. The default fetches a single hour of events for each request. This value may not be more than 24h. Supports following suffixes - "h" (hour), "m" (minute), "s" (second), "ms" (millisecond), "us" (microsecond), and "ns" (nanosecond)
68+
show_user: true
69+
required: true
70+
default: 1h
6471
- name: resource_ssl
6572
type: yaml
6673
title: Resource SSL Configuration

‎packages/o365/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: o365
22
title: Microsoft 365
3-
version: "1.26.0"
3+
version: "1.27.0"
44
description: Collect logs from Microsoft 365 with Elastic Agent.
55
type: integration
66
format_version: "3.0.0"

0 commit comments

Comments
 (0)