Skip to content

Commit db2992f

Browse files
gmarouligmarouli
authored
[Failure Store] Expose failure store lifecycle information via the GET data stream API (#126668)
To retrieve the effective configuration you need to use the `GET` data streams API, for example, if a data stream has empty data stream options, it might still have failure store enabled from a cluster setting. The failure store is managed by default with a lifecycle with infinite (for now) retention, so the response will look like this: ``` GET _data_stream/* { "data_streams": [ { "name": "my-data-stream", "timestamp_field": { "name": "@timestamp" }, ..... "failure_store": { "enabled": true, "lifecycle": { "enabled": true }, "rollover_on_write": false, "indices": [ { "index_name": ".fs-my-data-stream-2099.03.08-000003", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "managed_by": "Data stream lifecycle" } ] } },... ] ``` In case there is a failure indexed managed by ILM the failure index info will be displayed as follows. ``` { "index_name": ".fs-my-data-stream-2099.03.08-000002", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "my-lifecycle-policy", "managed_by": "Index Lifecycle Management" } ```
1 parent 00654c8 commit db2992f

File tree

4 files changed

+338
-16
lines changed

4 files changed

+338
-16
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
4545
)
4646
)
4747
);
48+
public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failure_store.lifecycle";
49+
private static final Set<String> CAPABILITIES = Set.of(
50+
DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY,
51+
FAILURES_LIFECYCLE_API_CAPABILITY
52+
);
4853

4954
@Override
5055
public String getName() {
@@ -79,7 +84,7 @@ public boolean allowSystemIndexAccessByDefault() {
7984

8085
@Override
8186
public Set<String> supportedCapabilities() {
82-
return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
87+
return CAPABILITIES;
8388
}
8489

8590
@Override

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java

+68-7
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
3232
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName;
3333
import static org.hamcrest.Matchers.is;
34+
import static org.hamcrest.Matchers.notNullValue;
3435
import static org.hamcrest.Matchers.nullValue;
3536

3637
public class GetDataStreamsResponseTests extends ESTestCase {
@@ -135,13 +136,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
135136
);
136137

137138
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
138-
List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
139-
Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
140-
assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
141-
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
142-
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
139+
List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
140+
Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
141+
assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
142+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
143+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
143144
assertThat(
144-
failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
145+
failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
145146
is(ManagedBy.LIFECYCLE.displayValue)
146147
);
147148
}
@@ -227,14 +228,74 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
227228
List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
228229
Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
229230
assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
230-
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
231+
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
231232
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
232233
assertThat(
233234
failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
234235
is(ManagedBy.UNMANAGED.displayValue)
235236
);
236237
}
237238
}
239+
240+
{
241+
// one failure index that have ILM policy
242+
DataStream logs = DataStream.builder("logs", indices)
243+
.setGeneration(3)
244+
.setAllowCustomRouting(true)
245+
.setIndexMode(IndexMode.STANDARD)
246+
.setLifecycle(DataStreamLifecycle.DEFAULT_DATA_LIFECYCLE)
247+
.setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
248+
.setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
249+
.build();
250+
251+
String ilmPolicyName = "rollover-30days";
252+
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
253+
firstGenerationIndex,
254+
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
255+
secondGenerationIndex,
256+
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null),
257+
writeIndex,
258+
new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE, null),
259+
failureStoreIndex,
260+
new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE, null)
261+
);
262+
263+
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
264+
logs,
265+
true,
266+
ClusterHealthStatus.GREEN,
267+
"index-template",
268+
null,
269+
null,
270+
indexSettingsValues,
271+
false,
272+
null,
273+
null
274+
);
275+
Response response = new Response(List.of(dataStreamInfo));
276+
XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
277+
response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
278+
279+
BytesReference bytes = BytesReference.bytes(contentBuilder);
280+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
281+
Map<String, Object> map = parser.map();
282+
List<Object> dataStreams = (List<Object>) map.get(Response.DATA_STREAMS_FIELD.getPreferredName());
283+
assertThat(dataStreams.size(), is(1));
284+
Map<String, Object> dataStreamMap = (Map<String, Object>) dataStreams.get(0);
285+
assertThat(dataStreamMap.get(DataStream.NAME_FIELD.getPreferredName()), is(dataStreamName));
286+
287+
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
288+
List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
289+
Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
290+
assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
291+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), notNullValue());
292+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(ilmPolicyName));
293+
assertThat(
294+
failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
295+
is(ManagedBy.LIFECYCLE.displayValue)
296+
);
297+
}
298+
}
238299
}
239300

240301
public void testManagedByDisplayValuesDontAccidentalyChange() {

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml

+239
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
setup:
2+
- requires:
3+
test_runner_features: [ capabilities, allowed_warnings ]
4+
reason: "Exposing failures lifecycle config in templates was added in 9.1+"
5+
capabilities:
6+
- method: GET
7+
path: /_data_stream/{target}
8+
capabilities: [ 'failure_store.lifecycle' ]
9+
- do:
10+
allowed_warnings:
11+
- "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
12+
indices.put_index_template:
13+
name: my-template1
14+
body:
15+
index_patterns: [fs-data-stream]
16+
template:
17+
settings:
18+
index.number_of_replicas: 1
19+
mappings:
20+
properties:
21+
'@timestamp':
22+
type: date
23+
count:
24+
type: long
25+
lifecycle: {}
26+
data_stream_options:
27+
failure_store:
28+
enabled: true
29+
data_stream: {}
30+
31+
- do:
32+
allowed_warnings:
33+
- "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
34+
indices.put_index_template:
35+
name: my-template2
36+
body:
37+
index_patterns: [ fs-default-* ]
38+
template:
39+
settings:
40+
index.number_of_replicas: 1
41+
mappings:
42+
properties:
43+
'@timestamp':
44+
type: date
45+
count:
46+
type: long
47+
lifecycle: {}
48+
data_stream: { }
49+
50+
- do:
51+
allowed_warnings:
52+
- "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
53+
indices.put_index_template:
54+
name: my-template3
55+
body:
56+
index_patterns: [ no-fs-* ]
57+
template:
58+
settings:
59+
index.number_of_replicas: 1
60+
mappings:
61+
properties:
62+
'@timestamp':
63+
type: date
64+
count:
65+
type: long
66+
data_stream: { }
67+
68+
- do:
69+
cluster.put_settings:
70+
body:
71+
persistent:
72+
data_streams.failure_store.enabled: 'fs-default*'
73+
74+
---
75+
teardown:
76+
- do:
77+
indices.delete_data_stream:
78+
name: fs-data-stream
79+
ignore: 404
80+
81+
- do:
82+
indices.delete_index_template:
83+
name: fs-default-data-stream
84+
ignore: 404
85+
86+
- do:
87+
indices.delete_index_template:
88+
name: no-fs-data-stream
89+
ignore: 404
90+
91+
---
92+
"Get failure store info from explicitly enabled failure store":
93+
- do:
94+
indices.create_data_stream:
95+
name: fs-data-stream
96+
- is_true: acknowledged
97+
98+
- do:
99+
indices.get_data_stream:
100+
name: "fs-data-stream"
101+
- match: { data_streams.0.name: fs-data-stream }
102+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
103+
- match: { data_streams.0.generation: 1 }
104+
- length: { data_streams.0.indices: 1 }
105+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
106+
- match: { data_streams.0.template: 'my-template1' }
107+
- match: { data_streams.0.failure_store.enabled: true }
108+
- match: { data_streams.0.failure_store.lifecycle.enabled: true}
109+
- match: { data_streams.0.failure_store.indices: [] }
110+
111+
# Initialize failure store
112+
- do:
113+
index:
114+
index: fs-data-stream
115+
refresh: true
116+
body:
117+
'@timestamp': '2020-12-12'
118+
count: 'invalid value'
119+
120+
- do:
121+
indices.get_data_stream:
122+
name: "fs-data-stream"
123+
- match: { data_streams.0.name: fs-data-stream }
124+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
125+
- match: { data_streams.0.generation: 2 }
126+
- length: { data_streams.0.indices: 1 }
127+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
128+
- match: { data_streams.0.template: 'my-template1' }
129+
- match: { data_streams.0.failure_store.enabled: true }
130+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
131+
- length: { data_streams.0.failure_store.indices: 1 }
132+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
133+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
134+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }
135+
136+
---
137+
"Get failure store info from disabled failure store":
138+
- do:
139+
indices.create_data_stream:
140+
name: no-fs-data-stream
141+
- is_true: acknowledged
142+
143+
- do:
144+
indices.get_data_stream:
145+
name: "no-fs-data-stream"
146+
- match: { data_streams.0.name: no-fs-data-stream }
147+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
148+
- match: { data_streams.0.generation: 1 }
149+
- length: { data_streams.0.indices: 1 }
150+
- match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
151+
- match: { data_streams.0.template: 'my-template3' }
152+
- match: { data_streams.0.failure_store.enabled: false }
153+
- is_false: data_streams.0.failure_store.lifecycle
154+
- match: { data_streams.0.failure_store.indices: [] }
155+
156+
---
157+
"Get failure store info from explicitly enabled failure store and disabled lifecycle":
158+
- do:
159+
indices.create_data_stream:
160+
name: fs-data-stream
161+
- is_true: acknowledged
162+
163+
- do:
164+
indices.put_data_lifecycle:
165+
name: "fs-data-stream"
166+
body:
167+
enabled: false
168+
169+
- is_true: acknowledged
170+
171+
# Initialize failure store
172+
- do:
173+
index:
174+
index: fs-data-stream
175+
refresh: true
176+
body:
177+
'@timestamp': '2020-12-12'
178+
count: 'invalid value'
179+
180+
- do:
181+
indices.get_data_stream:
182+
name: "fs-data-stream"
183+
- match: { data_streams.0.name: fs-data-stream }
184+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
185+
- match: { data_streams.0.generation: 2 }
186+
- length: { data_streams.0.indices: 1 }
187+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
188+
- match: { data_streams.0.template: 'my-template1' }
189+
- match: { data_streams.0.failure_store.enabled: true }
190+
- match: { data_streams.0.failure_store.lifecycle.enabled: false }
191+
- length: { data_streams.0.failure_store.indices: 1 }
192+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
193+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
194+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' }
195+
196+
---
197+
"Get failure store info from cluster setting enabled failure store":
198+
- do:
199+
indices.create_data_stream:
200+
name: fs-default-data-stream
201+
- is_true: acknowledged
202+
203+
- do:
204+
indices.get_data_stream:
205+
name: "fs-default-data-stream"
206+
- match: { data_streams.0.name: fs-default-data-stream }
207+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
208+
- match: { data_streams.0.generation: 1 }
209+
- length: { data_streams.0.indices: 1 }
210+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
211+
- match: { data_streams.0.template: 'my-template2' }
212+
- match: { data_streams.0.failure_store.enabled: true }
213+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
214+
- match: { data_streams.0.failure_store.indices: [] }
215+
216+
# Initialize failure store
217+
- do:
218+
index:
219+
index: fs-default-data-stream
220+
refresh: true
221+
body:
222+
'@timestamp': '2020-12-12'
223+
count: 'invalid value'
224+
225+
- do:
226+
indices.get_data_stream:
227+
name: "fs-default-data-stream"
228+
- match: { data_streams.0.name: fs-default-data-stream }
229+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
230+
- match: { data_streams.0.generation: 2 }
231+
- length: { data_streams.0.indices: 1 }
232+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
233+
- match: { data_streams.0.template: 'my-template2' }
234+
- match: { data_streams.0.failure_store.enabled: true }
235+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
236+
- length: { data_streams.0.failure_store.indices: 1 }
237+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
238+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
239+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }

0 commit comments

Comments
 (0)