Skip to content

Commit b991708

Browse files
gmarouligmarouli
authored
Create dedicated factory methods for data lifecycle (elastic#126487)
The class `DataStreamLifecycle` is currently capturing the lifecycle configuration that currently manages all data stream indices, but soon enough it will be split into two variants, the data and the failures lifecycle. Some pre-work has been done already but as we are progressing in our POC, we see that it will be really useful if the `DataStreamLifecycle` is "aware" of the target index component. This will allow us to correctly apply global retention or to throw an error if a downsampling configuration is provided to a failure lifecycle. In this PR, we perform a small refactoring to reduce the noise in elastic#125658. Here we introduce the following: - A factory method that creates a data lifecycle, for now it's trivial but it will be more useful soon. - We rename the "empty" builder to explicitly mention the index component it refers to.
1 parent 1e7c6ab commit b991708

File tree

35 files changed

+225
-164
lines changed

35 files changed

+225
-164
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1384,7 +1384,9 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13841384

13851385
public void testGetDataStream() throws Exception {
13861386
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1387-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
1387+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
1388+
.dataRetention(randomPositiveTimeValue())
1389+
.buildTemplate();
13881390
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13891391
int numDocsFoo = randomIntBetween(2, 16);
13901392
indexDocs("metrics-foo", numDocsFoo);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -401,17 +401,23 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
401401
Type.EXTERNAL,
402402
ComposableIndexTemplate.builder()
403403
.indexPatterns(List.of(".test-failure-store"))
404-
.template(Template.builder().mappings(new CompressedXContent("""
405-
{
406-
"properties": {
407-
"@timestamp" : {
408-
"type": "date"
409-
},
410-
"count": {
411-
"type": "long"
412-
}
413-
}
414-
}""")).dataStreamOptions(new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))))
404+
.template(
405+
Template.builder()
406+
.mappings(new CompressedXContent("""
407+
{
408+
"properties": {
409+
"@timestamp" : {
410+
"type": "date"
411+
},
412+
"count": {
413+
"type": "long"
414+
}
415+
}
416+
}"""))
417+
.dataStreamOptions(
418+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
419+
)
420+
)
415421
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
416422
.build(),
417423
Map.of(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.util.List;
2626

2727
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate;
28-
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate;
28+
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomDataLifecycleTemplate;
2929
import static org.hamcrest.Matchers.equalTo;
3030
import static org.hamcrest.Matchers.is;
3131
import static org.hamcrest.Matchers.notNullValue;
@@ -39,7 +39,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3939
}
4040

4141
public void testGetLifecycle() throws Exception {
42-
DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate();
42+
DataStreamLifecycle.Template lifecycle = randomDataLifecycleTemplate();
4343
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
4444
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
4545
{
@@ -226,7 +226,9 @@ public void testPutLifecycle() throws Exception {
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
230+
.dataRetention(randomPositiveTimeValue())
231+
.buildTemplate();
230232
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231233
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232234
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
207207
Template.builder()
208208
.settings(Settings.EMPTY)
209209
.mappings(mappings)
210-
.lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()))
210+
.lifecycle(DataStreamLifecycle.dataLifecycleBuilder().dataRetention(randomPositiveTimeValue()))
211211
)
212212
.dataStreamTemplate(new DataStreamTemplate())
213213
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ public void testRolloverLifecycle() throws Exception {
168168
}
169169

170170
public void testRolloverAndRetention() throws Exception {
171-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate();
171+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).buildTemplate();
172172

173173
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
174174

@@ -290,7 +290,9 @@ public void testOriginationDate() throws Exception {
290290
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
291291
* origination date to have been deleted, and the one with the newer origination date to remain.
292292
*/
293-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate();
293+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
294+
.dataRetention(TimeValue.timeValueDays(7))
295+
.buildTemplate();
294296

295297
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
296298

@@ -836,7 +838,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
836838

837839
public void testReenableDataStreamLifecycle() throws Exception {
838840
// start with a lifecycle that's not enabled
839-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate();
841+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate();
840842

841843
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
842844
String dataStreamName = "metrics-foo";
@@ -892,7 +894,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
892894

893895
public void testLifecycleAppliedToFailureStore() throws Exception {
894896
// We configure a lifecycle with downsampling to ensure it doesn't fail
895-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder()
897+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
896898
.dataRetention(TimeValue.timeValueSeconds(20))
897899
.downsampling(
898900
List.of(
@@ -1088,7 +1090,8 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
10881090
Template.builder()
10891091
.settings(Settings.EMPTY)
10901092
.lifecycle(
1091-
DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
1093+
DataStreamLifecycle.dataLifecycleBuilder()
1094+
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
10921095
)
10931096
)
10941097
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ public void testExplainFailuresLifecycle() throws Exception {
265265
null,
266266
null,
267267
DataStreamLifecycle.Template.DATA_DEFAULT,
268-
new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))
268+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
269269
);
270270
String dataStreamName = "metrics-foo";
271271
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
@@ -400,7 +400,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
400400
null,
401401
null,
402402
lifecycle,
403-
new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))
403+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
404404
);
405405
String dataStreamName = "metrics-foo";
406406
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
@@ -501,7 +501,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
501501
List.of("metrics-foo*"),
502502
null,
503503
null,
504-
DataStreamLifecycle.builder().enabled(false).buildTemplate()
504+
DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate()
505505
);
506506
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
507507
TEST_REQUEST_TIMEOUT,

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2828
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
2929
import org.elasticsearch.cluster.metadata.DataStreamOptions;
30-
import org.elasticsearch.cluster.metadata.ResettableValue;
3130
import org.elasticsearch.cluster.metadata.Template;
3231
import org.elasticsearch.common.compress.CompressedXContent;
3332
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
@@ -271,11 +270,9 @@ private String createDataStream() throws Exception {
271270

272271
private String createDataStream(boolean hidden, boolean failureStore) throws Exception {
273272
String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault());
274-
ResettableValue<DataStreamOptions.Template> failureStoreOptions = failureStore == false
275-
? ResettableValue.undefined()
276-
: ResettableValue.create(
277-
new DataStreamOptions.Template(ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true))))
278-
);
273+
DataStreamOptions.Template failureStoreOptions = failureStore == false
274+
? null
275+
: new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate());
279276
Template idxTemplate = new Template(null, new CompressedXContent("""
280277
{"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}}
281278
"""), null, null, failureStoreOptions);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,11 @@ public void testLifecycleComposition() {
144144
}
145145
// One lifecycle results to this lifecycle as the final
146146
{
147-
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling());
147+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
148+
true,
149+
randomRetention(),
150+
randomDownsampling()
151+
);
148152
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
149153
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
150154
// Defaults to true
@@ -155,7 +159,11 @@ public void testLifecycleComposition() {
155159
// If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones
156160
// Enabled is always true unless it's explicitly set to false
157161
{
158-
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
162+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
163+
false,
164+
randomPositiveTimeValue(),
165+
randomRounds()
166+
);
159167
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT);
160168
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
161169
assertThat(result.enabled(), equalTo(true));
@@ -164,8 +172,16 @@ public void testLifecycleComposition() {
164172
}
165173
// If both lifecycle have all properties, then the latest one overwrites all the others
166174
{
167-
DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
168-
DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds());
175+
DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate(
176+
false,
177+
randomPositiveTimeValue(),
178+
randomRounds()
179+
);
180+
DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate(
181+
true,
182+
randomPositiveTimeValue(),
183+
randomRounds()
184+
);
169185
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
170186
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
171187
assertThat(result.enabled(), equalTo(lifecycle2.enabled()));

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
153153
.setGeneration(3)
154154
.setAllowCustomRouting(true)
155155
.setIndexMode(IndexMode.STANDARD)
156-
.setLifecycle(new DataStreamLifecycle(false, null, null))
156+
.setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null))
157157
.setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
158158
.setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
159159
.build();

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ static void putComposableIndexTemplate(
128128
assertTrue(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet().isAcknowledged());
129129
}
130130

131-
static DataStreamLifecycle.Template randomLifecycleTemplate() {
132-
return new DataStreamLifecycle.Template(
131+
static DataStreamLifecycle.Template randomDataLifecycleTemplate() {
132+
return DataStreamLifecycle.createDataLifecycleTemplate(
133133
frequently(),
134134
randomResettable(ESTestCase::randomTimeValue),
135135
randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)

0 commit comments

Comments
 (0)