Manage retention of partial snapshots in SLM (#47833)

Currently, partial snapshots will eventually build up unless they are
manually deleted. Partial snapshots may be useful if there is not a more
recent successful snapshot, but should eventually be deleted if they are
no longer useful.

With this change, partial snapshots are deleted using the following
strategy: PARTIAL snapshots will be kept until the configured
expire_after period has passed, if present, and then be deleted. If
there is no configured expire_after in the retention policy, then they
will be deleted if there is at least one more recent successful snapshot
from this policy (as they may otherwise be useful for troubleshooting
purposes). Partial snapshots are not counted towards either min_count or
max_count.

Author: gwbrown

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfiguration.java

@@ -127,14 +127,15 @@ public Predicate<SnapshotInfo> getSnapshotDeletionPredicate(final List<SnapshotI
             .mapToLong(SnapshotInfo::startTime)
             .max()
             .orElse(Long.MIN_VALUE);
+        final Set<SnapshotState> unsuccessfulStates = Set.of(SnapshotState.FAILED, SnapshotState.PARTIAL);
 
         return si -> {
             final String snapName = si.snapshotId().getName();
 
             // First, if there's no expire_after and a more recent successful snapshot, we can delete all the failed ones
-            if (this.expireAfter == null && SnapshotState.FAILED.equals(si.state()) && newestSuccessfulTimestamp > si.startTime()) {
+            if (this.expireAfter == null && unsuccessfulStates.contains(si.state()) && newestSuccessfulTimestamp > si.startTime()) {
                 // There's no expire_after and there's a more recent successful snapshot, delete this failed one
-                logger.trace("[{}]: ELIGIBLE as it is FAILED and there is a more recent successful snapshot", snapName);
+                logger.trace("[{}]: ELIGIBLE as it is {} and there is a more recent successful snapshot", snapName, si.state());
                 return true;
             }
 
@@ -167,13 +168,13 @@ public Predicate<SnapshotInfo> getSnapshotDeletionPredicate(final List<SnapshotI
             // expiration time
             if (this.minimumSnapshotCount != null) {
                 if (successfulSnapshotCount <= this.minimumSnapshotCount)
-                    if (SnapshotState.FAILED.equals(si.state()) == false) {
+                    if (unsuccessfulStates.contains(si.state()) == false) {
                         logger.trace("[{}]: INELIGIBLE as there are {} non-failed snapshots ({} total) and {} minimum snapshots needed",
                             snapName, successfulSnapshotCount, totalSnapshotCount, this.minimumSnapshotCount);
                         return false;
                     } else {
                         logger.trace("[{}]: SKIPPING minimum snapshot count check as this snapshot is {} and not counted " +
-                            "towards the minimum snapshot count.", snapName, SnapshotState.FAILED);
+                            "towards the minimum snapshot count.", snapName, si.state());
                     }
             }
 
@@ -190,10 +191,11 @@ public Predicate<SnapshotInfo> getSnapshotDeletionPredicate(final List<SnapshotI
                     final Stream<SnapshotInfo> successfulSnapsEligibleForExpiration = sortedSnapshots.stream()
                         .filter(snap -> SnapshotState.SUCCESS.equals(snap.state()))
                         .limit(eligibleForExpiration);
-                    final Stream<SnapshotInfo> failedSnaps = sortedSnapshots.stream()
-                        .filter(snap -> SnapshotState.FAILED.equals(snap.state()));
+                    final Stream<SnapshotInfo> unsucessfulSnaps = sortedSnapshots.stream()
+                        .filter(snap -> unsuccessfulStates.contains(snap.state()));
 
-                    final Set<SnapshotInfo> snapsEligibleForExpiration = Stream.concat(successfulSnapsEligibleForExpiration, failedSnaps)
+                    final Set<SnapshotInfo> snapsEligibleForExpiration = Stream
+                        .concat(successfulSnapsEligibleForExpiration, unsucessfulSnaps)
                         .collect(Collectors.toSet());
 
                     if (snapsEligibleForExpiration.contains(si) == false) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionConfigurationTests.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
+import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
@@ -103,13 +104,21 @@ public void testMaximum() {
     }
 
     public void testFailuresDeletedIfExpired() {
+        assertUnsuccessfulDeletedIfExpired(true);
+    }
+
+    public void testPartialsDeletedIfExpired() {
+        assertUnsuccessfulDeletedIfExpired(false);
+    }
+
+    private void assertUnsuccessfulDeletedIfExpired(boolean failure) {
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(
             () -> TimeValue.timeValueDays(1).millis() + 1,
             TimeValue.timeValueDays(1), null, null);
-        SnapshotInfo oldInfo = makeFailureInfo(0);
+        SnapshotInfo oldInfo = makeFailureOrPartial(0, failure);
         assertThat(conf.getSnapshotDeletionPredicate(Collections.singletonList(oldInfo)).test(oldInfo), equalTo(true));
 
-        SnapshotInfo newInfo = makeFailureInfo(1);
+        SnapshotInfo newInfo = makeFailureOrPartial(1, failure);
         assertThat(conf.getSnapshotDeletionPredicate(Collections.singletonList(newInfo)).test(newInfo), equalTo(false));
 
         List<SnapshotInfo> infos = new ArrayList<>();
@@ -120,10 +129,18 @@ public void testFailuresDeletedIfExpired() {
     }
 
     public void testFailuresDeletedIfNoExpiryAndMoreRecentSuccessExists() {
+        assertUnsuccessfulDeletedIfNoExpiryAndMoreRecentSuccessExists(true);
+    }
+
+    public void testPartialsDeletedIfNoExpiryAndMoreRecentSuccessExists() {
+        assertUnsuccessfulDeletedIfNoExpiryAndMoreRecentSuccessExists(false);
+    }
+
+    private void assertUnsuccessfulDeletedIfNoExpiryAndMoreRecentSuccessExists(boolean failure) {
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(() -> 1, null, 2, 5);
         SnapshotInfo s1 = makeInfo(1);
         SnapshotInfo s2 = makeInfo(2);
-        SnapshotInfo s3 = makeFailureInfo(3);
+        SnapshotInfo s3 = makeFailureOrPartial(3, failure);
         SnapshotInfo s4 = makeInfo(4);
 
         List<SnapshotInfo> infos = Arrays.asList(s1 , s2, s3, s4);
@@ -134,12 +151,20 @@ public void testFailuresDeletedIfNoExpiryAndMoreRecentSuccessExists() {
     }
 
     public void testFailuresKeptIfNoExpiryAndNoMoreRecentSuccess() {
+        assertUnsuccessfulKeptIfNoExpiryAndNoMoreRecentSuccess(true);
+    }
+
+    public void testPartialsKeptIfNoExpiryAndNoMoreRecentSuccess() {
+        assertUnsuccessfulKeptIfNoExpiryAndNoMoreRecentSuccess(false);
+    }
+
+    private void assertUnsuccessfulKeptIfNoExpiryAndNoMoreRecentSuccess(boolean failure) {
         // Also tests that failures are not counted towards the maximum
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(() -> 1, null, 2, 3);
         SnapshotInfo s1 = makeInfo(1);
         SnapshotInfo s2 = makeInfo(2);
         SnapshotInfo s3 = makeInfo(3);
-        SnapshotInfo s4 = makeFailureInfo(4);
+        SnapshotInfo s4 = makeFailureOrPartial(4, failure);
 
         List<SnapshotInfo> infos = Arrays.asList(s1 , s2, s3, s4);
         assertThat(conf.getSnapshotDeletionPredicate(infos).test(s1), equalTo(false));
@@ -149,11 +174,19 @@ public void testFailuresKeptIfNoExpiryAndNoMoreRecentSuccess() {
     }
 
     public void testFailuresNotCountedTowardsMaximum() {
+        assertUnsuccessfulNotCountedTowardsMaximum(true);
+    }
+
+    public void testPartialsNotCountedTowardsMaximum() {
+        assertUnsuccessfulNotCountedTowardsMaximum(false);
+    }
+
+    private void assertUnsuccessfulNotCountedTowardsMaximum(boolean failure) {
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(() -> 1, TimeValue.timeValueDays(1), 2, 2);
         SnapshotInfo s1 = makeInfo(1);
-        SnapshotInfo s2 = makeFailureInfo(2);
-        SnapshotInfo s3 = makeFailureInfo(3);
-        SnapshotInfo s4 = makeFailureInfo(4);
+        SnapshotInfo s2 = makeFailureOrPartial(2, failure);
+        SnapshotInfo s3 = makeFailureOrPartial(3, failure);
+        SnapshotInfo s4 = makeFailureOrPartial(4, failure);
         SnapshotInfo s5 = makeInfo(5);
 
         List<SnapshotInfo> infos = Arrays.asList(s1 , s2, s3, s4, s5);
@@ -165,10 +198,18 @@ public void testFailuresNotCountedTowardsMaximum() {
     }
 
     public void testFailuresNotCountedTowardsMinimum() {
+        assertUnsuccessfulNotCountedTowardsMinimum(true);
+    }
+
+    public void testPartialsNotCountedTowardsMinimum() {
+        assertUnsuccessfulNotCountedTowardsMinimum(false);
+    }
+
+    private void assertUnsuccessfulNotCountedTowardsMinimum(boolean failure) {
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(() -> TimeValue.timeValueDays(1).millis() + 1,
             TimeValue.timeValueDays(1), 2, null);
         SnapshotInfo oldInfo = makeInfo(0);
-        SnapshotInfo failureInfo = makeFailureInfo( 1);
+        SnapshotInfo failureInfo = makeFailureOrPartial(1, failure);
         SnapshotInfo newInfo = makeInfo(2);
 
         List<SnapshotInfo> infos = new ArrayList<>();
@@ -186,12 +227,14 @@ public void testFailuresNotCountedTowardsMinimum() {
         assertThat(conf.getSnapshotDeletionPredicate(infos).test(oldInfo), equalTo(true));
     }
 
+
     public void testMostRecentSuccessfulTimestampIsUsed() {
+        boolean failureBeforePartial = randomBoolean();
         SnapshotRetentionConfiguration conf = new SnapshotRetentionConfiguration(() -> 1, null, 2, 2);
         SnapshotInfo s1 = makeInfo(1);
         SnapshotInfo s2 = makeInfo(2);
-        SnapshotInfo s3 = makeFailureInfo(3);
-        SnapshotInfo s4 = makeFailureInfo(4);
+        SnapshotInfo s3 = makeFailureOrPartial(3, failureBeforePartial);
+        SnapshotInfo s4 = makeFailureOrPartial(4, failureBeforePartial == false);
 
         List<SnapshotInfo> infos = Arrays.asList(s1 , s2, s3, s4);
         assertThat(conf.getSnapshotDeletionPredicate(infos).test(s1), equalTo(false));
@@ -204,15 +247,25 @@ private SnapshotInfo makeInfo(long startTime) {
         final Map<String, Object> meta = new HashMap<>();
         meta.put(SnapshotLifecyclePolicy.POLICY_ID_METADATA_FIELD, REPO);
         final int totalShards = between(1,20);
-        return new SnapshotInfo(new SnapshotId("snap-" + randomAlphaOfLength(3), "uuid"),
+        SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-" + randomAlphaOfLength(3), "uuid"),
             Collections.singletonList("foo"),
             startTime,
             null,
-            startTime + between(1,10000),
+            startTime + between(1, 10000),
             totalShards,
             new ArrayList<>(),
             false,
             meta);
+        assertThat(snapInfo.state(), equalTo(SnapshotState.SUCCESS));
+        return snapInfo;
+    }
+
+    private SnapshotInfo makeFailureOrPartial(long startTime, boolean failure) {
+        if (failure) {
+            return makeFailureInfo(startTime);
+        } else {
+            return makePartialInfo(startTime);
+        }
     }
 
     private SnapshotInfo makeFailureInfo(long startTime) {
@@ -225,14 +278,39 @@ private SnapshotInfo makeFailureInfo(long startTime) {
             failures.add(new SnapshotShardFailure("nodeId", new ShardId("index-name", "index-uuid", i), "failed"));
         }
         assert failureCount == failures.size();
-        return new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"),
+        SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"),
             Collections.singletonList("foo-fail"),
             startTime,
             "forced-failure",
-            startTime + between(1,10000),
+            startTime + between(1, 10000),
+            totalShards,
+            failures,
+            randomBoolean(),
+            meta);
+        assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED));
+        return snapInfo;
+    }
+
+    private SnapshotInfo makePartialInfo(long startTime) {
+        final Map<String, Object> meta = new HashMap<>();
+        meta.put(SnapshotLifecyclePolicy.POLICY_ID_METADATA_FIELD, REPO);
+        final int totalShards = between(2,20);
+        final List<SnapshotShardFailure> failures = new ArrayList<>();
+        final int failureCount = between(1,totalShards - 1);
+        for (int i = 0; i < failureCount; i++) {
+            failures.add(new SnapshotShardFailure("nodeId", new ShardId("index-name", "index-uuid", i), "failed"));
+        }
+        assert failureCount == failures.size();
+        SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"),
+            Collections.singletonList("foo-fail"),
+            startTime,
+            null,
+            startTime + between(1, 10000),
             totalShards,
             failures,
             randomBoolean(),
             meta);
+        assertThat(snapInfo.state(), equalTo(SnapshotState.PARTIAL));
+        return snapInfo;
     }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java

@@ -240,7 +240,7 @@ void getAllRetainableSnapshots(Collection<String> repositories, ActionListener<M
                 @Override
                 public void onResponse(final GetSnapshotsResponse resp) {
                     Map<String, List<SnapshotInfo>> snapshots = new HashMap<>();
-                    final Set<SnapshotState> retainableStates = Set.of(SnapshotState.SUCCESS, SnapshotState.FAILED);
+                    final Set<SnapshotState> retainableStates = Set.of(SnapshotState.SUCCESS, SnapshotState.FAILED, SnapshotState.PARTIAL);
                     repositories.forEach(repo -> {
                         snapshots.put(repo,
                             // Only return snapshots in the SUCCESS state