Shortcut total hit count when terminate_after is used (elastic#94889)

We have for long considered terminate_after a "filtered collector",
same as min_score and post_filter, for which we disable the
hit count optimization as we want to make sure that the total hit count
reflects the documents that are in fact going to be collected.

When we moved to leveraging Weight#count from Lucene (elastic#88396), we automatically
moved to shortcutting the total hit count when using terminate_after and
size is set to 0, meaning that in this scenario the total hits may be
higher than the terminate after value that is set to the request, in case the
total hit count is retrieved from the index statistics through Weight#count.

This is acceptable, given that terminate_after is intended to stop
additional work after collecting a certain number of docs, but no
additional work is performed to collect a higher and accurante total hit
count in this case, given it could be retrieved from the index
statistics. This also raises that `EarlyTerminatingCollector` isn't
necessarily to be considered as a collector that filters documents out.

The current situation is inconsistent as we will shortcut the total hit
count when size is 0 and terminate after is set, but we won't if size is
greater than 0.

This commit treats terminate_after the same way regardless of how size
is set: the total hit count may be higher than the terminate after if it
did not require actual counting to be computed.

Author: javanna

docs/changelog/94889.yaml

@@ -0,0 +1,5 @@
+pr: 94889
+summary: Shortcut total hit count when `terminate_after` is used
+area: Search
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/search/query/QueryPhase.java

@@ -132,8 +132,6 @@ static void executeInternal(SearchContext searchContext) throws QueryPhaseExecut
                 // add terminate_after before the filter collectors
                 // it will only be applied on documents accepted by these filter collectors
                 collectors.add(createEarlyTerminationCollectorContext(searchContext.terminateAfter()));
-                // this collector can filter documents during the collection
-                hasFilterCollector = true;
             }
             if (searchContext.parsedPostFilter() != null) {
                 // add post filters before aggregations

server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java

@@ -410,18 +410,19 @@ public void testTerminateAfterEarlyTermination() throws Exception {
         }
 
         context.terminateAfter(1);
+        long countDocs = countDocUpTo.applyAsInt(1);
         {
             context.setSize(1);
             QueryPhase.executeInternal(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(countDocs));
             assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
 
             context.setSize(0);
             QueryPhase.executeInternal(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) countDocUpTo.applyAsInt(1)));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(countDocs));
             assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0));
         }
@@ -430,7 +431,7 @@ public void testTerminateAfterEarlyTermination() throws Exception {
             context.setSize(1);
             QueryPhase.executeInternal(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(countDocs));
             assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
         }
@@ -440,7 +441,7 @@ public void testTerminateAfterEarlyTermination() throws Exception {
             context.registerAggsCollector(collector);
             QueryPhase.executeInternal(context);
             assertTrue(context.queryResult().terminatedEarly());
-            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L));
+            assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(countDocs));
             assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1));
             // TotalHitCountCollector counts num docs in the first leaf
@@ -501,7 +502,7 @@ public void testTerminateAfterEarlyTermination() throws Exception {
                     equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)
                 );
             } else {
-                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(7L));
+                assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) countDocUpTo.applyAsInt(7)));
                 assertThat(context.queryResult().topDocs().topDocs.totalHits.relation, equalTo(TotalHits.Relation.EQUAL_TO));
             }
             assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(7));