Refactor nested field handling in FieldFetcher (elastic#97683)

The current recursive nested field handling implementation in FieldFetcher
can be O(n^2) in the number of nested mappings, whether or not a nested
field has been requested or not. For indexes with a very large number of
nested fields, this can mean it takes multiple seconds to build a FieldFetcher,
making the fetch phase of queries extremely slow, even if no nested fields
are actually asked for.

This commit reworks the logic so that building nested fetchers is only
O(n log n) in the number of nested mappers; additionally, we only pay this
cost for nested fields that have been requested.

Author: romseygeek

docs/changelog/97683.yaml

@@ -0,0 +1,5 @@
+pr: 97683
+summary: Refactor nested field handling in `FieldFetcher`
+area: Search
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/NestedLookup.java

@@ -10,6 +10,7 @@
 
 import org.apache.lucene.search.Query;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -42,6 +43,12 @@ public interface NestedLookup {
      */
     String getNestedParent(String path);
 
+    /**
+     * Given a nested object path, returns a list of paths of its
+     * immediate children
+     */
+    List<String> getImmediateChildMappers(String path);
+
     /**
      * A NestedLookup for a mapping with no nested mappers
      */
@@ -60,6 +67,11 @@ public Map<String, Query> getNestedParentFilters() {
         public String getNestedParent(String path) {
             return null;
         }
+
+        @Override
+        public List<String> getImmediateChildMappers(String path) {
+            return List.of();
+        }
     };
 
     /**
@@ -84,6 +96,7 @@ static NestedLookup build(List<NestedObjectMapper> mappers) {
             previous = mapper;
         }
         List<String> nestedPathNames = mappers.stream().map(NestedObjectMapper::name).toList();
+
         return new NestedLookup() {
 
             @Override
@@ -98,6 +111,9 @@ public Map<String, Query> getNestedParentFilters() {
 
             @Override
             public String getNestedParent(String path) {
+                if (path.contains(".") == false) {
+                    return null;
+                }
                 String parent = null;
                 for (String parentPath : nestedPathNames) {
                     if (path.startsWith(parentPath + ".")) {
@@ -108,6 +124,33 @@ public String getNestedParent(String path) {
                 }
                 return parent;
             }
+
+            @Override
+            public List<String> getImmediateChildMappers(String path) {
+                String prefix = "".equals(path) ? "" : path + ".";
+                List<String> childMappers = new ArrayList<>();
+                int parentPos = Collections.binarySearch(nestedPathNames, path);
+                if (parentPos < -1 || parentPos >= nestedPathNames.size() - 1) {
+                    return List.of();
+                }
+                int i = parentPos + 1;
+                String lastChild = nestedPathNames.get(i);
+                if (lastChild.startsWith(prefix)) {
+                    childMappers.add(lastChild);
+                }
+                i++;
+                while (i < nestedPathNames.size() && nestedPathNames.get(i).startsWith(prefix)) {
+                    if (nestedPathNames.get(i).startsWith(lastChild + ".")) {
+                        // child of child, skip
+                        i++;
+                        continue;
+                    }
+                    lastChild = nestedPathNames.get(i);
+                    childMappers.add(lastChild);
+                    i++;
+                }
+                return childMappers;
+            }
         };
     }
 }

server/src/main/java/org/elasticsearch/search/NestedUtils.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Utility methods for dealing with nested mappers
+ */
+public final class NestedUtils {
+
+    private NestedUtils() {}
+
+    /**
+     * Partition a set of input objects by the children of a specific nested scope
+     *
+     * The returned map will contain an entry for all children, even if some of them
+     * are empty in the inputs.
+     *
+     * All children, and all input paths, must begin with the scope.  Both children
+     * and inputs should be in sorted order.
+     *
+     * @param scope         the nested scope to base partitions on
+     * @param children      the immediate children of the nested scope
+     * @param inputs        a set of inputs to partition
+     * @param pathFunction  a function to retrieve a path for each input
+     * @param <T>           the type of the inputs
+     * @return              a map of nested paths to lists of inputs
+     */
+    public static <T> Map<String, List<T>> partitionByChildren(
+        String scope,
+        List<String> children,
+        List<T> inputs,
+        Function<T, String> pathFunction
+    ) {
+        // No immediate nested children, so we can shortcut and just return all inputs
+        // under the current scope
+        if (children.isEmpty()) {
+            return Map.of(scope, inputs);
+        }
+
+        // Set up the output map, with one entry for the current scope and one for each
+        // of its children
+        Map<String, List<T>> output = new HashMap<>();
+        output.put(scope, new ArrayList<>());
+        for (String child : children) {
+            output.put(child, new ArrayList<>());
+        }
+
+        // No inputs, so we can return the output map with all entries empty
+        if (inputs.isEmpty()) {
+            return output;
+        }
+
+        Iterator<String> childrenIterator = children.iterator();
+        String currentChild = childrenIterator.next();
+        Iterator<T> inputIterator = inputs.iterator();
+        T currentInput = inputIterator.next();
+        String currentInputName = pathFunction.apply(currentInput);
+        assert currentInputName.startsWith(scope);
+
+        // Find all the inputs that sort before the first child, and add them to the current scope entry
+        while (currentInputName.compareTo(currentChild) < 0) {
+            output.get(scope).add(currentInput);
+            if (inputIterator.hasNext() == false) {
+                return output;
+            }
+            currentInput = inputIterator.next();
+            currentInputName = pathFunction.apply(currentInput);
+            assert currentInputName.startsWith(scope);
+        }
+
+        // Iterate through all the children
+        while (currentChild != null) {
+            if (currentInputName.startsWith(currentChild + ".")) {
+                // If this input sits under the current child, add it to that child scope
+                // and then get the next input
+                output.get(currentChild).add(currentInput);
+                if (inputIterator.hasNext() == false) {
+                    // return if no more inputs
+                    return output;
+                }
+                currentInput = inputIterator.next();
+                currentInputName = pathFunction.apply(currentInput);
+                assert currentInputName.startsWith(scope);
+            } else {
+                // If there are no more children then skip to filling up the parent scope again
+                if (childrenIterator.hasNext() == false) {
+                    break;
+                }
+                // Move to the next child
+                currentChild = childrenIterator.next();
+                if (currentChild == null || currentInputName.compareTo(currentChild) < 0) {
+                    // If we still sort before the next child, then add to the parent scope
+                    // and move to the next input
+                    output.get(scope).add(currentInput);
+                    if (inputIterator.hasNext() == false) {
+                        // if no more inputs then return
+                        return output;
+                    }
+                    currentInput = inputIterator.next();
+                    currentInputName = pathFunction.apply(currentInput);
+                    assert currentInputName.startsWith(scope);
+                }
+            }
+        }
+        output.get(scope).add(currentInput);
+
+        // if there are inputs left, then they all sort after the last child but
+        // are not contained by them, so just add them all to the parent scope
+        while (inputIterator.hasNext()) {
+            currentInput = inputIterator.next();
+            currentInputName = pathFunction.apply(currentInput);
+            assert currentInputName.startsWith(scope);
+            output.get(scope).add(currentInput);
+        }
+        return output;
+    }
+
+}