Refactor header warning pattern to avoid stackoverflow (elastic#96409)

an alternative combined with * in a regex is causing a stackoverflow when parsing
very long inputs (stack size have to be linearly correlated to the warning length).

This commit refactors the warning value aka qdText as per rfc7230 into iterative approach

closes elastic#95972

Author: pgomulka

server/src/main/java/org/elasticsearch/common/logging/HeaderWarning.java

@@ -23,6 +23,8 @@
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 /**
  * This is a simplistic logger that adds warning messages to HTTP headers.
@@ -39,7 +41,8 @@ public class HeaderWarning {
         "Elasticsearch-" + // warn agent
         "\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-" + // warn agent
         "(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent
-        "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\"( " + // quoted warning value, captured
+        // quoted warning value, captured. Do not add more greedy qualifiers later to avoid excessive backtracking
+        "\"(?<quotedStringValue>.*)\"( " +
         // quoted RFC 1123 date format
         "\"" + // opening quote
         "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday
@@ -48,7 +51,31 @@ public class HeaderWarning {
         "\\d{4} " + // 4-digit year
         "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second)
         "GMT" + // GMT
-        "\")?"); // closing quote (optional, since an older version can still send a warn-date)
+        "\")?",// closing quote (optional, since an older version can still send a warn-date)
+        Pattern.DOTALL
+    ); // in order to parse new line inside the qdText
+
+    /**
+     * quoted-string is defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
+     * quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
+     * qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
+     * obs-text       = %x80-FF
+     * <p>
+     * this was previously used in WARNING_HEADER_PATTERN, but can cause stackoverflow
+     * "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\"( " + // quoted warning value, captured
+     * the individual chars from qdText can be validated using the set of chars
+     * the \\\\|\\\\\" (escaped '\' 0x20 and '"' 0x5c) which is used for quoted-pair has to be validated as strings
+     */
+    private static BitSet qdTextChars = Stream.of(
+        IntStream.of(0x09),// HTAB
+        IntStream.of(0x20), // SPACE
+        IntStream.of(0x21), // !
+        // excluding 0x22 '"\"' which has to be escaped
+        IntStream.rangeClosed(0x23, 0x5B),// ascii #-[
+        // excluding 0x5c '\' which has to be escaped
+        IntStream.rangeClosed(0x5D, 0x7E),// ascii ]-~
+        IntStream.rangeClosed(0x80, 0xFF)// obs-text -bear in mind it contains 0x85 new line. Which requires DOT_ALL flag
+    ).flatMapToInt(i -> i).collect(BitSet::new, BitSet::set, BitSet::or);
     public static final Pattern WARNING_XCONTENT_LOCATION_PATTERN = Pattern.compile("^\\[.*?]\\[-?\\d+:-?\\d+] ");
 
     /*
@@ -182,7 +209,26 @@ private static boolean assertWarningValue(final String s, final String warningVa
         final Matcher matcher = WARNING_HEADER_PATTERN.matcher(s);
         final boolean matches = matcher.matches();
         assert matches;
-        return matcher.group(1).equals(warningValue);
+        String quotedStringValue = matcher.group("quotedStringValue");
+        assert matchesQuotedString(quotedStringValue);
+        return quotedStringValue.equals(warningValue);
+    }
+
+    // this is meant to be in testing only
+    public static boolean warningHeaderPatternMatches(final String s) {
+        final Matcher matcher = WARNING_HEADER_PATTERN.matcher(s);
+        final boolean matches = matcher.matches();
+        if (matches) {
+            String quotedStringValue = matcher.group("quotedStringValue");
+            return matchesQuotedString(quotedStringValue);
+        }
+        return false;
+    }
+
+    private static boolean matchesQuotedString(String qdtext) {
+        qdtext = qdtext.replaceAll("\\\\\"", "");
+        qdtext = qdtext.replaceAll("\\\\", "");
+        return qdtext.chars().allMatch(c -> qdTextChars.get(c));
     }
 
     /**
@@ -330,9 +376,8 @@ static void addWarning(Set<ThreadContext> threadContexts, String message, Object
         if (iterator.hasNext()) {
             final String formattedMessage = LoggerMessageFormat.format(message, params);
             final String warningHeaderValue = formatWarning(formattedMessage);
-            // TODO #95972: Temporarily commented out to avoid StackOverflowError, see https://github.com/elastic/elasticsearch/issues/95972
-            // assert WARNING_HEADER_PATTERN.matcher(warningHeaderValue).matches();
-            // assert extractWarningValueFromWarningHeader(warningHeaderValue, false).equals(escapeAndEncode(formattedMessage));
+            assert warningHeaderPatternMatches(warningHeaderValue);
+            assert extractWarningValueFromWarningHeader(warningHeaderValue, false).equals(escapeAndEncode(formattedMessage));
             while (iterator.hasNext()) {
                 try {
                     final ThreadContext next = iterator.next();

server/src/test/java/org/elasticsearch/common/logging/HeaderWarningTests.java

@@ -335,4 +335,45 @@ public void testAddComplexWarning() {
         assertThat(responses.get(0), containsString("\"legacy template [global] has index patterns"));
         assertThat(responses.get(0), containsString(Integer.toString(299)));
     }
+
+    public void testHeaderWarningValidation() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
+
+        HeaderWarning.addWarning(threadContexts, allAllowedChars());
+
+        // LoggerMessageFormat.format makes sure all not allowed chars are escaped
+        HeaderWarning.addWarning(threadContexts, "\"");
+        HeaderWarning.addWarning(threadContexts, "\\");
+        HeaderWarning.addWarning(threadContexts, allNotAllowedChars());
+    }
+
+    private String allNotAllowedChars() {
+        StringBuilder chars = new StringBuilder();
+        for (char c = 0; c < 256; c++) {
+            if (c < '\t' || ('\t' < c && c < 0x20) || c == 0x7f) {
+                chars.append(c);
+            }
+        }
+        return chars.toString();
+    }
+
+    private static String allAllowedChars() {
+        StringBuilder allPossibleChars = new StringBuilder();
+        allPossibleChars.append('\t');
+        allPossibleChars.append(' ');
+        allPossibleChars.append('!');
+        for (int i = 0x23; i <= 0x5b; i++) {
+            allPossibleChars.append((char) i);
+        }
+        for (int i = 0x5d; i <= 0x7e; i++) {
+            allPossibleChars.append((char) i);
+        }
+        for (int i = 0x80; i <= 0xff; i++) {
+            allPossibleChars.append((char) i);
+        }
+        allPossibleChars.append("\\");
+        allPossibleChars.append("\\\"");
+        return allPossibleChars.toString();
+    }
 }

test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java

@@ -38,7 +38,6 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
@@ -438,8 +437,7 @@ void checkWarningHeaders(final List<String> warningHeaders, String testPath) {
             .collect(toCollection(LinkedHashSet::new));
         final Set<Pattern> expectedRegex = new LinkedHashSet<>(expectedWarningHeadersRegex);
         for (final String header : warningHeaders) {
-            final Matcher matcher = HeaderWarning.WARNING_HEADER_PATTERN.matcher(header);
-            final boolean matches = matcher.matches();
+            final boolean matches = HeaderWarning.warningHeaderPatternMatches(header);
             if (matches) {
                 final String message = HeaderWarning.extractWarningValueFromWarningHeader(header, true);
                 if (allowed.contains(message)) {